From 722aab33d747a7588bf15ee7acc169491fcb6bfb Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Tue, 5 May 2026 13:11:26 -0400 Subject: [PATCH 01/77] Initial release --- .claude-plugin/plugin.json | 17 + .github/ISSUE_TEMPLATE/bug_report.yml | 47 + .github/ISSUE_TEMPLATE/feature_request.yml | 32 + .github/pull_request_template.md | 14 + .github/workflows/docs-release.yml | 69 + .github/workflows/push.yml | 40 + .github/workflows/skill-eval.yml | 30 + .gitignore | 47 + AGENTS.md | 89 ++ CHANGELOG.md | 10 + CLAUDE.md | 1 + LICENSE | 201 +++ Makefile | 48 + README.md | 145 ++ docs/.gitignore | 8 + docs/README.md | 33 + docs/app/(home)/layout.tsx | 7 + docs/app/(home)/page.tsx | 27 + docs/app/docs/[[...slug]]/page.tsx | 39 + docs/app/docs/layout.tsx | 12 + docs/app/global.css | 5 + docs/app/layout.config.tsx | 22 + docs/app/layout.tsx | 27 + docs/bun.lock | 762 +++++++++ docs/content/docs/index.mdx | 36 + docs/content/docs/installation.mdx | 73 + docs/content/docs/meta.json | 10 + docs/content/docs/usage-guide.mdx | 85 + docs/lib/source.ts | 7 + docs/next.config.mjs | 18 + docs/package.json | 29 + docs/postcss.config.mjs | 5 + docs/source.config.ts | 7 + docs/tsconfig.json | 45 + pyproject.toml | 68 + skills/ingest/SKILL.md | 190 +++ skills/migrate/SKILL.md | 201 +++ skills/migrate/references/workflow.md | 167 ++ skills/prepare/SKILL.md | 163 ++ skills/translate/SKILL.md | 207 +++ .../translate/references/activity-mapping.md | 174 ++ .../references/expression-functions.md | 154 ++ src/AGENTS.md | 103 ++ src/orchestra/__init__.py | 3 + src/orchestra/bundler/__init__.py | 11 + src/orchestra/bundler/dab_writer.py | 1020 ++++++++++++ src/orchestra/bundler/inner_job_params.py | 363 +++++ src/orchestra/bundler/notebook_writer.py | 22 + src/orchestra/bundler/prereqs_writer.py | 538 +++++++ src/orchestra/bundler/setup_generator.py | 308 ++++ src/orchestra/models/__init__.py | 0 src/orchestra/models/adf_ast.py | 294 ++++ src/orchestra/models/dab.py | 149 ++ src/orchestra/models/ir.py | 560 +++++++ src/orchestra/models/motifs.py | 196 +++ src/orchestra/models/source_types.py | 50 + src/orchestra/motifs/__init__.py | 6 + src/orchestra/motifs/collapser.py | 202 +++ src/orchestra/motifs/detector.py | 840 ++++++++++ src/orchestra/parser/__init__.py | 0 src/orchestra/parser/adf_loader.py | 698 ++++++++ src/orchestra/parser/expression_parser.py | 1397 ++++++++++++++++ src/orchestra/preparer/__init__.py | 15 + .../preparer/activity_preparers/__init__.py | 1 + .../activity_preparers/append_variable.py | 38 + .../preparer/activity_preparers/copy.py | 175 ++ .../activity_preparers/databricks_job.py | 58 + .../preparer/activity_preparers/delete.py | 34 + .../activity_preparers/execute_pipeline.py | 55 + .../preparer/activity_preparers/filter.py | 32 + .../preparer/activity_preparers/for_each.py | 165 ++ .../preparer/activity_preparers/helpers.py | 86 + .../activity_preparers/if_condition.py | 93 ++ .../preparer/activity_preparers/lookup.py | 42 + .../preparer/activity_preparers/motif.py | 22 + .../preparer/activity_preparers/naming.py | 30 + .../preparer/activity_preparers/notebook.py | 133 ++ .../activity_preparers/set_variable.py | 30 + .../preparer/activity_preparers/spark_jar.py | 98 ++ .../activity_preparers/spark_python.py | 72 + .../preparer/activity_preparers/switch.py | 156 ++ .../preparer/activity_preparers/wait.py | 24 + .../activity_preparers/web_activity.py | 43 + src/orchestra/preparer/code_generator.py | 1410 +++++++++++++++++ src/orchestra/preparer/workflow_preparer.py | 281 ++++ .../preparer/workspace_downloader.py | 210 +++ src/orchestra/translator/__init__.py | 0 .../activity_translators/__init__.py | 0 .../activity_translators/append_variable.py | 63 + .../translator/activity_translators/copy.py | 368 +++++ .../activity_translators/databricks_job.py | 44 + .../translator/activity_translators/delete.py | 56 + .../activity_translators/execute_pipeline.py | 46 + .../translator/activity_translators/filter.py | 69 + .../activity_translators/for_each.py | 77 + .../activity_translators/if_condition.py | 242 +++ .../translator/activity_translators/lookup.py | 48 + .../activity_translators/notebook.py | 51 + .../activity_translators/resolve.py | 64 + .../activity_translators/set_variable.py | 77 + .../activity_translators/spark_jar.py | 70 + .../activity_translators/spark_python.py | 65 + .../translator/activity_translators/switch.py | 121 ++ .../translator/activity_translators/wait.py | 36 + .../activity_translators/web_activity.py | 104 ++ src/orchestra/translator/engine.py | 807 ++++++++++ src/orchestra/utils.py | 129 ++ tests/conftest.py | 22 + tests/integration/__init__.py | 0 tests/integration/conftest.py | 28 + tests/integration/test_adf_live.py | 377 +++++ tests/integration/test_end_to_end.py | 363 +++++ tests/integration/test_golden_output.py | 524 ++++++ .../json/datasets/dataset_avro_adls.json | 43 + .../json/datasets/dataset_azure_blob.json | 47 + .../datasets/dataset_azure_sql_table.json | 73 + .../json/datasets/dataset_csv_adls.json | 59 + .../json/datasets/dataset_delta_table.json | 49 + .../json/datasets/dataset_json_adls.json | 41 + .../json/datasets/dataset_mysql_table.json | 68 + .../json/datasets/dataset_orc_adls.json | 43 + .../json/datasets/dataset_parquet_adls.json | 44 + .../datasets/dataset_postgresql_table.json | 55 + .../json/datasets/ds_csv_adls_customers.json | 22 + .../json/datasets/ds_delta_customers.json | 13 + .../json/linked_services/ls_adls_gen2.json | 26 + .../json/linked_services/ls_azure_blob.json | 26 + .../json/linked_services/ls_azure_sql.json | 26 + .../ls_databricks_existing_cluster.json | 29 + .../ls_databricks_new_cluster.json | 52 + .../json/linked_services/ls_key_vault.json | 14 + .../json/linked_services/ls_postgresql.json | 26 + .../pipeline_all_activity_types.json | 261 +++ .../pipeline_all_dependency_conditions.json | 243 +++ .../pipeline_append_variable_loop.json | 136 ++ .../json/pipelines/pipeline_complex_etl.json | 324 ++++ .../pipeline_complex_orchestration.json | 336 ++++ .../pipelines/pipeline_copy_csv_to_delta.json | 116 ++ .../pipeline_copy_parquet_to_delta.json | 93 ++ .../pipelines/pipeline_copy_sql_to_delta.json | 114 ++ .../pipelines/pipeline_delete_recursive.json | 114 ++ .../pipeline_execute_pipeline_nested.json | 93 ++ .../json/pipelines/pipeline_filter_array.json | 115 ++ .../pipelines/pipeline_foreach_switch.json | 129 ++ .../pipelines/pipeline_foreach_with_copy.json | 93 ++ .../pipeline_if_condition_branching.json | 164 ++ .../pipeline_lookup_and_foreach.json | 114 ++ .../pipelines/pipeline_mixed_agentic.json | 108 ++ .../pipeline_mixed_deterministic_agentic.json | 210 +++ .../pipelines/pipeline_notebook_basic.json | 35 + .../pipeline_notebook_with_params.json | 68 + .../pipeline_set_variable_chain.json | 151 ++ .../pipelines/pipeline_spark_jar_job.json | 74 + .../pipelines/pipeline_spark_python_job.json | 98 ++ .../pipelines/pipeline_switch_multi_case.json | 243 +++ .../pipeline_wait_between_steps.json | 138 ++ .../pipelines/pipeline_web_activity_auth.json | 119 ++ .../pl_test_appendvariable_coverage.json | 82 + .../json/pipelines/pl_test_copy_coverage.json | 196 +++ .../pipelines/pl_test_delete_coverage.json | 51 + .../pl_test_executepipeline_coverage.json | 75 + .../pipelines/pl_test_filter_coverage.json | 107 ++ .../pipelines/pl_test_foreach_coverage.json | 87 + .../pl_test_ifcondition_coverage.json | 205 +++ .../pipelines/pl_test_lookup_coverage.json | 107 ++ .../pipelines/pl_test_notebook_coverage.json | 103 ++ .../pl_test_setvariable_coverage.json | 132 ++ .../pipelines/pl_test_sparkjar_coverage.json | 71 + .../pl_test_sparkpython_coverage.json | 78 + .../pipelines/pl_test_switch_coverage.json | 126 ++ .../json/pipelines/pl_test_wait_coverage.json | 51 + .../pl_test_webactivity_coverage.json | 131 ++ .../json/triggers/tr_daily_schedule.json | 22 + .../json/triggers/trigger_blob_event.json | 33 + .../json/triggers/trigger_schedule.json | 58 + .../triggers/trigger_tumbling_window.json | 66 + tests/unit/__init__.py | 0 tests/unit/conftest.py | 1 + tests/unit/test_adf_loader.py | 304 ++++ tests/unit/test_bundler.py | 312 ++++ tests/unit/test_code_generator.py | 507 ++++++ tests/unit/test_expression_parser.py | 756 +++++++++ tests/unit/test_helpers.py | 230 +++ tests/unit/test_motifs.py | 252 +++ tests/unit/test_preparers.py | 776 +++++++++ tests/unit/test_resolve_field.py | 149 ++ tests/unit/test_translators.py | 642 ++++++++ tests/unit/test_workspace_downloader.py | 52 + uv.lock | 387 +++++ 189 files changed, 27879 insertions(+) create mode 100644 .claude-plugin/plugin.json create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/docs-release.yml create mode 100644 .github/workflows/push.yml create mode 100644 .github/workflows/skill-eval.yml create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 CHANGELOG.md create mode 100644 CLAUDE.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 docs/.gitignore create mode 100644 docs/README.md create mode 100644 docs/app/(home)/layout.tsx create mode 100644 docs/app/(home)/page.tsx create mode 100644 docs/app/docs/[[...slug]]/page.tsx create mode 100644 docs/app/docs/layout.tsx create mode 100644 docs/app/global.css create mode 100644 docs/app/layout.config.tsx create mode 100644 docs/app/layout.tsx create mode 100644 docs/bun.lock create mode 100644 docs/content/docs/index.mdx create mode 100644 docs/content/docs/installation.mdx create mode 100644 docs/content/docs/meta.json create mode 100644 docs/content/docs/usage-guide.mdx create mode 100644 docs/lib/source.ts create mode 100644 docs/next.config.mjs create mode 100644 docs/package.json create mode 100644 docs/postcss.config.mjs create mode 100644 docs/source.config.ts create mode 100644 docs/tsconfig.json create mode 100644 pyproject.toml create mode 100644 skills/ingest/SKILL.md create mode 100644 skills/migrate/SKILL.md create mode 100644 skills/migrate/references/workflow.md create mode 100644 skills/prepare/SKILL.md create mode 100644 skills/translate/SKILL.md create mode 100644 skills/translate/references/activity-mapping.md create mode 100644 skills/translate/references/expression-functions.md create mode 100644 src/AGENTS.md create mode 100644 src/orchestra/__init__.py create mode 100644 src/orchestra/bundler/__init__.py create mode 100644 src/orchestra/bundler/dab_writer.py create mode 100644 src/orchestra/bundler/inner_job_params.py create mode 100644 src/orchestra/bundler/notebook_writer.py create mode 100644 src/orchestra/bundler/prereqs_writer.py create mode 100644 src/orchestra/bundler/setup_generator.py create mode 100644 src/orchestra/models/__init__.py create mode 100644 src/orchestra/models/adf_ast.py create mode 100644 src/orchestra/models/dab.py create mode 100644 src/orchestra/models/ir.py create mode 100644 src/orchestra/models/motifs.py create mode 100644 src/orchestra/models/source_types.py create mode 100644 src/orchestra/motifs/__init__.py create mode 100644 src/orchestra/motifs/collapser.py create mode 100644 src/orchestra/motifs/detector.py create mode 100644 src/orchestra/parser/__init__.py create mode 100644 src/orchestra/parser/adf_loader.py create mode 100644 src/orchestra/parser/expression_parser.py create mode 100644 src/orchestra/preparer/__init__.py create mode 100644 src/orchestra/preparer/activity_preparers/__init__.py create mode 100644 src/orchestra/preparer/activity_preparers/append_variable.py create mode 100644 src/orchestra/preparer/activity_preparers/copy.py create mode 100644 src/orchestra/preparer/activity_preparers/databricks_job.py create mode 100644 src/orchestra/preparer/activity_preparers/delete.py create mode 100644 src/orchestra/preparer/activity_preparers/execute_pipeline.py create mode 100644 src/orchestra/preparer/activity_preparers/filter.py create mode 100644 src/orchestra/preparer/activity_preparers/for_each.py create mode 100644 src/orchestra/preparer/activity_preparers/helpers.py create mode 100644 src/orchestra/preparer/activity_preparers/if_condition.py create mode 100644 src/orchestra/preparer/activity_preparers/lookup.py create mode 100644 src/orchestra/preparer/activity_preparers/motif.py create mode 100644 src/orchestra/preparer/activity_preparers/naming.py create mode 100644 src/orchestra/preparer/activity_preparers/notebook.py create mode 100644 src/orchestra/preparer/activity_preparers/set_variable.py create mode 100644 src/orchestra/preparer/activity_preparers/spark_jar.py create mode 100644 src/orchestra/preparer/activity_preparers/spark_python.py create mode 100644 src/orchestra/preparer/activity_preparers/switch.py create mode 100644 src/orchestra/preparer/activity_preparers/wait.py create mode 100644 src/orchestra/preparer/activity_preparers/web_activity.py create mode 100644 src/orchestra/preparer/code_generator.py create mode 100644 src/orchestra/preparer/workflow_preparer.py create mode 100644 src/orchestra/preparer/workspace_downloader.py create mode 100644 src/orchestra/translator/__init__.py create mode 100644 src/orchestra/translator/activity_translators/__init__.py create mode 100644 src/orchestra/translator/activity_translators/append_variable.py create mode 100644 src/orchestra/translator/activity_translators/copy.py create mode 100644 src/orchestra/translator/activity_translators/databricks_job.py create mode 100644 src/orchestra/translator/activity_translators/delete.py create mode 100644 src/orchestra/translator/activity_translators/execute_pipeline.py create mode 100644 src/orchestra/translator/activity_translators/filter.py create mode 100644 src/orchestra/translator/activity_translators/for_each.py create mode 100644 src/orchestra/translator/activity_translators/if_condition.py create mode 100644 src/orchestra/translator/activity_translators/lookup.py create mode 100644 src/orchestra/translator/activity_translators/notebook.py create mode 100644 src/orchestra/translator/activity_translators/resolve.py create mode 100644 src/orchestra/translator/activity_translators/set_variable.py create mode 100644 src/orchestra/translator/activity_translators/spark_jar.py create mode 100644 src/orchestra/translator/activity_translators/spark_python.py create mode 100644 src/orchestra/translator/activity_translators/switch.py create mode 100644 src/orchestra/translator/activity_translators/wait.py create mode 100644 src/orchestra/translator/activity_translators/web_activity.py create mode 100644 src/orchestra/translator/engine.py create mode 100644 src/orchestra/utils.py create mode 100644 tests/conftest.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/test_adf_live.py create mode 100644 tests/integration/test_end_to_end.py create mode 100644 tests/integration/test_golden_output.py create mode 100644 tests/resources/json/datasets/dataset_avro_adls.json create mode 100644 tests/resources/json/datasets/dataset_azure_blob.json create mode 100644 tests/resources/json/datasets/dataset_azure_sql_table.json create mode 100644 tests/resources/json/datasets/dataset_csv_adls.json create mode 100644 tests/resources/json/datasets/dataset_delta_table.json create mode 100644 tests/resources/json/datasets/dataset_json_adls.json create mode 100644 tests/resources/json/datasets/dataset_mysql_table.json create mode 100644 tests/resources/json/datasets/dataset_orc_adls.json create mode 100644 tests/resources/json/datasets/dataset_parquet_adls.json create mode 100644 tests/resources/json/datasets/dataset_postgresql_table.json create mode 100644 tests/resources/json/datasets/ds_csv_adls_customers.json create mode 100644 tests/resources/json/datasets/ds_delta_customers.json create mode 100644 tests/resources/json/linked_services/ls_adls_gen2.json create mode 100644 tests/resources/json/linked_services/ls_azure_blob.json create mode 100644 tests/resources/json/linked_services/ls_azure_sql.json create mode 100644 tests/resources/json/linked_services/ls_databricks_existing_cluster.json create mode 100644 tests/resources/json/linked_services/ls_databricks_new_cluster.json create mode 100644 tests/resources/json/linked_services/ls_key_vault.json create mode 100644 tests/resources/json/linked_services/ls_postgresql.json create mode 100644 tests/resources/json/pipelines/pipeline_all_activity_types.json create mode 100644 tests/resources/json/pipelines/pipeline_all_dependency_conditions.json create mode 100644 tests/resources/json/pipelines/pipeline_append_variable_loop.json create mode 100644 tests/resources/json/pipelines/pipeline_complex_etl.json create mode 100644 tests/resources/json/pipelines/pipeline_complex_orchestration.json create mode 100644 tests/resources/json/pipelines/pipeline_copy_csv_to_delta.json create mode 100644 tests/resources/json/pipelines/pipeline_copy_parquet_to_delta.json create mode 100644 tests/resources/json/pipelines/pipeline_copy_sql_to_delta.json create mode 100644 tests/resources/json/pipelines/pipeline_delete_recursive.json create mode 100644 tests/resources/json/pipelines/pipeline_execute_pipeline_nested.json create mode 100644 tests/resources/json/pipelines/pipeline_filter_array.json create mode 100644 tests/resources/json/pipelines/pipeline_foreach_switch.json create mode 100644 tests/resources/json/pipelines/pipeline_foreach_with_copy.json create mode 100644 tests/resources/json/pipelines/pipeline_if_condition_branching.json create mode 100644 tests/resources/json/pipelines/pipeline_lookup_and_foreach.json create mode 100644 tests/resources/json/pipelines/pipeline_mixed_agentic.json create mode 100644 tests/resources/json/pipelines/pipeline_mixed_deterministic_agentic.json create mode 100644 tests/resources/json/pipelines/pipeline_notebook_basic.json create mode 100644 tests/resources/json/pipelines/pipeline_notebook_with_params.json create mode 100644 tests/resources/json/pipelines/pipeline_set_variable_chain.json create mode 100644 tests/resources/json/pipelines/pipeline_spark_jar_job.json create mode 100644 tests/resources/json/pipelines/pipeline_spark_python_job.json create mode 100644 tests/resources/json/pipelines/pipeline_switch_multi_case.json create mode 100644 tests/resources/json/pipelines/pipeline_wait_between_steps.json create mode 100644 tests/resources/json/pipelines/pipeline_web_activity_auth.json create mode 100644 tests/resources/json/pipelines/pl_test_appendvariable_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_copy_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_delete_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_executepipeline_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_filter_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_foreach_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_ifcondition_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_lookup_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_notebook_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_setvariable_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_sparkjar_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_sparkpython_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_switch_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_wait_coverage.json create mode 100644 tests/resources/json/pipelines/pl_test_webactivity_coverage.json create mode 100644 tests/resources/json/triggers/tr_daily_schedule.json create mode 100644 tests/resources/json/triggers/trigger_blob_event.json create mode 100644 tests/resources/json/triggers/trigger_schedule.json create mode 100644 tests/resources/json/triggers/trigger_tumbling_window.json create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/conftest.py create mode 100644 tests/unit/test_adf_loader.py create mode 100644 tests/unit/test_bundler.py create mode 100644 tests/unit/test_code_generator.py create mode 100644 tests/unit/test_expression_parser.py create mode 100644 tests/unit/test_helpers.py create mode 100644 tests/unit/test_motifs.py create mode 100644 tests/unit/test_preparers.py create mode 100644 tests/unit/test_resolve_field.py create mode 100644 tests/unit/test_translators.py create mode 100644 tests/unit/test_workspace_downloader.py create mode 100644 uv.lock diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..28e8695 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,17 @@ +{ + "name": "flowx", + "version": "0.2.0", + "description": "Translate Azure Data Factory pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. Deterministic translation for known activity types with agentic fallback.", + "author": { + "name": "Greg Hansen", + "email": "gregory.hansen@databricks.com" + }, + "license": "MIT", + "keywords": ["adf", "databricks", "migration", "dabs", "lakeflow", "orchestration", "azure-data-factory"], + "skills": [ + "./skills/ingest", + "./skills/translate", + "./skills/prepare", + "./skills/migrate" + ] +} diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..78d82a7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,47 @@ +# See https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms +# and https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-githubs-form-schema +name: Bug Report +description: Something is not working with Flowx +title: "[BUG]: " +labels: ["bug"] +body: + - type: checkboxes + attributes: + label: Is there an existing issue for this? + description: Please search to see if an issue already exists for the bug you encountered. + options: + - label: I have searched the existing issues + required: true + - type: textarea + attributes: + label: Current Behavior + description: | + A concise description of what you're experiencing. + **Do not paste links to attachments with logs and/or images, as all issues will attachments will get deleted.** + Use the `Relevant log output` field to paste redacted log output without personal identifying information (PII). + You can Ctrl/Cmd+V the screenshot, which would appear as a rendered image if it doesn't contain any PII. + validations: + required: true + - type: textarea + attributes: + label: Expected Behavior + description: A concise description of what you expected to happen. + validations: + required: true + - type: textarea + attributes: + label: Steps To Reproduce + description: Steps to reproduce the behavior. + placeholder: | + 1. In this environment... + 1. With this config... + 1. Run '...' + 1. See error... + validations: + required: true + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..b8f2257 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,32 @@ +# See https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms +# and https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-githubs-form-schema +name: Feature Request +description: Something new needs to happen with Flowx +title: "[FEATURE]: " +labels: ["enhancement"] +body: + - type: checkboxes + attributes: + label: Is there an existing issue for this? + description: Please search to see if an issue already exists for the feature request you're willing to submit + options: + - label: I have searched the existing issues + required: true + - type: textarea + attributes: + label: Problem statement + description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + validations: + required: true + - type: textarea + attributes: + label: Proposed Solution + description: A clear and concise description of what you want to happen. + validations: + required: true + - type: textarea + attributes: + label: Additional Context + description: Add any other context, references or screenshots about the feature request here. + validations: + required: false diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..75a9f2a --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,14 @@ +## Changes + + +### Linked issues + + +Resolves #.. + +### Tests + + +- [ ] manually tested +- [ ] added unit tests +- [ ] added integration tests diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml new file mode 100644 index 0000000..d71e6ab --- /dev/null +++ b/.github/workflows/docs-release.yml @@ -0,0 +1,69 @@ +name: Docs Release + +on: + push: + branches: + - main + paths: + - 'docs/**' + - '.github/workflows/docs-release.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + name: Build documentation site + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: latest + + - name: Scrub internal proxy URLs from bun.lock + # The Databricks-internal npm proxy is unreachable from public runners; + # rewrite to the default npm registry. Hashes are unaffected by the URL. + run: sed -i 's|https://npm-proxy\.dev\.databricks\.com|https://registry.npmjs.org|g' docs/bun.lock + + - name: Install dependencies + run: make docs-install + + - name: Build website + run: make docs-build + + - name: Add .nojekyll + run: touch docs/site/.nojekyll + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 + with: + path: docs/site + + deploy: + name: Deploy to GitHub Pages + needs: build + runs-on: ubuntu-latest + + permissions: + pages: write + id-token: write + + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..97d7338 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,40 @@ +name: build + +on: + pull_request: + push: + branches: [main] + +jobs: + ci: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Scrub internal proxy URLs from uv.lock + run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock + - run: uv sync --frozen + - run: make test + + fmt: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Scrub internal proxy URLs from uv.lock + run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock + - run: uv sync --frozen + - run: make fmt + - name: Check for formatting changes + run: | + if [ -n "$(git diff)" ]; then + echo "Code formatting issues detected. Run 'make fmt' locally." + git diff + exit 1 + fi diff --git a/.github/workflows/skill-eval.yml b/.github/workflows/skill-eval.yml new file mode 100644 index 0000000..c1a93d6 --- /dev/null +++ b/.github/workflows/skill-eval.yml @@ -0,0 +1,30 @@ +name: skill-eval + +on: + pull_request: + workflow_dispatch: + +jobs: + integration: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Scrub internal proxy URLs from uv.lock + run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock + - run: uv sync --frozen + + # Azure login for live ADF integration tests. + # Requires AZURE_CREDENTIALS secret configured with a service principal + # that has Reader access to the flowx-rg resource group. + # Tests skip gracefully when credentials are not available. + - name: Azure Login + if: ${{ secrets.AZURE_CREDENTIALS != '' }} + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - run: make integration diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..20db1d0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,47 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +*.egg-info/ +*.egg +dist/ +build/ +.eggs/ + +# Virtual environments +.venv/ +venv/ +ENV/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.mypy_cache/ +.ruff_cache/ + +# OS +.DS_Store +Thumbs.db + +# Flowx output directories +orchestra_output/ +dab_output/ + +# Temporary ingest downloads +tmp_adf_ingest_*/ + +# Secrets and credentials +.env +.env.* +*.pem +*.key +credentials.json diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..dc79f38 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,89 @@ +# AI Agent Guidelines for Flowx + +## Quick Command Reference + +```bash +make dev # Install dependencies +make test # Unit tests +make integration # Integration tests (requires ADF fixtures) +make fmt # Format + lint (ruff + mypy) +make clean # Remove build artifacts +``` + +## Project Overview + +Flowx is an agent plugin that translates Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs via Declarative Automation Bundles (DABs). + +## Data Flow + +``` +ADF JSON -> Parse (AST) -> Classify (Inventory) -> Translate (IR) -> Prepare (Tasks + Notebooks) -> Bundle (DABs) +``` + +## Architecture + +### Three-Phase Pipeline +1. **Ingest** -- Parse ADF JSON from UC volumes -> typed AST -> inventory.json +2. **Translate** -- Registry dispatch + topological sort -> Pipeline IR (deterministic + agentic gaps) +3. **Prepare** -- IR -> DAB YAML + generated notebooks + setup scripts + +### Key Patterns +- `@dataclass(slots=True, kw_only=True)` for all models +- Immutable `TranslationContext` threaded through visitors +- Registry-based dispatch with match statement for control-flow types +- `TranslationStrategy` enum: DETERMINISTIC > AGENTIC > UNSUPPORTED + +## Module Descriptions + +| Module | Purpose | +|--------|---------| +| `models/adf_ast.py` | Typed AST nodes for ADF definitions | +| `models/ir.py` | Databricks intermediate representation | +| `models/dab.py` | DAB output schema types | +| `parser/adf_loader.py` | Parses ADF exports, produces inventory.json | +| `parser/expression_parser.py` | Translates ADF expressions (@activity, @pipeline, @variables) | +| `translator/engine.py` | Registry dispatch, topological sort, context threading | +| `translator/activity_translators/` | One module per deterministic activity type (16 total) | +| `preparer/workflow_preparer.py` | Orchestrates activity preparers | +| `preparer/code_generator.py` | Notebook code generation for activity types | +| `preparer/activity_preparers/` | One module per activity type | +| `bundler/dab_writer.py` | Generates databricks.yml, job YAML, resources | +| `bundler/notebook_writer.py` | Writes generated notebooks to bundle | +| `bundler/setup_generator.py` | Setup scripts for UC volumes, secrets, connections | + +## Activity Types + +### Deterministic Types (16) +Copy, DatabricksNotebook, DatabricksSparkJar, DatabricksSparkPython, ForEach, IfCondition, SetVariable, Lookup, WebActivity, Delete, ExecutePipeline, DatabricksJob, Switch, Wait, Filter, AppendVariable + +### Agentic Fallback Types (12) +ExecuteDataFlow, SqlServerStoredProcedure, AzureFunction, WebHook, Custom, ExecuteSSISPackage, AzureMLExecutePipeline, GetMetadata, Validation, Fail, Script, Until + +## Testing Standards + +- Unit tests in `tests/unit/`, one test file per translator +- Integration tests in `tests/integration/`, require ADF fixture files +- Test fixtures in `tests/resources/json/` +- Run `make test` for unit tests, `make integration` for integration tests +- All translators must have corresponding test coverage + +## Code Style Rules + +- Python 3.12+, line length 120 characters +- `ruff` for formatting and linting, `mypy` for type checking +- `@dataclass(slots=True, kw_only=True)` for all models +- Never modify TranslationContext in place -- always return a new instance +- Control-flow types (ForEach, IfCondition, Switch, SetVariable, AppendVariable) thread context +- Leaf types return Activity only, control-flow returns (Activity, TranslationContext) +- Use `parse_expression()` for ADF expression translation, return None for unsupported + +## Adding a New Deterministic Translator + +1. Add IR dataclass to `src/flowx/models/ir.py` +2. Create translator at `src/flowx/translator/activity_translators/.py` +3. Create preparer at `src/flowx/preparer/activity_preparers/.py` +4. Add notebook generator to `src/flowx/preparer/code_generator.py` if needed +5. Register in engine.py (TRANSLATOR_REGISTRY for leaf, match statement for control-flow) +6. Move from AGENTIC_TYPES to DETERMINISTIC_TYPES in adf_loader.py +7. Update activity-mapping.md reference +8. Add test fixtures and unit tests diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1d612ff --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Flowx Changelog + +All notable changes to Flowx will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## [0.0.1] + +### Added +- Initial release of the Flowx library diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..34416d7 --- /dev/null +++ b/Makefile @@ -0,0 +1,48 @@ +.PHONY: clean dev ci test integration fmt help docs-install docs-clean docs-build docs-serve + +clean: + rm -rf .venv .pytest_cache .ruff_cache .mypy_cache __pycache__ + find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true + find . -type f -name "*.pyc" -delete 2>/dev/null || true + +dev: + uv sync + +ci: + uv sync --frozen + +test: + PYTHONPATH=src uv run pytest tests/unit -v + +integration: + PYTHONPATH=src uv run pytest tests/integration -v -m "not slow" + +fmt: + uv run ruff format src/ tests/ + uv run ruff check src/ tests/ --fix + uv run mypy src/flowx/ + +docs-install: + cd docs && bun install --frozen-lockfile + +docs-clean: + rm -rf docs/.next docs/.source docs/out docs/site docs/node_modules + +docs-build: docs-install + cd docs && bun run build && rm -rf site && mv out site + +docs-serve: docs-build + cd docs && bun run dev + +help: + @echo "Available targets:" + @echo " dev Install dependencies" + @echo " ci Install dependencies (frozen lockfile)" + @echo " test Run unit tests" + @echo " integration Run integration tests" + @echo " fmt Format and lint code" + @echo " clean Remove build artifacts" + @echo " docs-install Install docs dependencies (bun)" + @echo " docs-clean Remove docs build artifacts" + @echo " docs-build Build the static docs site to docs/site" + @echo " docs-serve Run the docs dev server (next dev)" diff --git a/README.md b/README.md new file mode 100644 index 0000000..f265657 --- /dev/null +++ b/README.md @@ -0,0 +1,145 @@ +# Flowx + +ADF to Databricks Lakeflow Jobs translator via Declarative Automation Bundles. + +Flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). It deterministically translates known activity types and falls back to agentic LLM-assisted translation for complex or rare types. + +## Architecture + +``` + Flowx Pipeline + ================== + + ADF JSON (UC Volumes) + | + v + +------------------+ + | 1. INGEST | Parse ADF ARM/JSON exports + | adf_loader.py | -> Typed AST -> inventory.json + +------------------+ + | + v + +------------------+ + | 2. TRANSLATE | Registry dispatch + topological sort + | engine.py | -> Pipeline IR (deterministic + agentic gaps) + +------------------+ + | + v + +------------------+ + | 3. PREPARE | IR -> DAB YAML + notebooks + setup scripts + | dab_writer.py | -> Deployable DABs project + +------------------+ + | + v + databricks bundle validate / deploy +``` + +## Quick Start + +1. Install the plugin in Claude Code: + ```bash + claude plugin install ghanse/flowx + ``` + +2. Run the end-to-end migration: + ``` + /flowx:migrate + ``` + + Or run individual phases: + ``` + /flowx:ingest # Parse ADF JSON, produce inventory + /flowx:translate # Deterministic + agentic translation + /flowx:prepare # Generate DABs project + ``` + +## Supported ADF Activity Types + +### Deterministic (16 types) + +| ADF Activity | Databricks Task | Category | +|---|---|---| +| Copy | Notebook task | Data movement | +| DatabricksNotebook | Notebook task | Compute | +| DatabricksSparkJar | Spark JAR task | Compute | +| DatabricksSparkPython | Spark Python task | Compute | +| ForEach | for_each_task | Control flow | +| IfCondition | if_else_task | Control flow | +| Switch | if_else_task chain | Control flow | +| SetVariable | run_job_task | Control flow | +| AppendVariable | run_job_task | Control flow | +| Filter | Notebook task | Control flow | +| Wait | Notebook task (sleep) | Control flow | +| Lookup | Notebook task | Data access | +| WebActivity | Notebook task | External | +| Delete | Notebook task | Data management | +| ExecutePipeline | run_job_task | Orchestration | +| DatabricksJob | run_job_task | Compute | + +### Agentic Fallback (12 types) + +| ADF Activity | Strategy | +|---|---| +| ExecuteDataFlow | LLM-assisted via adf-to-databricks-plugin | +| SqlServerStoredProcedure | LLM-assisted via adf-to-databricks-plugin | +| AzureFunction | LLM-assisted via adf-to-databricks-plugin | +| WebHook | LLM-assisted via adf-to-databricks-plugin | +| Custom | LLM-assisted via adf-to-databricks-plugin | +| ExecuteSSISPackage | LLM-assisted via adf-to-databricks-plugin | +| AzureMLExecutePipeline | LLM-assisted via adf-to-databricks-plugin | +| GetMetadata | LLM-assisted via adf-to-databricks-plugin | +| Validation | LLM-assisted via adf-to-databricks-plugin | +| Fail | LLM-assisted via adf-to-databricks-plugin | +| Script | LLM-assisted via adf-to-databricks-plugin | +| Until | LLM-assisted via adf-to-databricks-plugin | + +## How It Works + +### Phase 1: Ingest +Reads ADF JSON definitions from Unity Catalog volumes, normalizes ARM template format, parses into typed AST nodes, and classifies each activity as deterministic, agentic, or unsupported. Produces `inventory.json`. + +### Phase 2: Translate +Applies deterministic translators via registry dispatch, resolves dependencies through topological sort, and threads immutable `TranslationContext` through control-flow visitors. Agentic gaps are flagged for LLM-assisted translation. Produces Pipeline IR. + +### Phase 3: Prepare +Converts Pipeline IR into a deployable DABs project: `databricks.yml`, per-job YAML resource files, generated Python notebooks, and setup scripts for UC volumes, secrets, and connections. + +## Output Format + +``` +dab_output/ + databricks.yml # Bundle configuration + resources/ + jobs/ + .yml # One job per ADF pipeline + src/ + notebooks/ + / + .py # Generated notebooks per activity + setup/ + create_volumes.py # UC volume setup + create_secrets.py # Secret scope setup + create_connections.py # Connection setup +``` + +## Development + +```bash +make dev # Install dependencies +make test # Run unit tests +make integration # Run integration tests +make fmt # Format + lint (ruff + mypy) +make clean # Remove build artifacts +``` + +### Prerequisites +- Python 3.12+ +- [uv](https://docs.astral.sh/uv/) package manager + +## Contributing + +1. Fork the repository +2. Create a feature branch (`git checkout -b feature/my-feature`) +3. Follow the [adding a new translator](CLAUDE.md#adding-a-new-deterministic-translator) guide +4. Run `make fmt && make test` before committing +5. Open a pull request diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..7db33f9 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,8 @@ +node_modules/ +.next/ +.source/ +out/ +site/ +*.tsbuildinfo +next-env.d.ts +.env*.local diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..6e9f65b --- /dev/null +++ b/docs/README.md @@ -0,0 +1,33 @@ +# Flowx docs + +Documentation site for [flowx](https://github.com/ghanse/flowx), built with [fumadocs](https://fumadocs.dev) and deployed to GitHub Pages. + +## Local development + +From the repo root: + +```bash +make docs-install # one-time: install bun deps +make docs-serve # next dev — http://localhost:3000 +``` + +## Build + +```bash +make docs-build # static export to docs/site +``` + +The `docs-release.yml` workflow runs the same build and publishes `docs/site` to GitHub Pages on every push to `main` that touches `docs/**`. + +## Authoring + +Pages live in `content/docs/` as `.mdx` files. The page order is controlled by `content/docs/meta.json`. Frontmatter fields: + +```yaml +--- +title: Page title +description: Short summary used as the page subtitle and meta description. +--- +``` + +Components available out of the box: `Tabs` / `Tab`, `Callout`, `Steps` / `Step` — imported from `fumadocs-ui/components/...` at the top of each MDX file. diff --git a/docs/app/(home)/layout.tsx b/docs/app/(home)/layout.tsx new file mode 100644 index 0000000..1dd4684 --- /dev/null +++ b/docs/app/(home)/layout.tsx @@ -0,0 +1,7 @@ +import type { ReactNode } from 'react'; +import { HomeLayout } from 'fumadocs-ui/layouts/home'; +import { baseOptions } from '@/app/layout.config'; + +export default function Layout({ children }: { children: ReactNode }) { + return {children}; +} diff --git a/docs/app/(home)/page.tsx b/docs/app/(home)/page.tsx new file mode 100644 index 0000000..266cfcf --- /dev/null +++ b/docs/app/(home)/page.tsx @@ -0,0 +1,27 @@ +import Link from 'next/link'; + +export default function HomePage() { + return ( +
+

Flowx

+

+ Programmatically translate your data pipelines to Databricks Lakeflow jobs. +

+
+
+ + Read the docs + + + View on GitHub + +
+
+ ); +} diff --git a/docs/app/docs/[[...slug]]/page.tsx b/docs/app/docs/[[...slug]]/page.tsx new file mode 100644 index 0000000..40a24cf --- /dev/null +++ b/docs/app/docs/[[...slug]]/page.tsx @@ -0,0 +1,39 @@ +import { source } from '@/lib/source'; +import { DocsPage, DocsBody, DocsTitle, DocsDescription } from 'fumadocs-ui/page'; +import { notFound } from 'next/navigation'; + +export default async function Page(props: { + params: Promise<{ slug?: string[] }>; +}) { + const params = await props.params; + const page = source.getPage(params.slug); + if (!page) notFound(); + + const MDX = page.data.body; + + return ( + + {page.data.title} + {page.data.description} + + + + + ); +} + +export function generateStaticParams() { + return source.generateParams(); +} + +export async function generateMetadata(props: { + params: Promise<{ slug?: string[] }>; +}) { + const params = await props.params; + const page = source.getPage(params.slug); + if (!page) notFound(); + return { + title: page.data.title, + description: page.data.description, + }; +} diff --git a/docs/app/docs/layout.tsx b/docs/app/docs/layout.tsx new file mode 100644 index 0000000..a475cec --- /dev/null +++ b/docs/app/docs/layout.tsx @@ -0,0 +1,12 @@ +import type { ReactNode } from 'react'; +import { DocsLayout } from 'fumadocs-ui/layouts/docs'; +import { baseOptions } from '@/app/layout.config'; +import { source } from '@/lib/source'; + +export default function Layout({ children }: { children: ReactNode }) { + return ( + + {children} + + ); +} diff --git a/docs/app/global.css b/docs/app/global.css new file mode 100644 index 0000000..7408c0a --- /dev/null +++ b/docs/app/global.css @@ -0,0 +1,5 @@ +@import 'tailwindcss'; +@import 'fumadocs-ui/css/neutral.css'; +@import 'fumadocs-ui/css/preset.css'; + +@source '../node_modules/fumadocs-ui/dist/**/*.js'; diff --git a/docs/app/layout.config.tsx b/docs/app/layout.config.tsx new file mode 100644 index 0000000..9741c3e --- /dev/null +++ b/docs/app/layout.config.tsx @@ -0,0 +1,22 @@ +import type { BaseLayoutProps } from 'fumadocs-ui/layouts/shared'; + +export const baseOptions: BaseLayoutProps = { + nav: { + title: ( + flowx + ), + }, + links: [ + { + text: 'Documentation', + url: '/docs', + active: 'nested-url', + }, + { + text: 'GitHub', + url: 'https://github.com/ghanse/flowx', + external: true, + }, + ], + githubUrl: 'https://github.com/ghanse/flowx', +}; diff --git a/docs/app/layout.tsx b/docs/app/layout.tsx new file mode 100644 index 0000000..975d91b --- /dev/null +++ b/docs/app/layout.tsx @@ -0,0 +1,27 @@ +import './global.css'; +import type { ReactNode } from 'react'; +import { RootProvider } from 'fumadocs-ui/provider/next'; +import { Inter } from 'next/font/google'; + +const inter = Inter({ + subsets: ['latin'], +}); + +export const metadata = { + title: { + default: 'Flowx', + template: '%s | Flowx', + }, + description: + 'Translate Azure Data Factory pipelines to Databricks Lakeflow Jobs.', +}; + +export default function Layout({ children }: { children: ReactNode }) { + return ( + + + {children} + + + ); +} diff --git a/docs/bun.lock b/docs/bun.lock new file mode 100644 index 0000000..0348b0d --- /dev/null +++ b/docs/bun.lock @@ -0,0 +1,762 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "flowx-docs", + "dependencies": { + "fumadocs-core": "^16.0.0", + "fumadocs-mdx": "^14.0.0", + "fumadocs-ui": "^16.0.0", + "next": "^16.0.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + }, + "devDependencies": { + "@tailwindcss/postcss": "^4.0.0", + "@types/mdx": "^2.0.13", + "@types/node": "^22.10.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", + "postcss": "^8.4.49", + "tailwindcss": "^4.0.0", + "typescript": "^5.7.2", + }, + }, + }, + "packages": { + "@alloc/quick-lru": ["@alloc/quick-lru@5.2.0", "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", {}, "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw=="], + + "@emnapi/runtime": ["@emnapi/runtime@1.10.0", "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="], + + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.0", "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.0.tgz", { "os": "aix", "cpu": "ppc64" }, "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA=="], + + "@esbuild/android-arm": ["@esbuild/android-arm@0.28.0", "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.0.tgz", { "os": "android", "cpu": "arm" }, "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ=="], + + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.0", "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.0.tgz", { "os": "android", "cpu": "arm64" }, "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw=="], + + "@esbuild/android-x64": ["@esbuild/android-x64@0.28.0", "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.0.tgz", { "os": "android", "cpu": "x64" }, "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA=="], + + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.0", "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.0.tgz", { "os": "darwin", "cpu": "arm64" }, "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q=="], + + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.0", "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.0.tgz", { "os": "darwin", "cpu": "x64" }, "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ=="], + + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.0", "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.0.tgz", { "os": "freebsd", "cpu": "arm64" }, "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q=="], + + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.0", "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.0.tgz", { "os": "freebsd", "cpu": "x64" }, "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw=="], + + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.0", "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.0.tgz", { "os": "linux", "cpu": "arm" }, "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw=="], + + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.0", "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.0.tgz", { "os": "linux", "cpu": "arm64" }, "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A=="], + + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.0", "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.0.tgz", { "os": "linux", "cpu": "ia32" }, "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ=="], + + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.0", "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.0.tgz", { "os": "linux", "cpu": "none" }, "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg=="], + + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.0", "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.0.tgz", { "os": "linux", "cpu": "none" }, "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w=="], + + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.0", "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.0.tgz", { "os": "linux", "cpu": "ppc64" }, "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg=="], + + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.0", "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.0.tgz", { "os": "linux", "cpu": "none" }, "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ=="], + + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.0", "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.0.tgz", { "os": "linux", "cpu": "s390x" }, "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q=="], + + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.0", "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.0.tgz", { "os": "linux", "cpu": "x64" }, "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ=="], + + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.0", "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.0.tgz", { "os": "none", "cpu": "arm64" }, "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw=="], + + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.0", "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.0.tgz", { "os": "none", "cpu": "x64" }, "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw=="], + + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.0", "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.0.tgz", { "os": "openbsd", "cpu": "arm64" }, "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g=="], + + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.0", "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.0.tgz", { "os": "openbsd", "cpu": "x64" }, "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA=="], + + "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.0", "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.0.tgz", { "os": "none", "cpu": "arm64" }, "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w=="], + + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.0", "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.0.tgz", { "os": "sunos", "cpu": "x64" }, "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw=="], + + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.0", "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.0.tgz", { "os": "win32", "cpu": "arm64" }, "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA=="], + + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.0", "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.0.tgz", { "os": "win32", "cpu": "ia32" }, "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA=="], + + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.0", "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.0.tgz", { "os": "win32", "cpu": "x64" }, "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw=="], + + "@floating-ui/core": ["@floating-ui/core@1.7.5", "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.5.tgz", { "dependencies": { "@floating-ui/utils": "^0.2.11" } }, "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ=="], + + "@floating-ui/dom": ["@floating-ui/dom@1.7.6", "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.6.tgz", { "dependencies": { "@floating-ui/core": "^1.7.5", "@floating-ui/utils": "^0.2.11" } }, "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ=="], + + "@floating-ui/react-dom": ["@floating-ui/react-dom@2.1.8", "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.8.tgz", { "dependencies": { "@floating-ui/dom": "^1.7.6" }, "peerDependencies": { "react": ">=16.8.0", "react-dom": ">=16.8.0" } }, "sha512-cC52bHwM/n/CxS87FH0yWdngEZrjdtLW/qVruo68qg+prK7ZQ4YGdut2GyDVpoGeAYe/h899rVeOVm6Oi40k2A=="], + + "@floating-ui/utils": ["@floating-ui/utils@0.2.11", "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.11.tgz", {}, "sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg=="], + + "@fumadocs/tailwind": ["@fumadocs/tailwind@0.0.5", "https://registry.npmjs.org/@fumadocs/tailwind/-/tailwind-0.0.5.tgz", { "peerDependencies": { "@tailwindcss/oxide": "^4.0.0", "tailwindcss": "^4.0.0" }, "optionalPeers": ["@tailwindcss/oxide", "tailwindcss"] }, "sha512-ENKPWUDRmriccsrUDE4bDBq3FNr/ms3BP2rWlsAEMV1yP23pcCaan+ceGfeBUsAQjw7sj9Q3R4Kl3g/TCStPzQ=="], + + "@img/colour": ["@img/colour@1.1.0", "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], + + "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], + + "@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="], + + "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], + + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], + + "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + + "@mdx-js/mdx": ["@mdx-js/mdx@3.1.1", "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-3.1.1.tgz", { "dependencies": { "@types/estree": "^1.0.0", "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdx": "^2.0.0", "acorn": "^8.0.0", "collapse-white-space": "^2.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "estree-util-scope": "^1.0.0", "estree-walker": "^3.0.0", "hast-util-to-jsx-runtime": "^2.0.0", "markdown-extensions": "^2.0.0", "recma-build-jsx": "^1.0.0", "recma-jsx": "^1.0.0", "recma-stringify": "^1.0.0", "rehype-recma": "^1.0.0", "remark-mdx": "^3.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "source-map": "^0.7.0", "unified": "^11.0.0", "unist-util-position-from-estree": "^2.0.0", "unist-util-stringify-position": "^4.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ=="], + + "@next/env": ["@next/env@16.2.4", "https://registry.npmjs.org/@next/env/-/env-16.2.4.tgz", {}, "sha512-dKkkOzOSwFYe5RX6y26fZgkSpVAlIOJKQHIiydQcrWH6y/97+RceSOAdjZ14Qa3zLduVUy0TXcn+EiM6t4rPgw=="], + + "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@16.2.4", "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.2.4.tgz", { "os": "darwin", "cpu": "arm64" }, "sha512-OXTFFox5EKN1Ym08vfrz+OXxmCcEjT4SFMbNRsWZE99dMqt2Kcusl5MqPXcW232RYkMLQTy0hqgAMEsfEd/l2A=="], + + "@next/swc-darwin-x64": ["@next/swc-darwin-x64@16.2.4", "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.2.4.tgz", { "os": "darwin", "cpu": "x64" }, "sha512-XhpVnUfmYWvD3YrXu55XdcAkQtOnvaI6wtQa8fuF5fGoKoxIUZ0kWPtcOfqJEWngFF/lOS9l3+O9CcownhiQxQ=="], + + "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@16.2.4", "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.2.4.tgz", { "os": "linux", "cpu": "arm64" }, "sha512-Mx/tjlNA3G8kg14QvuGAJ4xBwPk1tUHq56JxZ8CXnZwz1Etz714soCEzGQQzVMz4bEnGPowzkV6Xrp6wAkEWOQ=="], + + "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@16.2.4", "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.2.4.tgz", { "os": "linux", "cpu": "arm64" }, "sha512-iVMMp14514u7Nup2umQS03nT/bN9HurK8ufylC3FZNykrwjtx7V1A7+4kvhbDSCeonTVqV3Txnv0Lu+m2oDXNg=="], + + "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@16.2.4", "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.2.4.tgz", { "os": "linux", "cpu": "x64" }, "sha512-EZOvm1aQWgnI/N/xcWOlnS3RQBk0VtVav5Zo7n4p0A7UKyTDx047k8opDbXgBpHl4CulRqRfbw3QrX2w5UOXMQ=="], + + "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@16.2.4", "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.2.4.tgz", { "os": "linux", "cpu": "x64" }, "sha512-h9FxsngCm9cTBf71AR4fGznDEDx1hS7+kSEiIRjq5kO1oXWm07DxVGZjCvk0SGx7TSjlUqhI8oOyz7NfwAdPoA=="], + + "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@16.2.4", "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.2.4.tgz", { "os": "win32", "cpu": "arm64" }, "sha512-3NdJV5OXMSOeJYijX+bjaLge3mJBlh4ybydbT4GFoB/2hAojWHtMhl3CYlYoMrjPuodp0nzFVi4Tj2+WaMg+Ow=="], + + "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@16.2.4", "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.2.4.tgz", { "os": "win32", "cpu": "x64" }, "sha512-kMVGgsqhO5YTYODD9IPGGhA6iprWidQckK3LmPeW08PIFENRmgfb4MjXHO+p//d+ts2rpjvK5gXWzXSMrPl9cw=="], + + "@orama/orama": ["@orama/orama@3.1.18", "https://registry.npmjs.org/@orama/orama/-/orama-3.1.18.tgz", {}, "sha512-a61ljmRVVyG5MC/698C8/FfFDw5a8LOIvyOLW5fztgUXqUpc1jOfQzOitSCbge657OgXXThmY3Tk8fpiDb4UcA=="], + + "@radix-ui/number": ["@radix-ui/number@1.1.1", "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz", {}, "sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g=="], + + "@radix-ui/primitive": ["@radix-ui/primitive@1.1.3", "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.3.tgz", {}, "sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg=="], + + "@radix-ui/react-accordion": ["@radix-ui/react-accordion@1.2.12", "https://registry.npmjs.org/@radix-ui/react-accordion/-/react-accordion-1.2.12.tgz", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collapsible": "1.1.12", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-T4nygeh9YE9dLRPhAHSeOZi7HBXo+0kYIPJXayZfvWOWA0+n3dESrZbjfDPUABkUNym6Hd+f2IR113To8D2GPA=="], + + "@radix-ui/react-arrow": ["@radix-ui/react-arrow@1.1.7", "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.7.tgz", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-F+M1tLhO+mlQaOWspE8Wstg+z6PwxwRd8oQ8IXceWz92kfAmalTRf0EjrouQeo7QssEPfCn05B4Ihs1K9WQ/7w=="], + + "@radix-ui/react-collapsible": ["@radix-ui/react-collapsible@1.1.12", "https://registry.npmjs.org/@radix-ui/react-collapsible/-/react-collapsible-1.1.12.tgz", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Uu+mSh4agx2ib1uIGPP4/CKNULyajb3p92LsVXmH2EHVMTfZWpll88XJ0j4W0z3f8NK1eYl1+Mf/szHPmcHzyA=="], + + "@radix-ui/react-collection": ["@radix-ui/react-collection@1.1.7", "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.7.tgz", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Fh9rGN0MoI4ZFUNyfFVNU4y9LUz93u9/0K+yLgA2bwRojxM8JU1DyvvMBabnZPBgMWREAJvU2jjVzq+LrFUglw=="], + + "@radix-ui/react-compose-refs": ["@radix-ui/react-compose-refs@1.1.2", "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.2.tgz", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-z4eqJvfiNnFMHIIvXP3CY57y2WJs5g2v3X0zm9mEJkrkNv4rDxu+sg9Jh8EkXyeqBkB7SOcboo9dMVqhyrACIg=="], + + "@radix-ui/react-context": ["@radix-ui/react-context@1.1.2", "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.2.tgz", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA=="], + + "@radix-ui/react-dialog": ["@radix-ui/react-dialog@1.1.15", "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.1.15.tgz", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-focus-guards": "1.1.3", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-controllable-state": "1.2.2", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-TCglVRtzlffRNxRMEyR36DGBLJpeusFcgMVD9PZEzAKnUs1lKCgX5u9BmC2Yg+LL9MgZDugFFs1Vl+Jp4t/PGw=="], + + "@radix-ui/react-direction": ["@radix-ui/react-direction@1.1.1", "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.1.tgz", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-1UEWRX6jnOA2y4H5WczZ44gOOjTEmlqv1uNW4GAJEO5+bauCBhv8snY65Iw5/VOS/ghKN9gr2KjnLKxrsvoMVw=="], + + "@radix-ui/react-dismissable-layer": ["@radix-ui/react-dismissable-layer@1.1.11", "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.11.tgz", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-escape-keydown": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Nqcp+t5cTB8BinFkZgXiMJniQH0PsUt2k51FUhbdfeKvc4ACcG2uQniY/8+h1Yv6Kza4Q7lD7PQV0z0oicE0Mg=="], + + "@radix-ui/react-focus-guards": ["@radix-ui/react-focus-guards@1.1.3", "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.3.tgz", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-0rFg/Rj2Q62NCm62jZw0QX7a3sz6QCQU0LpZdNrJX8byRGaGVTqbrW9jAoIAHyMQqsNpeZ81YgSizOt5WXq0Pw=="], + + "@radix-ui/react-focus-scope": ["@radix-ui/react-focus-scope@1.1.7", "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.7.tgz", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-t2ODlkXBQyn7jkl6TNaw/MtVEVvIGelJDCG41Okq/KwUsJBwQ4XVZsHAVUkK4mBv3ewiAS3PGuUWuY2BoK4ZUw=="], + + "@radix-ui/react-id": ["@radix-ui/react-id@1.1.1", "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.1.tgz", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-kGkGegYIdQsOb4XjsfM97rXsiHaBwco+hFI66oO4s9LU+PLAC5oJ7khdOVFxkhsmlbpUqDAvXw11CluXP+jkHg=="], + + "@radix-ui/react-navigation-menu": ["@radix-ui/react-navigation-menu@1.2.14", "https://registry.npmjs.org/@radix-ui/react-navigation-menu/-/react-navigation-menu-1.2.14.tgz", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-YB9mTFQvCOAQMHU+C/jVl96WmuWeltyUEpRJJky51huhds5W2FQr1J8D/16sQlf0ozxkPK8uF3niQMdUwZPv5w=="], + + "@radix-ui/react-popover": ["@radix-ui/react-popover@1.1.15", "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.15.tgz", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-focus-guards": "1.1.3", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-popper": "1.2.8", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-controllable-state": "1.2.2", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-kr0X2+6Yy/vJzLYJUPCZEc8SfQcf+1COFoAqauJm74umQhta9M7lNJHP7QQS3vkvcGLQUbWpMzwrXYwrYztHKA=="], + + "@radix-ui/react-popper": ["@radix-ui/react-popper@1.2.8", "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.2.8.tgz", { "dependencies": { "@floating-ui/react-dom": "^2.0.0", "@radix-ui/react-arrow": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-rect": "1.1.1", "@radix-ui/react-use-size": "1.1.1", "@radix-ui/rect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-0NJQ4LFFUuWkE7Oxf0htBKS6zLkkjBH+hM1uk7Ng705ReR8m/uelduy1DBo0PyBXPKVnBA6YBlU94MBGXrSBCw=="], + + "@radix-ui/react-portal": ["@radix-ui/react-portal@1.1.9", "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.9.tgz", { "dependencies": { "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-bpIxvq03if6UNwXZ+HTK71JLh4APvnXntDc6XOX8UVq4XQOVl7lwok0AvIl+b8zgCw3fSaVTZMpAPPagXbKmHQ=="], + + "@radix-ui/react-presence": ["@radix-ui/react-presence@1.1.5", "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.5.tgz", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-/jfEwNDdQVBCNvjkGit4h6pMOzq8bHkopq458dPt2lMjx+eBQUohZNG9A7DtO/O5ukSbxuaNGXMjHicgwy6rQQ=="], + + "@radix-ui/react-primitive": ["@radix-ui/react-primitive@2.1.3", "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.3.tgz", { "dependencies": { "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ=="], + + "@radix-ui/react-roving-focus": ["@radix-ui/react-roving-focus@1.1.11", "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.11.tgz", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA=="], + + "@radix-ui/react-scroll-area": ["@radix-ui/react-scroll-area@1.2.10", "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.2.10.tgz", { "dependencies": { "@radix-ui/number": "1.1.1", "@radix-ui/primitive": "1.1.3", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-tAXIa1g3sM5CGpVT0uIbUx/U3Gs5N8T52IICuCtObaos1S8fzsrPXG5WObkQN3S6NVl6wKgPhAIiBGbWnvc97A=="], + + "@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.4", "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.2.4.tgz", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Jl+bCv8HxKnlTLVrcDE8zTMJ09R9/ukw4qBs/oZClOfoQk/cOTbDn+NceXfV7j09YPVQUryJPHurafcSg6EVKA=="], + + "@radix-ui/react-tabs": ["@radix-ui/react-tabs@1.1.13", "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.13.tgz", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-7xdcatg7/U+7+Udyoj2zodtI9H/IIopqo+YOIcZOq1nJwXWBZ9p8xiu5llXlekDbZkca79a/fozEYQXIA4sW6A=="], + + "@radix-ui/react-use-callback-ref": ["@radix-ui/react-use-callback-ref@1.1.1", "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.1.tgz", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-FkBMwD+qbGQeMu1cOHnuGB6x4yzPjho8ap5WtbEJ26umhgqVXbhekKUQO+hZEL1vU92a3wHwdp0HAcqAUF5iDg=="], + + "@radix-ui/react-use-controllable-state": ["@radix-ui/react-use-controllable-state@1.2.2", "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.2.tgz", { "dependencies": { "@radix-ui/react-use-effect-event": "0.0.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-BjasUjixPFdS+NKkypcyyN5Pmg83Olst0+c6vGov0diwTEo6mgdqVR6hxcEgFuh4QrAs7Rc+9KuGJ9TVCj0Zzg=="], + + "@radix-ui/react-use-effect-event": ["@radix-ui/react-use-effect-event@0.0.2", "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.2.tgz", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Qp8WbZOBe+blgpuUT+lw2xheLP8q0oatc9UpmiemEICxGvFLYmHm9QowVZGHtJlGbS6A6yJ3iViad/2cVjnOiA=="], + + "@radix-ui/react-use-escape-keydown": ["@radix-ui/react-use-escape-keydown@1.1.1", "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.1.tgz", { "dependencies": { "@radix-ui/react-use-callback-ref": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Il0+boE7w/XebUHyBjroE+DbByORGR9KKmITzbR7MyQ4akpORYP/ZmbhAr0DG7RmmBqoOnZdy2QlvajJ2QA59g=="], + + "@radix-ui/react-use-layout-effect": ["@radix-ui/react-use-layout-effect@1.1.1", "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.1.tgz", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-RbJRS4UWQFkzHTTwVymMTUv8EqYhOp8dOOviLj2ugtTiXRaRQS7GLGxZTLL1jWhMeoSCf5zmcZkqTl9IiYfXcQ=="], + + "@radix-ui/react-use-previous": ["@radix-ui/react-use-previous@1.1.1", "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.1.1.tgz", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-2dHfToCj/pzca2Ck724OZ5L0EVrr3eHRNsG/b3xQJLA2hZpVCS99bLAX+hm1IHXDEnzU6by5z/5MIY794/a8NQ=="], + + "@radix-ui/react-use-rect": ["@radix-ui/react-use-rect@1.1.1", "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.1.tgz", { "dependencies": { "@radix-ui/rect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-QTYuDesS0VtuHNNvMh+CjlKJ4LJickCMUAqjlE3+j8w+RlRpwyX3apEQKGFzbZGdo7XNG1tXa+bQqIE7HIXT2w=="], + + "@radix-ui/react-use-size": ["@radix-ui/react-use-size@1.1.1", "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.1.tgz", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-ewrXRDTAqAXlkl6t/fkXWNAhFX9I+CkKlw6zjEwk86RSPKwZr3xpBRso655aqYafwtnbpHLj6toFzmd6xdVptQ=="], + + "@radix-ui/react-visually-hidden": ["@radix-ui/react-visually-hidden@1.2.3", "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.2.3.tgz", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-pzJq12tEaaIhqjbzpCuv/OypJY/BPavOofm+dbab+MHLajy277+1lLm6JFcGgF5eskJ6mquGirhXY2GD/8u8Ug=="], + + "@radix-ui/rect": ["@radix-ui/rect@1.1.1", "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.1.tgz", {}, "sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw=="], + + "@shikijs/core": ["@shikijs/core@4.0.2", "https://registry.npmjs.org/@shikijs/core/-/core-4.0.2.tgz", { "dependencies": { "@shikijs/primitive": "4.0.2", "@shikijs/types": "4.0.2", "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4", "hast-util-to-html": "^9.0.5" } }, "sha512-hxT0YF4ExEqB8G/qFdtJvpmHXBYJ2lWW7qTHDarVkIudPFE6iCIrqdgWxGn5s+ppkGXI0aEGlibI0PAyzP3zlw=="], + + "@shikijs/engine-javascript": ["@shikijs/engine-javascript@4.0.2", "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-4.0.2.tgz", { "dependencies": { "@shikijs/types": "4.0.2", "@shikijs/vscode-textmate": "^10.0.2", "oniguruma-to-es": "^4.3.4" } }, "sha512-7PW0Nm49DcoUIQEXlJhNNBHyoGMjalRETTCcjMqEaMoJRLljy1Bi/EGV3/qLBgLKQejdspiiYuHGQW6dX94Nag=="], + + "@shikijs/engine-oniguruma": ["@shikijs/engine-oniguruma@4.0.2", "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-4.0.2.tgz", { "dependencies": { "@shikijs/types": "4.0.2", "@shikijs/vscode-textmate": "^10.0.2" } }, "sha512-UpCB9Y2sUKlS9z8juFSKz7ZtysmeXCgnRF0dlhXBkmQnek7lAToPte8DkxmEYGNTMii72zU/lyXiCB6StuZeJg=="], + + "@shikijs/langs": ["@shikijs/langs@4.0.2", "https://registry.npmjs.org/@shikijs/langs/-/langs-4.0.2.tgz", { "dependencies": { "@shikijs/types": "4.0.2" } }, "sha512-KaXby5dvoeuZzN0rYQiPMjFoUrz4hgwIE+D6Du9owcHcl6/g16/yT5BQxSW5cGt2MZBz6Hl0YuRqf12omRfUUg=="], + + "@shikijs/primitive": ["@shikijs/primitive@4.0.2", "https://registry.npmjs.org/@shikijs/primitive/-/primitive-4.0.2.tgz", { "dependencies": { "@shikijs/types": "4.0.2", "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-M6UMPrSa3fN5ayeJwFVl9qWofl273wtK1VG8ySDZ1mQBfhCpdd8nEx7nPZ/tk7k+TYcpqBZzj/AnwxT9lO+HJw=="], + + "@shikijs/themes": ["@shikijs/themes@4.0.2", "https://registry.npmjs.org/@shikijs/themes/-/themes-4.0.2.tgz", { "dependencies": { "@shikijs/types": "4.0.2" } }, "sha512-mjCafwt8lJJaVSsQvNVrJumbnnj1RI8jbUKrPKgE6E3OvQKxnuRoBaYC51H4IGHePsGN/QtALglWBU7DoKDFnA=="], + + "@shikijs/types": ["@shikijs/types@4.0.2", "https://registry.npmjs.org/@shikijs/types/-/types-4.0.2.tgz", { "dependencies": { "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-qzbeRooUTPnLE+sHD/Z8DStmaDgnbbc/pMrU203950aRqjX/6AFHeDYT+j00y2lPdz0ywJKx7o/7qnqTivtlXg=="], + + "@shikijs/vscode-textmate": ["@shikijs/vscode-textmate@10.0.2", "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-10.0.2.tgz", {}, "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg=="], + + "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], + + "@swc/helpers": ["@swc/helpers@0.5.15", "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g=="], + + "@tailwindcss/node": ["@tailwindcss/node@4.2.4", "https://registry.npmjs.org/@tailwindcss/node/-/node-4.2.4.tgz", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "enhanced-resolve": "^5.19.0", "jiti": "^2.6.1", "lightningcss": "1.32.0", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.2.4" } }, "sha512-Ai7+yQPxz3ddrDQzFfBKdHEVBg0w3Zl83jnjuwxnZOsnH9pGn93QHQtpU0p/8rYWxvbFZHneni6p1BSLK4DkGA=="], + + "@tailwindcss/oxide": ["@tailwindcss/oxide@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.2.4.tgz", { "optionalDependencies": { "@tailwindcss/oxide-android-arm64": "4.2.4", "@tailwindcss/oxide-darwin-arm64": "4.2.4", "@tailwindcss/oxide-darwin-x64": "4.2.4", "@tailwindcss/oxide-freebsd-x64": "4.2.4", "@tailwindcss/oxide-linux-arm-gnueabihf": "4.2.4", "@tailwindcss/oxide-linux-arm64-gnu": "4.2.4", "@tailwindcss/oxide-linux-arm64-musl": "4.2.4", "@tailwindcss/oxide-linux-x64-gnu": "4.2.4", "@tailwindcss/oxide-linux-x64-musl": "4.2.4", "@tailwindcss/oxide-wasm32-wasi": "4.2.4", "@tailwindcss/oxide-win32-arm64-msvc": "4.2.4", "@tailwindcss/oxide-win32-x64-msvc": "4.2.4" } }, "sha512-9El/iI069DKDSXwTvB9J4BwdO5JhRrOweGaK25taBAvBXyXqJAX+Jqdvs8r8gKpsI/1m0LeJLyQYTf/WLrBT1Q=="], + + "@tailwindcss/oxide-android-arm64": ["@tailwindcss/oxide-android-arm64@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.2.4.tgz", { "os": "android", "cpu": "arm64" }, "sha512-e7MOr1SAn9U8KlZzPi1ZXGZHeC5anY36qjNwmZv9pOJ8E4Q6jmD1vyEHkQFmNOIN7twGPEMXRHmitN4zCMN03g=="], + + "@tailwindcss/oxide-darwin-arm64": ["@tailwindcss/oxide-darwin-arm64@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.2.4.tgz", { "os": "darwin", "cpu": "arm64" }, "sha512-tSC/Kbqpz/5/o/C2sG7QvOxAKqyd10bq+ypZNf+9Fi2TvbVbv1zNpcEptcsU7DPROaSbVgUXmrzKhurFvo5eDg=="], + + "@tailwindcss/oxide-darwin-x64": ["@tailwindcss/oxide-darwin-x64@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.2.4.tgz", { "os": "darwin", "cpu": "x64" }, "sha512-yPyUXn3yO/ufR6+Kzv0t4fCg2qNr90jxXc5QqBpjlPNd0NqyDXcmQb/6weunH/MEDXW5dhyEi+agTDiqa3WsGg=="], + + "@tailwindcss/oxide-freebsd-x64": ["@tailwindcss/oxide-freebsd-x64@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.2.4.tgz", { "os": "freebsd", "cpu": "x64" }, "sha512-BoMIB4vMQtZsXdGLVc2z+P9DbETkiopogfWZKbWwM8b/1Vinbs4YcUwo+kM/KeLkX3Ygrf4/PsRndKaYhS8Eiw=="], + + "@tailwindcss/oxide-linux-arm-gnueabihf": ["@tailwindcss/oxide-linux-arm-gnueabihf@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.2.4.tgz", { "os": "linux", "cpu": "arm" }, "sha512-7pIHBLTHYRAlS7V22JNuTh33yLH4VElwKtB3bwchK/UaKUPpQ0lPQiOWcbm4V3WP2I6fNIJ23vABIvoy2izdwA=="], + + "@tailwindcss/oxide-linux-arm64-gnu": ["@tailwindcss/oxide-linux-arm64-gnu@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.2.4.tgz", { "os": "linux", "cpu": "arm64" }, "sha512-+E4wxJ0ZGOzSH325reXTWB48l42i93kQqMvDyz5gqfRzRZ7faNhnmvlV4EPGJU3QJM/3Ab5jhJ5pCRUsKn6OQw=="], + + "@tailwindcss/oxide-linux-arm64-musl": ["@tailwindcss/oxide-linux-arm64-musl@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.2.4.tgz", { "os": "linux", "cpu": "arm64" }, "sha512-bBADEGAbo4ASnppIziaQJelekCxdMaxisrk+fB7Thit72IBnALp9K6ffA2G4ruj90G9XRS2VQ6q2bCKbfFV82g=="], + + "@tailwindcss/oxide-linux-x64-gnu": ["@tailwindcss/oxide-linux-x64-gnu@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.2.4.tgz", { "os": "linux", "cpu": "x64" }, "sha512-7Mx25E4WTfnht0TVRTyC00j3i0M+EeFe7wguMDTlX4mRxafznw0CA8WJkFjWYH5BlgELd1kSjuU2JiPnNZbJDA=="], + + "@tailwindcss/oxide-linux-x64-musl": ["@tailwindcss/oxide-linux-x64-musl@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.2.4.tgz", { "os": "linux", "cpu": "x64" }, "sha512-2wwJRF7nyhOR0hhHoChc04xngV3iS+akccHTGtz965FwF0up4b2lOdo6kI1EbDaEXKgvcrFBYcYQQ/rrnWFVfA=="], + + "@tailwindcss/oxide-wasm32-wasi": ["@tailwindcss/oxide-wasm32-wasi@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.2.4.tgz", { "dependencies": { "@emnapi/core": "^1.8.1", "@emnapi/runtime": "^1.8.1", "@emnapi/wasi-threads": "^1.1.0", "@napi-rs/wasm-runtime": "^1.1.1", "@tybys/wasm-util": "^0.10.1", "tslib": "^2.8.1" }, "cpu": "none" }, "sha512-FQsqApeor8Fo6gUEklzmaa9994orJZZDBAlQpK2Mq+DslRKFJeD6AjHpBQ0kZFQohVr8o85PPh8eOy86VlSCmw=="], + + "@tailwindcss/oxide-win32-arm64-msvc": ["@tailwindcss/oxide-win32-arm64-msvc@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.2.4.tgz", { "os": "win32", "cpu": "arm64" }, "sha512-L9BXqxC4ToVgwMFqj3pmZRqyHEztulpUJzCxUtLjobMCzTPsGt1Fa9enKbOpY2iIyVtaHNeNvAK8ERP/64sqGQ=="], + + "@tailwindcss/oxide-win32-x64-msvc": ["@tailwindcss/oxide-win32-x64-msvc@4.2.4", "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.2.4.tgz", { "os": "win32", "cpu": "x64" }, "sha512-ESlKG0EpVJQwRjXDDa9rLvhEAh0mhP1sF7sap9dNZT0yyl9SAG6T7gdP09EH0vIv0UNTlo6jPWyujD6559fZvw=="], + + "@tailwindcss/postcss": ["@tailwindcss/postcss@4.2.4", "https://registry.npmjs.org/@tailwindcss/postcss/-/postcss-4.2.4.tgz", { "dependencies": { "@alloc/quick-lru": "^5.2.0", "@tailwindcss/node": "4.2.4", "@tailwindcss/oxide": "4.2.4", "postcss": "^8.5.6", "tailwindcss": "4.2.4" } }, "sha512-wgAVj6nUWAolAu8YFvzT2cTBIElWHkjZwFYovF+xsqKsW2ADxM/X2opxj5NsF/qVccAOjRNe8X2IdPzMsWyHTg=="], + + "@types/debug": ["@types/debug@4.1.13", "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", { "dependencies": { "@types/ms": "*" } }, "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw=="], + + "@types/estree": ["@types/estree@1.0.8", "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], + + "@types/estree-jsx": ["@types/estree-jsx@1.0.5", "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", { "dependencies": { "@types/estree": "*" } }, "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg=="], + + "@types/hast": ["@types/hast@3.0.4", "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", { "dependencies": { "@types/unist": "*" } }, "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ=="], + + "@types/mdast": ["@types/mdast@4.0.4", "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="], + + "@types/mdx": ["@types/mdx@2.0.13", "https://registry.npmjs.org/@types/mdx/-/mdx-2.0.13.tgz", {}, "sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw=="], + + "@types/ms": ["@types/ms@2.1.0", "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="], + + "@types/node": ["@types/node@22.19.17", "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q=="], + + "@types/react": ["@types/react@19.2.14", "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="], + + "@types/react-dom": ["@types/react-dom@19.2.3", "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="], + + "@types/unist": ["@types/unist@3.0.3", "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], + + "@ungap/structured-clone": ["@ungap/structured-clone@1.3.0", "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", {}, "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="], + + "acorn": ["acorn@8.16.0", "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", { "bin": { "acorn": "bin/acorn" } }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="], + + "acorn-jsx": ["acorn-jsx@5.3.2", "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", { "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ=="], + + "argparse": ["argparse@2.0.1", "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], + + "aria-hidden": ["aria-hidden@1.2.6", "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz", { "dependencies": { "tslib": "^2.0.0" } }, "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA=="], + + "astring": ["astring@1.9.0", "https://registry.npmjs.org/astring/-/astring-1.9.0.tgz", { "bin": { "astring": "bin/astring" } }, "sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg=="], + + "bail": ["bail@2.0.2", "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], + + "baseline-browser-mapping": ["baseline-browser-mapping@2.10.24", "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.24.tgz", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-I2NkZOOrj2XuguvWCK6OVh9GavsNjZjK908Rq3mIBK25+GD8vPX5w2WdxVqnQ7xx3SrZJiCiZFu+/Oz50oSYSA=="], + + "caniuse-lite": ["caniuse-lite@1.0.30001791", "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001791.tgz", {}, "sha512-yk0l/YSrOnFZk3UROpDLQD9+kC1l4meK/wed583AXrzoarMGJcbRi2Q4RaUYbKxYAsZ8sWmaSa/DsLmdBeI1vQ=="], + + "ccount": ["ccount@2.0.1", "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", {}, "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg=="], + + "character-entities": ["character-entities@2.0.2", "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", {}, "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ=="], + + "character-entities-html4": ["character-entities-html4@2.1.0", "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", {}, "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA=="], + + "character-entities-legacy": ["character-entities-legacy@3.0.0", "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", {}, "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ=="], + + "character-reference-invalid": ["character-reference-invalid@2.0.1", "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", {}, "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw=="], + + "chokidar": ["chokidar@5.0.0", "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", { "dependencies": { "readdirp": "^5.0.0" } }, "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw=="], + + "class-variance-authority": ["class-variance-authority@0.7.1", "https://registry.npmjs.org/class-variance-authority/-/class-variance-authority-0.7.1.tgz", { "dependencies": { "clsx": "^2.1.1" } }, "sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg=="], + + "client-only": ["client-only@0.0.1", "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", {}, "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="], + + "clsx": ["clsx@2.1.1", "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="], + + "collapse-white-space": ["collapse-white-space@2.1.0", "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-2.1.0.tgz", {}, "sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw=="], + + "comma-separated-tokens": ["comma-separated-tokens@2.0.3", "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", {}, "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg=="], + + "compute-scroll-into-view": ["compute-scroll-into-view@3.1.1", "https://registry.npmjs.org/compute-scroll-into-view/-/compute-scroll-into-view-3.1.1.tgz", {}, "sha512-VRhuHOLoKYOy4UbilLbUzbYg93XLjv2PncJC50EuTWPA3gaja1UjBsUP/D/9/juV3vQFr6XBEzn9KCAHdUvOHw=="], + + "csstype": ["csstype@3.2.3", "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], + + "debug": ["debug@4.4.3", "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "decode-named-character-reference": ["decode-named-character-reference@1.3.0", "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q=="], + + "dequal": ["dequal@2.0.3", "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="], + + "detect-libc": ["detect-libc@2.1.2", "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + + "detect-node-es": ["detect-node-es@1.1.0", "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", {}, "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="], + + "devlop": ["devlop@1.1.0", "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", { "dependencies": { "dequal": "^2.0.0" } }, "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA=="], + + "enhanced-resolve": ["enhanced-resolve@5.21.0", "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.21.0.tgz", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.3.3" } }, "sha512-otxSQPw4lkOZWkHpB3zaEQs6gWYEsmX4xQF68ElXC/TWvGxGMSGOvoNbaLXm6/cS/fSfHtsEdw90y20PCd+sCA=="], + + "entities": ["entities@6.0.1", "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + + "esast-util-from-estree": ["esast-util-from-estree@2.0.0", "https://registry.npmjs.org/esast-util-from-estree/-/esast-util-from-estree-2.0.0.tgz", { "dependencies": { "@types/estree-jsx": "^1.0.0", "devlop": "^1.0.0", "estree-util-visit": "^2.0.0", "unist-util-position-from-estree": "^2.0.0" } }, "sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ=="], + + "esast-util-from-js": ["esast-util-from-js@2.0.1", "https://registry.npmjs.org/esast-util-from-js/-/esast-util-from-js-2.0.1.tgz", { "dependencies": { "@types/estree-jsx": "^1.0.0", "acorn": "^8.0.0", "esast-util-from-estree": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw=="], + + "esbuild": ["esbuild@0.28.0", "https://registry.npmjs.org/esbuild/-/esbuild-0.28.0.tgz", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.0", "@esbuild/android-arm": "0.28.0", "@esbuild/android-arm64": "0.28.0", "@esbuild/android-x64": "0.28.0", "@esbuild/darwin-arm64": "0.28.0", "@esbuild/darwin-x64": "0.28.0", "@esbuild/freebsd-arm64": "0.28.0", "@esbuild/freebsd-x64": "0.28.0", "@esbuild/linux-arm": "0.28.0", "@esbuild/linux-arm64": "0.28.0", "@esbuild/linux-ia32": "0.28.0", "@esbuild/linux-loong64": "0.28.0", "@esbuild/linux-mips64el": "0.28.0", "@esbuild/linux-ppc64": "0.28.0", "@esbuild/linux-riscv64": "0.28.0", "@esbuild/linux-s390x": "0.28.0", "@esbuild/linux-x64": "0.28.0", "@esbuild/netbsd-arm64": "0.28.0", "@esbuild/netbsd-x64": "0.28.0", "@esbuild/openbsd-arm64": "0.28.0", "@esbuild/openbsd-x64": "0.28.0", "@esbuild/openharmony-arm64": "0.28.0", "@esbuild/sunos-x64": "0.28.0", "@esbuild/win32-arm64": "0.28.0", "@esbuild/win32-ia32": "0.28.0", "@esbuild/win32-x64": "0.28.0" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw=="], + + "escape-string-regexp": ["escape-string-regexp@5.0.0", "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], + + "estree-util-attach-comments": ["estree-util-attach-comments@3.0.0", "https://registry.npmjs.org/estree-util-attach-comments/-/estree-util-attach-comments-3.0.0.tgz", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw=="], + + "estree-util-build-jsx": ["estree-util-build-jsx@3.0.1", "https://registry.npmjs.org/estree-util-build-jsx/-/estree-util-build-jsx-3.0.1.tgz", { "dependencies": { "@types/estree-jsx": "^1.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "estree-walker": "^3.0.0" } }, "sha512-8U5eiL6BTrPxp/CHbs2yMgP8ftMhR5ww1eIKoWRMlqvltHF8fZn5LRDvTKuxD3DUn+shRbLGqXemcP51oFCsGQ=="], + + "estree-util-is-identifier-name": ["estree-util-is-identifier-name@3.0.0", "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", {}, "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg=="], + + "estree-util-scope": ["estree-util-scope@1.0.0", "https://registry.npmjs.org/estree-util-scope/-/estree-util-scope-1.0.0.tgz", { "dependencies": { "@types/estree": "^1.0.0", "devlop": "^1.0.0" } }, "sha512-2CAASclonf+JFWBNJPndcOpA8EMJwa0Q8LUFJEKqXLW6+qBvbFZuF5gItbQOs/umBUkjviCSDCbBwU2cXbmrhQ=="], + + "estree-util-to-js": ["estree-util-to-js@2.0.0", "https://registry.npmjs.org/estree-util-to-js/-/estree-util-to-js-2.0.0.tgz", { "dependencies": { "@types/estree-jsx": "^1.0.0", "astring": "^1.8.0", "source-map": "^0.7.0" } }, "sha512-WDF+xj5rRWmD5tj6bIqRi6CkLIXbbNQUcxQHzGysQzvHmdYG2G7p/Tf0J0gpxGgkeMZNTIjT/AoSvC9Xehcgdg=="], + + "estree-util-value-to-estree": ["estree-util-value-to-estree@3.5.0", "https://registry.npmjs.org/estree-util-value-to-estree/-/estree-util-value-to-estree-3.5.0.tgz", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-aMV56R27Gv3QmfmF1MY12GWkGzzeAezAX+UplqHVASfjc9wNzI/X6hC0S9oxq61WT4aQesLGslWP9tKk6ghRZQ=="], + + "estree-util-visit": ["estree-util-visit@2.0.0", "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-2.0.0.tgz", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/unist": "^3.0.0" } }, "sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww=="], + + "estree-walker": ["estree-walker@3.0.3", "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], + + "extend": ["extend@3.0.2", "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], + + "fdir": ["fdir@6.5.0", "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], + + "framer-motion": ["framer-motion@12.38.0", "https://registry.npmjs.org/framer-motion/-/framer-motion-12.38.0.tgz", { "dependencies": { "motion-dom": "^12.38.0", "motion-utils": "^12.36.0", "tslib": "^2.4.0" }, "peerDependencies": { "@emotion/is-prop-valid": "*", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["@emotion/is-prop-valid", "react", "react-dom"] }, "sha512-rFYkY/pigbcswl1XQSb7q424kSTQ8q6eAC+YUsSKooHQYuLdzdHjrt6uxUC+PRAO++q5IS7+TamgIw1AphxR+g=="], + + "fumadocs-core": ["fumadocs-core@16.8.5", "https://registry.npmjs.org/fumadocs-core/-/fumadocs-core-16.8.5.tgz", { "dependencies": { "@orama/orama": "^3.1.18", "estree-util-value-to-estree": "^3.5.0", "github-slugger": "^2.0.0", "hast-util-to-estree": "^3.1.3", "hast-util-to-jsx-runtime": "^2.3.6", "js-yaml": "^4.1.1", "mdast-util-mdx": "^3.0.0", "mdast-util-to-markdown": "^2.1.2", "remark": "^15.0.1", "remark-gfm": "^4.0.1", "remark-rehype": "^11.1.2", "scroll-into-view-if-needed": "^3.1.0", "shiki": "^4.0.2", "tinyglobby": "^0.2.16", "unified": "^11.0.5", "unist-util-visit": "^5.1.0", "vfile": "^6.0.3" }, "peerDependencies": { "@mdx-js/mdx": "*", "@mixedbread/sdk": "0.x.x", "@orama/core": "1.x.x", "@oramacloud/client": "2.x.x", "@tanstack/react-router": "1.x.x", "@types/estree-jsx": "*", "@types/hast": "*", "@types/mdast": "*", "@types/react": "*", "algoliasearch": "5.x.x", "flexsearch": "*", "lucide-react": "*", "next": "16.x.x", "react": "^19.2.0", "react-dom": "^19.2.0", "react-router": "7.x.x", "waku": "^0.26.0 || ^0.27.0 || ^1.0.0", "zod": "4.x.x" }, "optionalPeers": ["@mdx-js/mdx", "@mixedbread/sdk", "@orama/core", "@oramacloud/client", "@tanstack/react-router", "@types/estree-jsx", "@types/hast", "@types/mdast", "@types/react", "algoliasearch", "flexsearch", "lucide-react", "next", "react", "react-dom", "react-router", "waku", "zod"] }, "sha512-4MRqh/KWtR5Q5+LJd2SFv3nLDHtuZw3q8rwApd9nAWkunHVU30U17fUVq6nY+IDoLs7bSLnvDGvoE+Ynelrn3A=="], + + "fumadocs-mdx": ["fumadocs-mdx@14.3.2", "https://registry.npmjs.org/fumadocs-mdx/-/fumadocs-mdx-14.3.2.tgz", { "dependencies": { "@mdx-js/mdx": "^3.1.1", "@standard-schema/spec": "^1.1.0", "chokidar": "^5.0.0", "esbuild": "^0.28.0", "estree-util-value-to-estree": "^3.5.0", "js-yaml": "^4.1.1", "mdast-util-mdx": "^3.0.0", "mdast-util-to-markdown": "^2.1.2", "picocolors": "^1.1.1", "picomatch": "^4.0.4", "tinyexec": "^1.1.1", "tinyglobby": "^0.2.16", "unified": "^11.0.5", "unist-util-remove-position": "^5.0.0", "unist-util-visit": "^5.1.0", "vfile": "^6.0.3", "zod": "^4.3.6" }, "peerDependencies": { "@types/mdast": "*", "@types/mdx": "*", "@types/react": "*", "fumadocs-core": "^15.0.0 || ^16.0.0", "mdast-util-directive": "*", "next": "^15.3.0 || ^16.0.0", "react": "^19.2.0", "vite": "6.x.x || 7.x.x || 8.x.x" }, "optionalPeers": ["@types/mdast", "@types/mdx", "@types/react", "mdast-util-directive", "next", "react", "vite"], "bin": { "fumadocs-mdx": "dist/bin.js" } }, "sha512-73SoZkbUuqnD91G/0zBcaQdM1TMnYw5JJzKgkGvQTiZbtLQFuWTt8/uRqnzFMuNIUu/WY9Lo9d1iZ8G+jOVieA=="], + + "fumadocs-ui": ["fumadocs-ui@16.8.5", "https://registry.npmjs.org/fumadocs-ui/-/fumadocs-ui-16.8.5.tgz", { "dependencies": { "@fumadocs/tailwind": "0.0.5", "@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-collapsible": "^1.1.12", "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-direction": "^1.1.1", "@radix-ui/react-navigation-menu": "^1.2.14", "@radix-ui/react-popover": "^1.1.15", "@radix-ui/react-presence": "^1.1.5", "@radix-ui/react-scroll-area": "^1.2.10", "@radix-ui/react-slot": "^1.2.4", "@radix-ui/react-tabs": "^1.1.13", "class-variance-authority": "^0.7.1", "lucide-react": "^1.11.0", "motion": "^12.38.0", "next-themes": "^0.4.6", "react-remove-scroll": "^2.7.2", "rehype-raw": "^7.0.0", "scroll-into-view-if-needed": "^3.1.0", "shiki": "^4.0.2", "tailwind-merge": "^3.5.0", "unist-util-visit": "^5.1.0" }, "peerDependencies": { "@takumi-rs/image-response": "*", "@types/mdx": "*", "@types/react": "*", "fumadocs-core": "16.8.5", "next": "16.x.x", "react": "^19.2.0", "react-dom": "^19.2.0" }, "optionalPeers": ["@takumi-rs/image-response", "@types/mdx", "@types/react", "next"] }, "sha512-caJjSfUhNkwoqumOBKfHxE1UjVHxkTsoaUhA96IvCM3G82bU2OKhf1pYtf/GbZ0XVdIlmY8Z47Cqwsze0HlXjg=="], + + "get-nonce": ["get-nonce@1.0.1", "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", {}, "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q=="], + + "github-slugger": ["github-slugger@2.0.0", "https://registry.npmjs.org/github-slugger/-/github-slugger-2.0.0.tgz", {}, "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw=="], + + "graceful-fs": ["graceful-fs@4.2.11", "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], + + "hast-util-from-parse5": ["hast-util-from-parse5@8.0.3", "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.3.tgz", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "devlop": "^1.0.0", "hastscript": "^9.0.0", "property-information": "^7.0.0", "vfile": "^6.0.0", "vfile-location": "^5.0.0", "web-namespaces": "^2.0.0" } }, "sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg=="], + + "hast-util-parse-selector": ["hast-util-parse-selector@4.0.0", "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A=="], + + "hast-util-raw": ["hast-util-raw@9.1.0", "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-9.1.0.tgz", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "@ungap/structured-clone": "^1.0.0", "hast-util-from-parse5": "^8.0.0", "hast-util-to-parse5": "^8.0.0", "html-void-elements": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "parse5": "^7.0.0", "unist-util-position": "^5.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0", "web-namespaces": "^2.0.0", "zwitch": "^2.0.0" } }, "sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw=="], + + "hast-util-to-estree": ["hast-util-to-estree@3.1.3", "https://registry.npmjs.org/hast-util-to-estree/-/hast-util-to-estree-3.1.3.tgz", { "dependencies": { "@types/estree": "^1.0.0", "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "devlop": "^1.0.0", "estree-util-attach-comments": "^3.0.0", "estree-util-is-identifier-name": "^3.0.0", "hast-util-whitespace": "^3.0.0", "mdast-util-mdx-expression": "^2.0.0", "mdast-util-mdx-jsx": "^3.0.0", "mdast-util-mdxjs-esm": "^2.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0", "style-to-js": "^1.0.0", "unist-util-position": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-48+B/rJWAp0jamNbAAf9M7Uf//UVqAoMmgXhBdxTDJLGKY+LRnZ99qcG+Qjl5HfMpYNzS5v4EAwVEF34LeAj7w=="], + + "hast-util-to-html": ["hast-util-to-html@9.0.5", "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.5.tgz", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-whitespace": "^3.0.0", "html-void-elements": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0", "stringify-entities": "^4.0.0", "zwitch": "^2.0.4" } }, "sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw=="], + + "hast-util-to-jsx-runtime": ["hast-util-to-jsx-runtime@2.3.6", "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz", { "dependencies": { "@types/estree": "^1.0.0", "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "comma-separated-tokens": "^2.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "hast-util-whitespace": "^3.0.0", "mdast-util-mdx-expression": "^2.0.0", "mdast-util-mdx-jsx": "^3.0.0", "mdast-util-mdxjs-esm": "^2.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0", "style-to-js": "^1.0.0", "unist-util-position": "^5.0.0", "vfile-message": "^4.0.0" } }, "sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg=="], + + "hast-util-to-parse5": ["hast-util-to-parse5@8.0.1", "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-8.0.1.tgz", { "dependencies": { "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "devlop": "^1.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0", "web-namespaces": "^2.0.0", "zwitch": "^2.0.0" } }, "sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA=="], + + "hast-util-whitespace": ["hast-util-whitespace@3.0.0", "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw=="], + + "hastscript": ["hastscript@9.0.1", "https://registry.npmjs.org/hastscript/-/hastscript-9.0.1.tgz", { "dependencies": { "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-parse-selector": "^4.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0" } }, "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w=="], + + "html-void-elements": ["html-void-elements@3.0.0", "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz", {}, "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg=="], + + "inline-style-parser": ["inline-style-parser@0.2.7", "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.7.tgz", {}, "sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA=="], + + "is-alphabetical": ["is-alphabetical@2.0.1", "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", {}, "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ=="], + + "is-alphanumerical": ["is-alphanumerical@2.0.1", "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", { "dependencies": { "is-alphabetical": "^2.0.0", "is-decimal": "^2.0.0" } }, "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw=="], + + "is-decimal": ["is-decimal@2.0.1", "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", {}, "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A=="], + + "is-hexadecimal": ["is-hexadecimal@2.0.1", "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", {}, "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg=="], + + "is-plain-obj": ["is-plain-obj@4.1.0", "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", {}, "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg=="], + + "jiti": ["jiti@2.6.1", "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ=="], + + "js-yaml": ["js-yaml@4.1.1", "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], + + "lightningcss": ["lightningcss@1.32.0", "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.32.0", "lightningcss-darwin-arm64": "1.32.0", "lightningcss-darwin-x64": "1.32.0", "lightningcss-freebsd-x64": "1.32.0", "lightningcss-linux-arm-gnueabihf": "1.32.0", "lightningcss-linux-arm64-gnu": "1.32.0", "lightningcss-linux-arm64-musl": "1.32.0", "lightningcss-linux-x64-gnu": "1.32.0", "lightningcss-linux-x64-musl": "1.32.0", "lightningcss-win32-arm64-msvc": "1.32.0", "lightningcss-win32-x64-msvc": "1.32.0" } }, "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ=="], + + "lightningcss-android-arm64": ["lightningcss-android-arm64@1.32.0", "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", { "os": "android", "cpu": "arm64" }, "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg=="], + + "lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.32.0", "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", { "os": "darwin", "cpu": "arm64" }, "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ=="], + + "lightningcss-darwin-x64": ["lightningcss-darwin-x64@1.32.0", "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", { "os": "darwin", "cpu": "x64" }, "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w=="], + + "lightningcss-freebsd-x64": ["lightningcss-freebsd-x64@1.32.0", "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", { "os": "freebsd", "cpu": "x64" }, "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig=="], + + "lightningcss-linux-arm-gnueabihf": ["lightningcss-linux-arm-gnueabihf@1.32.0", "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", { "os": "linux", "cpu": "arm" }, "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw=="], + + "lightningcss-linux-arm64-gnu": ["lightningcss-linux-arm64-gnu@1.32.0", "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", { "os": "linux", "cpu": "arm64" }, "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ=="], + + "lightningcss-linux-arm64-musl": ["lightningcss-linux-arm64-musl@1.32.0", "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", { "os": "linux", "cpu": "arm64" }, "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg=="], + + "lightningcss-linux-x64-gnu": ["lightningcss-linux-x64-gnu@1.32.0", "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", { "os": "linux", "cpu": "x64" }, "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA=="], + + "lightningcss-linux-x64-musl": ["lightningcss-linux-x64-musl@1.32.0", "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", { "os": "linux", "cpu": "x64" }, "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg=="], + + "lightningcss-win32-arm64-msvc": ["lightningcss-win32-arm64-msvc@1.32.0", "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", { "os": "win32", "cpu": "arm64" }, "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw=="], + + "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.32.0", "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", { "os": "win32", "cpu": "x64" }, "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q=="], + + "longest-streak": ["longest-streak@3.1.0", "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="], + + "lucide-react": ["lucide-react@1.11.0", "https://registry.npmjs.org/lucide-react/-/lucide-react-1.11.0.tgz", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-UOhjdztXCgdBReRcIhsvz2siIBogfv/lhJEIViCpLt924dO+GDms9T7DNoucI23s6kEPpe988m5N0D2ajnzb2g=="], + + "magic-string": ["magic-string@0.30.21", "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], + + "markdown-extensions": ["markdown-extensions@2.0.0", "https://registry.npmjs.org/markdown-extensions/-/markdown-extensions-2.0.0.tgz", {}, "sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q=="], + + "markdown-table": ["markdown-table@3.0.4", "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="], + + "mdast-util-find-and-replace": ["mdast-util-find-and-replace@3.0.2", "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.2.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "escape-string-regexp": "^5.0.0", "unist-util-is": "^6.0.0", "unist-util-visit-parents": "^6.0.0" } }, "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg=="], + + "mdast-util-from-markdown": ["mdast-util-from-markdown@2.0.3", "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "mdast-util-to-string": "^4.0.0", "micromark": "^4.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q=="], + + "mdast-util-gfm": ["mdast-util-gfm@3.1.0", "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", { "dependencies": { "mdast-util-from-markdown": "^2.0.0", "mdast-util-gfm-autolink-literal": "^2.0.0", "mdast-util-gfm-footnote": "^2.0.0", "mdast-util-gfm-strikethrough": "^2.0.0", "mdast-util-gfm-table": "^2.0.0", "mdast-util-gfm-task-list-item": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ=="], + + "mdast-util-gfm-autolink-literal": ["mdast-util-gfm-autolink-literal@2.0.1", "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "ccount": "^2.0.0", "devlop": "^1.0.0", "mdast-util-find-and-replace": "^3.0.0", "micromark-util-character": "^2.0.0" } }, "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ=="], + + "mdast-util-gfm-footnote": ["mdast-util-gfm-footnote@2.1.0", "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "devlop": "^1.1.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0" } }, "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ=="], + + "mdast-util-gfm-strikethrough": ["mdast-util-gfm-strikethrough@2.0.0", "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg=="], + + "mdast-util-gfm-table": ["mdast-util-gfm-table@2.0.0", "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "markdown-table": "^3.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg=="], + + "mdast-util-gfm-task-list-item": ["mdast-util-gfm-task-list-item@2.0.0", "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ=="], + + "mdast-util-mdx": ["mdast-util-mdx@3.0.0", "https://registry.npmjs.org/mdast-util-mdx/-/mdast-util-mdx-3.0.0.tgz", { "dependencies": { "mdast-util-from-markdown": "^2.0.0", "mdast-util-mdx-expression": "^2.0.0", "mdast-util-mdx-jsx": "^3.0.0", "mdast-util-mdxjs-esm": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w=="], + + "mdast-util-mdx-expression": ["mdast-util-mdx-expression@2.0.1", "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ=="], + + "mdast-util-mdx-jsx": ["mdast-util-mdx-jsx@3.2.0", "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.2.0.tgz", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "devlop": "^1.1.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0", "parse-entities": "^4.0.0", "stringify-entities": "^4.0.0", "unist-util-stringify-position": "^4.0.0", "vfile-message": "^4.0.0" } }, "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q=="], + + "mdast-util-mdxjs-esm": ["mdast-util-mdxjs-esm@2.0.1", "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg=="], + + "mdast-util-phrasing": ["mdast-util-phrasing@4.1.0", "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "unist-util-is": "^6.0.0" } }, "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w=="], + + "mdast-util-to-hast": ["mdast-util-to-hast@13.2.1", "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "@ungap/structured-clone": "^1.0.0", "devlop": "^1.0.0", "micromark-util-sanitize-uri": "^2.0.0", "trim-lines": "^3.0.0", "unist-util-position": "^5.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA=="], + + "mdast-util-to-markdown": ["mdast-util-to-markdown@2.1.2", "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="], + + "mdast-util-to-string": ["mdast-util-to-string@4.0.0", "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="], + + "micromark": ["micromark@4.0.2", "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="], + + "micromark-core-commonmark": ["micromark-core-commonmark@2.0.3", "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="], + + "micromark-extension-gfm": ["micromark-extension-gfm@3.0.0", "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", { "dependencies": { "micromark-extension-gfm-autolink-literal": "^2.0.0", "micromark-extension-gfm-footnote": "^2.0.0", "micromark-extension-gfm-strikethrough": "^2.0.0", "micromark-extension-gfm-table": "^2.0.0", "micromark-extension-gfm-tagfilter": "^2.0.0", "micromark-extension-gfm-task-list-item": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w=="], + + "micromark-extension-gfm-autolink-literal": ["micromark-extension-gfm-autolink-literal@2.1.0", "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw=="], + + "micromark-extension-gfm-footnote": ["micromark-extension-gfm-footnote@2.1.0", "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", { "dependencies": { "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw=="], + + "micromark-extension-gfm-strikethrough": ["micromark-extension-gfm-strikethrough@2.1.0", "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", { "dependencies": { "devlop": "^1.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw=="], + + "micromark-extension-gfm-table": ["micromark-extension-gfm-table@2.1.1", "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.1.tgz", { "dependencies": { "devlop": "^1.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg=="], + + "micromark-extension-gfm-tagfilter": ["micromark-extension-gfm-tagfilter@2.0.0", "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", { "dependencies": { "micromark-util-types": "^2.0.0" } }, "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg=="], + + "micromark-extension-gfm-task-list-item": ["micromark-extension-gfm-task-list-item@2.1.0", "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", { "dependencies": { "devlop": "^1.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw=="], + + "micromark-extension-mdx-expression": ["micromark-extension-mdx-expression@3.0.1", "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-3.0.1.tgz", { "dependencies": { "@types/estree": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-mdx-expression": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-events-to-acorn": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-dD/ADLJ1AeMvSAKBwO22zG22N4ybhe7kFIZ3LsDI0GlsNr2A3KYxb0LdC1u5rj4Nw+CHKY0RVdnHX8vj8ejm4Q=="], + + "micromark-extension-mdx-jsx": ["micromark-extension-mdx-jsx@3.0.2", "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-3.0.2.tgz", { "dependencies": { "@types/estree": "^1.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "micromark-factory-mdx-expression": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-events-to-acorn": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-e5+q1DjMh62LZAJOnDraSSbDMvGJ8x3cbjygy2qFEi7HCeUT4BDKCvMozPozcD6WmOt6sVvYDNBKhFSz3kjOVQ=="], + + "micromark-extension-mdx-md": ["micromark-extension-mdx-md@2.0.0", "https://registry.npmjs.org/micromark-extension-mdx-md/-/micromark-extension-mdx-md-2.0.0.tgz", { "dependencies": { "micromark-util-types": "^2.0.0" } }, "sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ=="], + + "micromark-extension-mdxjs": ["micromark-extension-mdxjs@3.0.0", "https://registry.npmjs.org/micromark-extension-mdxjs/-/micromark-extension-mdxjs-3.0.0.tgz", { "dependencies": { "acorn": "^8.0.0", "acorn-jsx": "^5.0.0", "micromark-extension-mdx-expression": "^3.0.0", "micromark-extension-mdx-jsx": "^3.0.0", "micromark-extension-mdx-md": "^2.0.0", "micromark-extension-mdxjs-esm": "^3.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ=="], + + "micromark-extension-mdxjs-esm": ["micromark-extension-mdxjs-esm@3.0.0", "https://registry.npmjs.org/micromark-extension-mdxjs-esm/-/micromark-extension-mdxjs-esm-3.0.0.tgz", { "dependencies": { "@types/estree": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-events-to-acorn": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-position-from-estree": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A=="], + + "micromark-factory-destination": ["micromark-factory-destination@2.0.1", "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA=="], + + "micromark-factory-label": ["micromark-factory-label@2.0.1", "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", { "dependencies": { "devlop": "^1.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg=="], + + "micromark-factory-mdx-expression": ["micromark-factory-mdx-expression@2.0.3", "https://registry.npmjs.org/micromark-factory-mdx-expression/-/micromark-factory-mdx-expression-2.0.3.tgz", { "dependencies": { "@types/estree": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-events-to-acorn": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-position-from-estree": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-kQnEtA3vzucU2BkrIa8/VaSAsP+EJ3CKOvhMuJgOEGg9KDC6OAY6nSnNDVRiVNRqj7Y4SlSzcStaH/5jge8JdQ=="], + + "micromark-factory-space": ["micromark-factory-space@2.0.1", "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg=="], + + "micromark-factory-title": ["micromark-factory-title@2.0.1", "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", { "dependencies": { "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw=="], + + "micromark-factory-whitespace": ["micromark-factory-whitespace@2.0.1", "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", { "dependencies": { "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ=="], + + "micromark-util-character": ["micromark-util-character@2.1.1", "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", { "dependencies": { "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q=="], + + "micromark-util-chunked": ["micromark-util-chunked@2.0.1", "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA=="], + + "micromark-util-classify-character": ["micromark-util-classify-character@2.0.1", "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q=="], + + "micromark-util-combine-extensions": ["micromark-util-combine-extensions@2.0.1", "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", { "dependencies": { "micromark-util-chunked": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg=="], + + "micromark-util-decode-numeric-character-reference": ["micromark-util-decode-numeric-character-reference@2.0.2", "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw=="], + + "micromark-util-decode-string": ["micromark-util-decode-string@2.0.1", "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", { "dependencies": { "decode-named-character-reference": "^1.0.0", "micromark-util-character": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-symbol": "^2.0.0" } }, "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ=="], + + "micromark-util-encode": ["micromark-util-encode@2.0.1", "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", {}, "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw=="], + + "micromark-util-events-to-acorn": ["micromark-util-events-to-acorn@2.0.3", "https://registry.npmjs.org/micromark-util-events-to-acorn/-/micromark-util-events-to-acorn-2.0.3.tgz", { "dependencies": { "@types/estree": "^1.0.0", "@types/unist": "^3.0.0", "devlop": "^1.0.0", "estree-util-visit": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-jmsiEIiZ1n7X1Rr5k8wVExBQCg5jy4UXVADItHmNk1zkwEVhBuIUKRu3fqv+hs4nxLISi2DQGlqIOGiFxgbfHg=="], + + "micromark-util-html-tag-name": ["micromark-util-html-tag-name@2.0.1", "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", {}, "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA=="], + + "micromark-util-normalize-identifier": ["micromark-util-normalize-identifier@2.0.1", "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q=="], + + "micromark-util-resolve-all": ["micromark-util-resolve-all@2.0.1", "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", { "dependencies": { "micromark-util-types": "^2.0.0" } }, "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg=="], + + "micromark-util-sanitize-uri": ["micromark-util-sanitize-uri@2.0.1", "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-symbol": "^2.0.0" } }, "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ=="], + + "micromark-util-subtokenize": ["micromark-util-subtokenize@2.1.0", "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", { "dependencies": { "devlop": "^1.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA=="], + + "micromark-util-symbol": ["micromark-util-symbol@2.0.1", "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", {}, "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q=="], + + "micromark-util-types": ["micromark-util-types@2.0.2", "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", {}, "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA=="], + + "motion": ["motion@12.38.0", "https://registry.npmjs.org/motion/-/motion-12.38.0.tgz", { "dependencies": { "framer-motion": "^12.38.0", "tslib": "^2.4.0" }, "peerDependencies": { "@emotion/is-prop-valid": "*", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["@emotion/is-prop-valid", "react", "react-dom"] }, "sha512-uYfXzeHlgThchzwz5Te47dlv5JOUC7OB4rjJ/7XTUgtBZD8CchMN8qEJ4ZVsUmTyYA44zjV0fBwsiktRuFnn+w=="], + + "motion-dom": ["motion-dom@12.38.0", "https://registry.npmjs.org/motion-dom/-/motion-dom-12.38.0.tgz", { "dependencies": { "motion-utils": "^12.36.0" } }, "sha512-pdkHLD8QYRp8VfiNLb8xIBJis1byQ9gPT3Jnh2jqfFtAsWUA3dEepDlsWe/xMpO8McV+VdpKVcp+E+TGJEtOoA=="], + + "motion-utils": ["motion-utils@12.36.0", "https://registry.npmjs.org/motion-utils/-/motion-utils-12.36.0.tgz", {}, "sha512-eHWisygbiwVvf6PZ1vhaHCLamvkSbPIeAYxWUuL3a2PD/TROgE7FvfHWTIH4vMl798QLfMw15nRqIaRDXTlYRg=="], + + "ms": ["ms@2.1.3", "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "nanoid": ["nanoid@3.3.11", "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], + + "next": ["next@16.2.4", "https://registry.npmjs.org/next/-/next-16.2.4.tgz", { "dependencies": { "@next/env": "16.2.4", "@swc/helpers": "0.5.15", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" }, "optionalDependencies": { "@next/swc-darwin-arm64": "16.2.4", "@next/swc-darwin-x64": "16.2.4", "@next/swc-linux-arm64-gnu": "16.2.4", "@next/swc-linux-arm64-musl": "16.2.4", "@next/swc-linux-x64-gnu": "16.2.4", "@next/swc-linux-x64-musl": "16.2.4", "@next/swc-win32-arm64-msvc": "16.2.4", "@next/swc-win32-x64-msvc": "16.2.4", "sharp": "^0.34.5" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "babel-plugin-react-compiler", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-kPvz56wF5frc+FxlHI5qnklCzbq53HTwORaWBGdT0vNoKh1Aya9XC8aPauH4NJxqtzbWsS5mAbctm4cr+EkQ2Q=="], + + "next-themes": ["next-themes@0.4.6", "https://registry.npmjs.org/next-themes/-/next-themes-0.4.6.tgz", { "peerDependencies": { "react": "^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc", "react-dom": "^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc" } }, "sha512-pZvgD5L0IEvX5/9GWyHMf3m8BKiVQwsCMHfoFosXtXBMnaS0ZnIJ9ST4b4NqLVKDEm8QBxoNNGNaBv2JNF6XNA=="], + + "oniguruma-parser": ["oniguruma-parser@0.12.2", "https://registry.npmjs.org/oniguruma-parser/-/oniguruma-parser-0.12.2.tgz", {}, "sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw=="], + + "oniguruma-to-es": ["oniguruma-to-es@4.3.6", "https://registry.npmjs.org/oniguruma-to-es/-/oniguruma-to-es-4.3.6.tgz", { "dependencies": { "oniguruma-parser": "^0.12.2", "regex": "^6.1.0", "regex-recursion": "^6.0.2" } }, "sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA=="], + + "parse-entities": ["parse-entities@4.0.2", "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", { "dependencies": { "@types/unist": "^2.0.0", "character-entities-legacy": "^3.0.0", "character-reference-invalid": "^2.0.0", "decode-named-character-reference": "^1.0.0", "is-alphanumerical": "^2.0.0", "is-decimal": "^2.0.0", "is-hexadecimal": "^2.0.0" } }, "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw=="], + + "parse5": ["parse5@7.3.0", "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="], + + "picocolors": ["picocolors@1.1.1", "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], + + "picomatch": ["picomatch@4.0.4", "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="], + + "postcss": ["postcss@8.5.12", "https://registry.npmjs.org/postcss/-/postcss-8.5.12.tgz", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-W62t/Se6rA0Az3DfCL0AqJwXuKwBeYg6nOaIgzP+xZ7N5BFCI7DYi1qs6ygUYT6rvfi6t9k65UMLJC+PHZpDAA=="], + + "property-information": ["property-information@7.1.0", "https://registry.npmjs.org/property-information/-/property-information-7.1.0.tgz", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], + + "react": ["react@19.2.5", "https://registry.npmjs.org/react/-/react-19.2.5.tgz", {}, "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA=="], + + "react-dom": ["react-dom@19.2.5", "https://registry.npmjs.org/react-dom/-/react-dom-19.2.5.tgz", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.5" } }, "sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag=="], + + "react-remove-scroll": ["react-remove-scroll@2.7.2", "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz", { "dependencies": { "react-remove-scroll-bar": "^2.3.7", "react-style-singleton": "^2.2.3", "tslib": "^2.1.0", "use-callback-ref": "^1.3.3", "use-sidecar": "^1.1.3" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q=="], + + "react-remove-scroll-bar": ["react-remove-scroll-bar@2.3.8", "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz", { "dependencies": { "react-style-singleton": "^2.2.2", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" }, "optionalPeers": ["@types/react"] }, "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q=="], + + "react-style-singleton": ["react-style-singleton@2.2.3", "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz", { "dependencies": { "get-nonce": "^1.0.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ=="], + + "readdirp": ["readdirp@5.0.0", "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", {}, "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ=="], + + "recma-build-jsx": ["recma-build-jsx@1.0.0", "https://registry.npmjs.org/recma-build-jsx/-/recma-build-jsx-1.0.0.tgz", { "dependencies": { "@types/estree": "^1.0.0", "estree-util-build-jsx": "^3.0.0", "vfile": "^6.0.0" } }, "sha512-8GtdyqaBcDfva+GUKDr3nev3VpKAhup1+RvkMvUxURHpW7QyIvk9F5wz7Vzo06CEMSilw6uArgRqhpiUcWp8ew=="], + + "recma-jsx": ["recma-jsx@1.0.1", "https://registry.npmjs.org/recma-jsx/-/recma-jsx-1.0.1.tgz", { "dependencies": { "acorn-jsx": "^5.0.0", "estree-util-to-js": "^2.0.0", "recma-parse": "^1.0.0", "recma-stringify": "^1.0.0", "unified": "^11.0.0" }, "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-huSIy7VU2Z5OLv6oFLosQGGDqPqdO1iq6bWNAdhzMxSJP7RAso4fCZ1cKu8j9YHCZf3TPrq4dw3okhrylgcd7w=="], + + "recma-parse": ["recma-parse@1.0.0", "https://registry.npmjs.org/recma-parse/-/recma-parse-1.0.0.tgz", { "dependencies": { "@types/estree": "^1.0.0", "esast-util-from-js": "^2.0.0", "unified": "^11.0.0", "vfile": "^6.0.0" } }, "sha512-OYLsIGBB5Y5wjnSnQW6t3Xg7q3fQ7FWbw/vcXtORTnyaSFscOtABg+7Pnz6YZ6c27fG1/aN8CjfwoUEUIdwqWQ=="], + + "recma-stringify": ["recma-stringify@1.0.0", "https://registry.npmjs.org/recma-stringify/-/recma-stringify-1.0.0.tgz", { "dependencies": { "@types/estree": "^1.0.0", "estree-util-to-js": "^2.0.0", "unified": "^11.0.0", "vfile": "^6.0.0" } }, "sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g=="], + + "regex": ["regex@6.1.0", "https://registry.npmjs.org/regex/-/regex-6.1.0.tgz", { "dependencies": { "regex-utilities": "^2.3.0" } }, "sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg=="], + + "regex-recursion": ["regex-recursion@6.0.2", "https://registry.npmjs.org/regex-recursion/-/regex-recursion-6.0.2.tgz", { "dependencies": { "regex-utilities": "^2.3.0" } }, "sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg=="], + + "regex-utilities": ["regex-utilities@2.3.0", "https://registry.npmjs.org/regex-utilities/-/regex-utilities-2.3.0.tgz", {}, "sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng=="], + + "rehype-raw": ["rehype-raw@7.0.0", "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz", { "dependencies": { "@types/hast": "^3.0.0", "hast-util-raw": "^9.0.0", "vfile": "^6.0.0" } }, "sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww=="], + + "rehype-recma": ["rehype-recma@1.0.0", "https://registry.npmjs.org/rehype-recma/-/rehype-recma-1.0.0.tgz", { "dependencies": { "@types/estree": "^1.0.0", "@types/hast": "^3.0.0", "hast-util-to-estree": "^3.0.0" } }, "sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw=="], + + "remark": ["remark@15.0.1", "https://registry.npmjs.org/remark/-/remark-15.0.1.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "remark-parse": "^11.0.0", "remark-stringify": "^11.0.0", "unified": "^11.0.0" } }, "sha512-Eht5w30ruCXgFmxVUSlNWQ9iiimq07URKeFS3hNc8cUWy1llX4KDWfyEDZRycMc+znsN9Ux5/tJ/BFdgdOwA3A=="], + + "remark-gfm": ["remark-gfm@4.0.1", "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-gfm": "^3.0.0", "micromark-extension-gfm": "^3.0.0", "remark-parse": "^11.0.0", "remark-stringify": "^11.0.0", "unified": "^11.0.0" } }, "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg=="], + + "remark-mdx": ["remark-mdx@3.1.1", "https://registry.npmjs.org/remark-mdx/-/remark-mdx-3.1.1.tgz", { "dependencies": { "mdast-util-mdx": "^3.0.0", "micromark-extension-mdxjs": "^3.0.0" } }, "sha512-Pjj2IYlUY3+D8x00UJsIOg5BEvfMyeI+2uLPn9VO9Wg4MEtN/VTIq2NEJQfde9PnX15KgtHyl9S0BcTnWrIuWg=="], + + "remark-parse": ["remark-parse@11.0.0", "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-from-markdown": "^2.0.0", "micromark-util-types": "^2.0.0", "unified": "^11.0.0" } }, "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA=="], + + "remark-rehype": ["remark-rehype@11.1.2", "https://registry.npmjs.org/remark-rehype/-/remark-rehype-11.1.2.tgz", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "mdast-util-to-hast": "^13.0.0", "unified": "^11.0.0", "vfile": "^6.0.0" } }, "sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw=="], + + "remark-stringify": ["remark-stringify@11.0.0", "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-to-markdown": "^2.0.0", "unified": "^11.0.0" } }, "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw=="], + + "scheduler": ["scheduler@0.27.0", "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], + + "scroll-into-view-if-needed": ["scroll-into-view-if-needed@3.1.0", "https://registry.npmjs.org/scroll-into-view-if-needed/-/scroll-into-view-if-needed-3.1.0.tgz", { "dependencies": { "compute-scroll-into-view": "^3.0.2" } }, "sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ=="], + + "semver": ["semver@7.7.4", "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], + + "sharp": ["sharp@0.34.5", "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], + + "shiki": ["shiki@4.0.2", "https://registry.npmjs.org/shiki/-/shiki-4.0.2.tgz", { "dependencies": { "@shikijs/core": "4.0.2", "@shikijs/engine-javascript": "4.0.2", "@shikijs/engine-oniguruma": "4.0.2", "@shikijs/langs": "4.0.2", "@shikijs/themes": "4.0.2", "@shikijs/types": "4.0.2", "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-eAVKTMedR5ckPo4xne/PjYQYrU3qx78gtJZ+sHlXEg5IHhhoQhMfZVzetTYuaJS0L2Ef3AcCRzCHV8T0WI6nIQ=="], + + "source-map": ["source-map@0.7.6", "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], + + "source-map-js": ["source-map-js@1.2.1", "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + + "space-separated-tokens": ["space-separated-tokens@2.0.2", "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", {}, "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q=="], + + "stringify-entities": ["stringify-entities@4.0.4", "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", { "dependencies": { "character-entities-html4": "^2.0.0", "character-entities-legacy": "^3.0.0" } }, "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg=="], + + "style-to-js": ["style-to-js@1.1.21", "https://registry.npmjs.org/style-to-js/-/style-to-js-1.1.21.tgz", { "dependencies": { "style-to-object": "1.0.14" } }, "sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ=="], + + "style-to-object": ["style-to-object@1.0.14", "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.14.tgz", { "dependencies": { "inline-style-parser": "0.2.7" } }, "sha512-LIN7rULI0jBscWQYaSswptyderlarFkjQ+t79nzty8tcIAceVomEVlLzH5VP4Cmsv6MtKhs7qaAiwlcp+Mgaxw=="], + + "styled-jsx": ["styled-jsx@5.1.6", "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz", { "dependencies": { "client-only": "0.0.1" }, "peerDependencies": { "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0" } }, "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA=="], + + "tailwind-merge": ["tailwind-merge@3.5.0", "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.5.0.tgz", {}, "sha512-I8K9wewnVDkL1NTGoqWmVEIlUcB9gFriAEkXkfCjX5ib8ezGxtR3xD7iZIxrfArjEsH7F1CHD4RFUtxefdqV/A=="], + + "tailwindcss": ["tailwindcss@4.2.4", "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.4.tgz", {}, "sha512-HhKppgO81FQof5m6TEnuBWCZGgfRAWbaeOaGT00KOy/Pf/j6oUihdvBpA7ltCeAvZpFhW3j0PTclkxsd4IXYDA=="], + + "tapable": ["tapable@2.3.3", "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", {}, "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A=="], + + "tinyexec": ["tinyexec@1.1.1", "https://registry.npmjs.org/tinyexec/-/tinyexec-1.1.1.tgz", {}, "sha512-VKS/ZaQhhkKFMANmAOhhXVoIfBXblQxGX1myCQ2faQrfmobMftXeJPcZGp0gS07ocvGJWDLZGyOZDadDBqYIJg=="], + + "tinyglobby": ["tinyglobby@0.2.16", "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg=="], + + "trim-lines": ["trim-lines@3.0.1", "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", {}, "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg=="], + + "trough": ["trough@2.2.0", "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="], + + "tslib": ["tslib@2.8.1", "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "typescript": ["typescript@5.9.3", "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@6.21.0", "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + + "unified": ["unified@11.0.5", "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", { "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", "devlop": "^1.0.0", "extend": "^3.0.0", "is-plain-obj": "^4.0.0", "trough": "^2.0.0", "vfile": "^6.0.0" } }, "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA=="], + + "unist-util-is": ["unist-util-is@6.0.1", "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g=="], + + "unist-util-position": ["unist-util-position@5.0.0", "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA=="], + + "unist-util-position-from-estree": ["unist-util-position-from-estree@2.0.0", "https://registry.npmjs.org/unist-util-position-from-estree/-/unist-util-position-from-estree-2.0.0.tgz", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ=="], + + "unist-util-remove-position": ["unist-util-remove-position@5.0.0", "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-5.0.0.tgz", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-visit": "^5.0.0" } }, "sha512-Hp5Kh3wLxv0PHj9m2yZhhLt58KzPtEYKQQ4yxfYFEO7EvHwzyDYnduhHnY1mDxoqr7VUwVuHXk9RXKIiYS1N8Q=="], + + "unist-util-stringify-position": ["unist-util-stringify-position@4.0.0", "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ=="], + + "unist-util-visit": ["unist-util-visit@5.1.0", "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0", "unist-util-visit-parents": "^6.0.0" } }, "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg=="], + + "unist-util-visit-parents": ["unist-util-visit-parents@6.0.2", "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.2.tgz", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0" } }, "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ=="], + + "use-callback-ref": ["use-callback-ref@1.3.3", "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz", { "dependencies": { "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg=="], + + "use-sidecar": ["use-sidecar@1.1.3", "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.3.tgz", { "dependencies": { "detect-node-es": "^1.1.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ=="], + + "vfile": ["vfile@6.0.3", "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", { "dependencies": { "@types/unist": "^3.0.0", "vfile-message": "^4.0.0" } }, "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q=="], + + "vfile-location": ["vfile-location@5.0.3", "https://registry.npmjs.org/vfile-location/-/vfile-location-5.0.3.tgz", { "dependencies": { "@types/unist": "^3.0.0", "vfile": "^6.0.0" } }, "sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg=="], + + "vfile-message": ["vfile-message@4.0.3", "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw=="], + + "web-namespaces": ["web-namespaces@2.0.1", "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz", {}, "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ=="], + + "zod": ["zod@4.3.6", "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + + "zwitch": ["zwitch@2.0.4", "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], + + "@radix-ui/react-collection/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.2.3.tgz", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-dialog/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.2.3.tgz", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-popover/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.2.3.tgz", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-primitive/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.2.3.tgz", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.10.0", "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", { "dependencies": { "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" }, "bundled": true }, "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw=="], + + "@tailwindcss/oxide-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.10.0", "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="], + + "@tailwindcss/oxide-wasm32-wasi/@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.1", "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w=="], + + "@tailwindcss/oxide-wasm32-wasi/@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.4", "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" }, "bundled": true }, "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow=="], + + "@tailwindcss/oxide-wasm32-wasi/@tybys/wasm-util": ["@tybys/wasm-util@0.10.1", "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg=="], + + "@tailwindcss/oxide-wasm32-wasi/tslib": ["tslib@2.8.1", "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", { "bundled": true }, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "next/postcss": ["postcss@8.4.31", "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", { "dependencies": { "nanoid": "^3.3.6", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" } }, "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ=="], + + "parse-entities/@types/unist": ["@types/unist@2.0.11", "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="], + } +} diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx new file mode 100644 index 0000000..643662c --- /dev/null +++ b/docs/content/docs/index.mdx @@ -0,0 +1,36 @@ +--- +title: Introduction +description: What flowx is and where to start. +--- + +## Motivation + +Orchestration should be treated as a first class citizen during migrations. Because orchestrators drive the execution of data pipelines, +their configuration can impact data processing results as much as the logic being orchestrated. While significant tooling exists for code +conversion and data reconciliation, migrating from legacy orchestration systems is often manual, time-consuming, and prone to risk. + +Flowx was created to automate migrations of data pipelines between various orchestrators. It provides a robust, tested set of capabilities +to parse existing data pipeline definitions, create migration artifacts, and convert data pipeline definitions to Databricks' [Lakeflow jobs framework](https://docs.databricks.com/aws/en/jobs/). + +## How Flowx works + +Flowx is a set of agent skills and deterministic translators. Skills tell agentic tools (e.g. Databricks Genie Code, Claude Code, or any +agent that supports the open [Agent Skills](https://agentskills.io/) format) how to call deterministic translators that parse, translate, and generate Databricks resources. + +Translation runs in three phases: + +1. `ingest` parses Azure Resource Manager templates (e.g. for Data Factory pipelines, datasets, linked services, and triggers) into an execution +tree and builds an inventory. +2. `translate` processes the inventory and converts each activity into a Databricks-compatible intermediate representation. *Deterministic +activities* are translated by Python handlers while *agentic activities* are handed off to an LLM-assisted translator with the right context. +*Unsupported activities* are flagged as explicit gaps. +3. `prepare` converts each translated pipeline into a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) that +can be deployed to a Databricks workspace. Bundles include job configuration files, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). + +Each phase can be run independently, maintains its own input/output contract, produces artifacts you can inspect before moving to the next phase. + +## Where to go next + +- **[Installation](/docs/installation)** — install the flowx plugin in your agentic tool of choice. +- **[Usage Guide](/docs/usage-guide)** — an end-to-end walkthrough from raw ADF JSON to a deployable bundle. +- **[AI Tools & Skills](/docs/ai-tools-skills)** — the agent skills shipped with flowx and how each tool loads them. diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx new file mode 100644 index 0000000..84a17d1 --- /dev/null +++ b/docs/content/docs/installation.mdx @@ -0,0 +1,73 @@ +--- +title: Installation +description: Install flowx in Databricks Genie Code, Claude Code, or other agentic tools. +--- + +import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; +import { Callout } from 'fumadocs-ui/components/callout'; + +Flowx contains four [agent skills](https://github.com/ghanse/flowx/tree/main/skills) (`ingest`, `translate`, `prepare`, and `migrate`) that teach agentic tools how to use the flowx Python modules. +Installing flowx also installs all required dependencies to run the Python modules. + + + +Clone the flowx repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos), then copy the `skills/` directory into a user-level skills folder: + +```bash +databricks workspace import-dir skills /Users//.assistant/skills +``` + +To make flowx available for other workspace users, copy `skills/` into a workspace-level skills folder: + +```bash +databricks workspace import-dir skills /Workspace/.assistant/skills +``` + +Genie Code picks up skills from these directories automatically. Skills fire automatically when their description matches your request. +To invoke a specific skill, use the `@` prefix (e.g. `@migrate translate the ADF pipelines I exported to /Volumes/main/default/adf_export`). + +See the [Databricks Genie Code Skills documentation](https://docs.databricks.com/aws/en/genie-code/skills) for more details. + + + +Flowx is packaged as a Claude Code plugin. The plugin manifest lives at [`.claude-plugin/plugin.json`](https://github.com/ghanse/flowx/blob/main/.claude-plugin/plugin.json). To install +flowx, run the following command from a Claude Code session: + +```bash +/plugin marketplace add ghanse/flowx +/plugin install flowx +``` + +You can also copy the skill folders into your local `/.claude/skills` folder: + +```bash +cp -R skills/{ingest,translate,prepare,migrate} ~/.claude/skills/ +``` + +Once installed, the skills are can be invoked using `/flowx:migrate`, `/flowx:ingest`, etc. Claude Code can also invoke the skills when prompted. See the [Claude Code Plugins documentation](https://docs.claude.com/en/docs/claude-code/plugins) for more details. + + + +Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install the flowx skills. The general pattern: + +1. Copy each skill folder (`skills/ingest`, `skills/translate`, `skills/prepare`, `skills/migrate`) into the tool's configured skills directory. +2. Make sure the path contains `SKILL.md` directly, +3. Restart the tool if it caches skill metadata at startup. + + +If your tool expects a single Markdown file instead of a directory tree, use the following command to concatenate the skills: + +```bash +cat skills/*/SKILL.md > flowx-skills.md +``` + + + + +## Verifying the install + +Open your agent and ask: + +> What flowx skills do you have available? + +You should see all four skills listed with their descriptions. If only some appear, double-check the install path your tool watches for skills. diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json new file mode 100644 index 0000000..e65d2ac --- /dev/null +++ b/docs/content/docs/meta.json @@ -0,0 +1,10 @@ +{ + "title": "Flowx", + "pages": [ + "index", + "how-it-works", + "installation", + "usage-guide", + "ai-tools-skills" + ] +} diff --git a/docs/content/docs/usage-guide.mdx b/docs/content/docs/usage-guide.mdx new file mode 100644 index 0000000..720417a --- /dev/null +++ b/docs/content/docs/usage-guide.mdx @@ -0,0 +1,85 @@ +--- +title: Usage Guide +description: Use flowx to translate a pipeline from Azure Data Factory to Lakeflow Jobs. +--- + +import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; +import { Callout } from 'fumadocs-ui/components/callout'; +import { Steps, Step } from 'fumadocs-ui/components/steps'; + +This guide walks through a complete migration: install the flowx skills in your agentic tool, hand it a directory of Azure Data Factory JSON exports, and end up with a Databricks Asset Bundle you can deploy. + + + + +## Export pipeline templates as JSON + +In the Azure Data Factory portal, open *Manage* → *ARM template* → *Export ARM template*, or use the `Get-AzDataFactoryV2Pipeline` PowerShell cmdlet to dump each pipeline definition as JSON. You'll end up with a directory tree like: + +```text +adf-export/ +├── pipeline/ # one JSON file per pipeline +├── dataset/ # one JSON file per dataset +├── linkedService/ # one JSON file per linked service +└── trigger/ # one JSON file per trigger (optional) +``` + +Upload the directory to a Unity Catalog volume (recommended) or a local path the agent can read: + +```bash +databricks fs cp -r ./adf-export dbfs:/Volumes/main/default/adf_export +``` + + + + +## Run the end-to-end migration + +Open a fresh conversation and prompt your agent with the path to your JSON templates and a target directory for the output bundle: + +> Use flowx to migrate the ADF pipelines at `/Volumes/main/default/adf_export` into a Databricks Asset Bundle at `./bundle/`. + +Flowx will use the `migrate` skill to chain 3 other skills: + +1. `ingest` parses every JSON file, builds an inventory, and assigns a translation strategy for each resource. This can be deterministic, agentic, or unsupported. +2. `translate` converts each activity to an intermediate representation. The agent will ask for confirmation before running any LLM-based translation. +3. `prepare` creates a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) with job configuration files, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). + + + + + +## Review the output + +Flowx creates Declarative Automation Bundles in the local file system. The generated bundle can be reviewed and modified before deployment. + +Each bundle contains a top-level `databricks.yml` file with deployment targets and other variables, a `resources/` folder with job configuration, +a `src/` folder with code required to run the pipeline, and a `setup/` folder with scripts for creating supporting resources. + +The `translation_report.json` file lists every activity, its translation strategy, warnings raised during translation, and the location of any +generated artifacts. Review the translation report for any warnings, unsupported resources, or to-do items before deploying to your Databricks workspace. + + +Connection strings, credentials, and other protected configuration parameters are emitted as `SecretInstruction` setup steps that require [Databricks Secrets](https://docs.databricks.com/aws/en/security/secrets/). +Run the setup scripts and populate secret values before deploying and running pipelines in your Databricks workspace. + + + + + +## Deploy the bundle + +To deploy translated pipelines to your Databricks workspace, run the following command from the root of the bundle directory: + +```bash +databricks bundle validate +databricks bundle deploy --target +``` + + +Bundles created by flowx are standard Databricks Asset Bundles. You can target different environments, integrate with CI/CD, +or further customize the YAML before deploying. See the [Databricks Asset Bundles documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/) for more information. + + + + diff --git a/docs/lib/source.ts b/docs/lib/source.ts new file mode 100644 index 0000000..26878cd --- /dev/null +++ b/docs/lib/source.ts @@ -0,0 +1,7 @@ +import { docs } from '@/.source/server'; +import { loader } from 'fumadocs-core/source'; + +export const source = loader({ + baseUrl: '/docs', + source: docs.toFumadocsSource(), +}); diff --git a/docs/next.config.mjs b/docs/next.config.mjs new file mode 100644 index 0000000..600427e --- /dev/null +++ b/docs/next.config.mjs @@ -0,0 +1,18 @@ +import { createMDX } from 'fumadocs-mdx/next'; + +const withMDX = createMDX(); + +const repoBasePath = process.env.DOCS_BASE_PATH ?? '/flowx'; + +/** @type {import('next').NextConfig} */ +const config = { + output: 'export', + reactStrictMode: true, + basePath: repoBasePath, + trailingSlash: true, + images: { + unoptimized: true, + }, +}; + +export default withMDX(config); diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 0000000..5975330 --- /dev/null +++ b/docs/package.json @@ -0,0 +1,29 @@ +{ + "name": "flowx-docs", + "version": "0.0.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "postinstall": "fumadocs-mdx" + }, + "dependencies": { + "fumadocs-core": "^16.0.0", + "fumadocs-mdx": "^14.0.0", + "fumadocs-ui": "^16.0.0", + "next": "^16.0.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" + }, + "devDependencies": { + "@tailwindcss/postcss": "^4.0.0", + "@types/mdx": "^2.0.13", + "@types/node": "^22.10.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", + "postcss": "^8.4.49", + "tailwindcss": "^4.0.0", + "typescript": "^5.7.2" + } +} diff --git a/docs/postcss.config.mjs b/docs/postcss.config.mjs new file mode 100644 index 0000000..a34a3d5 --- /dev/null +++ b/docs/postcss.config.mjs @@ -0,0 +1,5 @@ +export default { + plugins: { + '@tailwindcss/postcss': {}, + }, +}; diff --git a/docs/source.config.ts b/docs/source.config.ts new file mode 100644 index 0000000..0b6ee4d --- /dev/null +++ b/docs/source.config.ts @@ -0,0 +1,7 @@ +import { defineDocs, defineConfig } from 'fumadocs-mdx/config'; + +export const docs = defineDocs({ + dir: 'content/docs', +}); + +export default defineConfig(); diff --git a/docs/tsconfig.json b/docs/tsconfig.json new file mode 100644 index 0000000..30a24d7 --- /dev/null +++ b/docs/tsconfig.json @@ -0,0 +1,45 @@ +{ + "compilerOptions": { + "target": "ESNext", + "lib": [ + "dom", + "dom.iterable", + "esnext" + ], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "react-jsx", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "baseUrl": ".", + "paths": { + "@/*": [ + "./*" + ] + } + }, + "include": [ + "next-env.d.ts", + "**/*.ts", + "**/*.tsx", + ".next/types/**/*.ts", + ".source", + ".next/dev/types/**/*.ts" + ], + "exclude": [ + "node_modules", + "out", + ".next" + ] +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..34e9e16 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,68 @@ +[project] +name = "flowx" +version = "0.2.0" +description = "ADF to Databricks Lakeflow Jobs translator via Declarative Automation Bundles" +readme = "README.md" +requires-python = ">=3.12" +authors = [{name = "Greg Hansen", email = "gregory.hansen@databricks.com"}] +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Environment :: Console", + "Framework :: Pytest", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Topic :: Software Development :: Libraries", + "Topic :: Utilities", +] +dependencies = [ + "pyyaml>=6.0", +] + +[dependency-groups] +dev = [ + "pytest>=8.3.3,<9", + "coverage>=7.6.1,<8", + "ruff>=0.14.0,<1", + "mypy>=1.18.2,<2", + "types-pyyaml>=6.0.12.20250915,<7", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/flowx"] + +[tool.mypy] +python_version = "3.12" +mypy_path = "src" +exclude = ['venv', '.venv', 'tests/*'] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--no-header" +cache_dir = ".venv/pytest-cache" +filterwarnings = ["ignore::DeprecationWarning"] +markers = [ + "integration: marks tests that require live Azure Data Factory access", + "slow: marks tests that are slow to run", +] + +[tool.ruff] +cache-dir = ".venv/ruff-cache" +target-version = "py312" +line-length = 120 +exclude = ["templates/*"] + +[tool.ruff.lint] +select = ["E", "F", "I"] + +[tool.ruff.lint.isort] +known-first-party = ["flowx"] diff --git a/skills/ingest/SKILL.md b/skills/ingest/SKILL.md new file mode 100644 index 0000000..7b38648 --- /dev/null +++ b/skills/ingest/SKILL.md @@ -0,0 +1,190 @@ +--- +name: ingest +description: > + Load and parse Azure Data Factory pipeline definitions from Unity Catalog volumes or local directories. + Produces a typed inventory that classifies every activity as deterministic, agentic, or unsupported. +triggers: + - "ingest ADF" + - "load ADF" + - "parse ADF" + - "import pipelines" + - "load pipelines" + - "parse pipelines" + - "inventory ADF" +--- + +# Ingest ADF Pipeline Definitions + +Parse Azure Data Factory pipeline, dataset, linked service, and trigger JSON files into a typed AST and produce a classified inventory. + +## Context + +This is phase 1 of the flowx migration workflow. It takes raw ADF JSON exports and produces an `inventory.json` file that the `translate` skill consumes. The inventory classifies every ADF activity into one of three strategies: + +- **Deterministic** — a built-in translator exists (Copy, DatabricksNotebook, ForEach, IfCondition, etc.) +- **Agentic** — requires LLM-assisted translation via the `adf-to-databricks-plugin` skills (ExecuteDataFlow, Switch, Until, StoredProc, etc.) +- **Unsupported** — no known translation path; requires manual intervention + +## Workflow + +Follow these steps in order: + +### Step 1 — Determine the ADF source path + +Ask the user for the location of their ADF JSON exports. Accept either: +- A Unity Catalog volume path (e.g., `/Volumes/main/default/adf_export`) +- A local directory path (e.g., `./adf_export/` or `/tmp/adf_json/`) + +The directory should contain subdirectories or files for: +- `pipeline/` or `pipelines/` — pipeline definition JSON files +- `dataset/` or `datasets/` — dataset definition JSON files (optional) +- `linkedService/` or `linked_services/` — linked service JSON files (optional) +- `trigger/` or `triggers/` — trigger definition JSON files (optional) + +### Step 2 — Download from UC volumes if needed + +If the source path starts with `/Volumes/`, the files live in a Unity Catalog volume and must be downloaded to a local temp directory first. + +Use the `databricks-execution-compute` skill to run the following on the Databricks workspace: + +```python +import os, json, shutil, tempfile + +volume_path = "" +local_dir = tempfile.mkdtemp(prefix="adf_ingest_") + +# Copy from volume to local +for root, dirs, files in os.walk(volume_path): + for f in files: + if f.endswith(".json"): + src = os.path.join(root, f) + rel = os.path.relpath(src, volume_path) + dst = os.path.join(local_dir, rel) + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + +print(f"Downloaded ADF files to: {local_dir}") +``` + +Alternatively, use the Databricks CLI: +```bash +databricks fs cp -r "dbfs:" "" --overwrite +``` + +Set the working source directory to the local temp path for subsequent steps. + +### Step 3 — Run the deterministic parser + +Execute the ADF loader to parse all JSON files and produce the inventory: + +```bash +python3 /src/flowx/parser/adf_loader.py \ + --source-dir \ + --output-dir +``` + +Where: +- `` is the root of the flowx plugin (the directory containing `src/`) +- `` is the local directory containing ADF JSON files +- `` is where to write the parsed output (default: `./orchestra_output/ingest/`) + +This produces: +- `inventory.json` — the classified activity inventory +- `ast/` directory — the typed AST for each pipeline +- `parse_errors.json` — any files that failed to parse + +### Step 4 — Read and validate the inventory + +Read the generated `inventory.json` file. It has this structure: + +```json +{ + "source_dir": "/path/to/adf/json", + "generated_at": "2026-04-07T12:00:00Z", + "pipelines": [ + { + "name": "PipelineName", + "file": "pipeline/PipelineName.json", + "activities": [ + { + "name": "CopyFromBlob", + "type": "Copy", + "strategy": "deterministic", + "translator": "copy.py" + }, + { + "name": "RunDataFlow", + "type": "ExecuteDataFlow", + "strategy": "agentic", + "skill": "adf-to-databricks:adf-dataflow-converter" + } + ] + } + ], + "summary": { + "pipeline_count": 12, + "activity_count": 47, + "deterministic_count": 35, + "agentic_count": 10, + "unsupported_count": 2, + "coverage_pct": 95.7 + } +} +``` + +### Step 5 — Present the summary + +Display a summary table to the user: + +``` +ADF Ingestion Summary +===================== +Pipelines parsed: 12 +Total activities: 47 + +Strategy Breakdown: + Deterministic: 35 (74.5%) + Agentic: 10 (21.3%) + Unsupported: 2 ( 4.3%) + +Coverage: 95.7% +``` + +### Step 6 — Detail agentic activities + +For activities classified as `agentic`, explain which skill from the `adf-to-databricks-plugin` will handle each: + +| Activity | Type | Handling Skill | +|---|---|---| +| RunDataFlow | ExecuteDataFlow | `adf-to-databricks:adf-dataflow-converter` | +| BranchLogic | Switch | `adf-to-databricks:adf-pipeline-converter` | +| ... | ... | ... | + +### Step 7 — Warn about unsupported activities + +For activities classified as `unsupported`, warn the user clearly: + +``` +WARNING: The following activities have no automated translation path: + - Pipeline "ETL_Main" / Activity "RunSSIS" (ExecuteSSISPackage) + Recommendation: Manual conversion to PySpark notebook required. +``` + +### Step 8 — Confirm output location + +Tell the user where the inventory and AST files were written, and confirm they can proceed to the `translate` phase. + +## Examples + +- "Ingest my ADF pipelines from /Volumes/main/default/adf_export" +- "Parse ADF definitions from ./tests/resources/json/" +- "Load the ADF pipeline JSON files and show me the inventory" +- "Import pipelines from /tmp/customer_adf_export" + +## Output Artifacts + +| File | Description | +|---|---| +| `inventory.json` | Classified activity inventory for the translate phase | +| `ast/*.json` | Typed AST for each pipeline | +| `parse_errors.json` | Any files that failed to parse | diff --git a/skills/migrate/SKILL.md b/skills/migrate/SKILL.md new file mode 100644 index 0000000..222e06d --- /dev/null +++ b/skills/migrate/SKILL.md @@ -0,0 +1,201 @@ +--- +name: migrate +description: > + End-to-end migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs. + Orchestrates ingest, translate, and prepare phases in sequence. +triggers: + - "migrate ADF" + - "migrate pipelines" + - "ADF to Databricks" + - "migrate to Lakeflow" + - "ADF migration" + - "convert ADF to Lakeflow" + - "migrate data factory" +--- + +# End-to-End ADF to Databricks Migration + +Orchestrate the complete migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. This skill runs all three phases in sequence: ingest, translate, prepare. + +## Context + +This is the top-level orchestration skill. It runs the full migration pipeline: + +1. **Ingest** — Parse ADF JSON exports into a typed inventory +2. **Translate** — Convert ADF activities to Databricks IR (deterministic + agentic) +3. **Prepare** — Generate Databricks Declarative Automation Bundles for deployment + +Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. + +## Workflow + +Follow these steps in order: + +### Step 1 — Gather inputs + +Ask the user for all required inputs upfront: + +| Parameter | Description | Required | Default | +|---|---|---|---| +| ADF source path | UC volume path or local directory with ADF JSON files | Yes | — | +| Output directory | Root directory for all flowx output | No | `./orchestra_output/` | +| Target catalog | Unity Catalog catalog for tables/volumes | No | `main` | +| Target schema | Schema within the catalog | No | `default` | +| Bundle name | Name for the generated DABs project | No | derived from pipelines | + +Example prompt: + +> To migrate your ADF pipelines, I need: +> 1. Where are your ADF JSON exports? (UC volume path like `/Volumes/main/default/adf_export` or local directory) +> 2. Where should I write the output? (default: `./orchestra_output/`) +> 3. What target catalog and schema? (default: `main.default`) + +### Step 2 — Phase 1: Ingest + +Invoke the `flowx:ingest` skill with the ADF source path and output directory set to `/ingest/`. + +Wait for the ingest to complete and present the inventory summary: + +``` +Phase 1: Ingest — Complete +========================== +Pipelines parsed: 12 +Total activities: 47 + Deterministic: 35 (74.5%) + Agentic: 10 (21.3%) + Unsupported: 2 ( 4.3%) +Coverage: 95.7% +``` + +### Step 3 — Checkpoint: confirm proceed + +Ask the user to review the inventory and confirm before continuing: + +> The ingest phase found 47 activities across 12 pipelines. 95.7% have a translation path (74.5% deterministic, 21.3% agentic). 2 activities are unsupported and will need manual handling. +> +> Proceed to the translation phase? (yes/no) + +If the user says no, explain the options: +- Re-run ingest with a different source directory +- Review the `inventory.json` to understand unsupported activities +- Manually classify activities before proceeding + +If the user says yes, proceed to step 4. + +### Step 4 — Phase 2: Translate + +Invoke the `flowx:translate` skill with: +- Inventory path: `/ingest/inventory.json` +- ADF source dir: the original ADF source path +- Output dir: `/translate/` + +Wait for the translation to complete and present the summary: + +``` +Phase 2: Translate — Complete +============================= +Deterministic translated: 35 (74.5%) +Agentic translated: 8 (17.0%) +Failed: 4 ( 8.5%) +Overall coverage: 91.5% +``` + +### Step 5 — Present translation details + +Show the user: +1. What was translated deterministically (bulk — just counts by type) +2. What was translated via agentic skills (list each with the skill used) +3. What failed and why (list each with the failure reason) + +For failures, suggest: +- Manual notebook creation +- Retry with additional context +- Skip and add placeholder + +### Step 6 — Checkpoint: confirm proceed to bundle generation + +> Translation is 91.5% complete. 4 activities could not be translated automatically. +> Options: +> 1. Proceed to bundle generation (failed activities will get placeholder tasks) +> 2. Retry failed translations with more context +> 3. Stop here and review the translation report +> +> What would you like to do? + +### Step 7 — Phase 3: Prepare + +Invoke the `flowx:prepare` skill with: +- Translation report: `/translate/translation_report.json` +- Output dir: `/dab_output/` +- Catalog: user-specified or `main` +- Schema: user-specified or `default` + +### Step 8 — Present final summary + +Display the complete migration summary: + +``` +Migration Complete +================== + +Source: /Volumes/main/default/adf_export (12 ADF pipelines) +Output: ./orchestra_output/dab_output/ + +Coverage: + Total activities: 47 + Successfully translated: 43 (91.5%) + Placeholder tasks: 4 ( 8.5%) + +Generated Files: + dab_output/ + databricks.yml + resources/ (3 job definitions) + src/notebooks/ (12 notebooks) + setup/ (3 setup scripts) + tests/ (3 test files) + +Setup Required: + - Run setup/create_volumes.py to create UC volumes + - Run setup/create_secrets.py to configure secrets (review credentials first) + - Run setup/register_connections.py to register external connections + +Next Steps: + 1. cd ./orchestra_output/dab_output/ + 2. Review generated files, especially notebooks and setup scripts + 3. databricks bundle validate --target dev + 4. Run setup scripts on the target workspace + 5. databricks bundle deploy --target dev + 6. databricks bundle run --target dev + 7. Verify job output and promote to staging/prod +``` + +### Step 9 — Offer follow-up actions + +Ask if the user wants to: +1. Validate the bundle now (`databricks bundle validate`) +2. Deploy to dev (`databricks bundle deploy --target dev`) +3. Review specific generated files +4. Re-translate any failed activities +5. Export a migration report for documentation + +## Reference + +See `references/workflow.md` for a detailed description of the three-phase architecture. + +## Examples + +- "Migrate my ADF pipelines to Databricks" +- "Convert ADF to Lakeflow jobs" +- "ADF to Databricks migration from /Volumes/main/default/adf_export" +- "Migrate data factory pipelines to catalog analytics, schema bronze" +- "Run the full ADF migration workflow" + +## Output Artifacts + +All artifacts from all three phases are produced under the output directory: + +| Directory | Phase | Contents | +|---|---|---| +| `ingest/` | Ingest | `inventory.json`, `ast/`, `parse_errors.json` | +| `translate/` | Translate | `translation_report.json`, `ir/`, `notebooks/`, `agentic_results/` | +| `dab_output/` | Prepare | `databricks.yml`, `resources/`, `src/`, `setup/`, `tests/` | diff --git a/skills/migrate/references/workflow.md b/skills/migrate/references/workflow.md new file mode 100644 index 0000000..93e997d --- /dev/null +++ b/skills/migrate/references/workflow.md @@ -0,0 +1,167 @@ +# Flowx Migration Workflow + +End-to-end architecture for migrating Azure Data Factory (ADF) pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles (DABs). + +## Overview + +Flowx follows a three-phase pipeline architecture. Each phase is independently runnable and produces artifacts consumed by the next phase. The design principle is **deterministic-first, agentic fallback**: well-known ADF patterns are translated by fast, reliable Python code, while complex or ambiguous patterns are handled by LLM-assisted skills. + +``` +ADF JSON Exports + | + v + Phase 1: INGEST + (parse + classify) + | + v + inventory.json + | + v + Phase 2: TRANSLATE + (deterministic + agentic) + | + v + translation_report.json + IR + notebooks + | + v + Phase 3: PREPARE + (bundle generation) + | + v + Databricks DABs Project + (databricks.yml + resources/ + src/ + setup/) + | + v + databricks bundle deploy +``` + +## Phase 1: Ingest + +**Skill:** `flowx:ingest` + +**Input:** Directory of ADF JSON export files (from ARM template export, Azure DevOps, or manual export) + +**Process:** +1. Discover and parse all JSON files (pipelines, datasets, linked services, triggers) +2. Build a typed AST for each pipeline, resolving activity references +3. Classify every activity into a translation strategy: + - **Deterministic** — a built-in translator handles this type + - **Agentic** — requires LLM-assisted translation + - **Unsupported** — no known translation path +4. Generate the inventory with summary statistics + +**Output:** +- `inventory.json` — classified activity inventory +- `ast/*.json` — typed AST per pipeline +- `parse_errors.json` — any parsing failures + +**Key decisions:** +- Classification is based on the activity `type` field, not the activity content. This makes classification fast and deterministic. +- Datasets and linked services are parsed for context but not independently translated — they inform the activity translators. +- Triggers are included in the inventory and translated in phase 2. + +## Phase 2: Translate + +**Skill:** `flowx:translate` + +**Input:** `inventory.json` from phase 1 + original ADF JSON files + +**Process:** +1. Run deterministic translators for all activities classified as `deterministic` +2. For each `agentic` activity, invoke the appropriate skill from the `adf-to-databricks-plugin`: + - `adf-dataflow-converter` for ExecuteDataFlow activities + - `adf-pipeline-converter` for control flow and external call activities + - `adf-expression-translator` for complex ADF expression conversion + - `adf-trigger-converter` for trigger schedule translation +3. Merge deterministic and agentic results into a unified translation report +4. Generate Databricks IR (intermediate representation) for each activity + +**Output:** +- `translation_report.json` — unified report with IR for all activities +- `ir/*.json` — Databricks IR per activity +- `notebooks/*.py` — generated helper notebooks +- `agentic_results/*.json` — raw agentic skill outputs + +**Key decisions:** +- Deterministic translators run first because they are fast and reliable. Agentic skills are only invoked for gaps. +- The IR is an intermediate format that decouples translation from DABs generation. This allows the prepare phase to target different output formats in the future. +- Each deterministic translator is a standalone Python module in `src/flowx/translator/activity_translators/`. Adding support for a new activity type means adding a new module. +- Agentic results are saved separately before merging, so they can be inspected, retried, or manually overridden. + +## Phase 3: Prepare + +**Skill:** `flowx:prepare` + +**Input:** `translation_report.json` from phase 2 + +**Process:** +1. Read the unified translation report +2. For each pipeline, generate a Databricks Lakeflow Job definition (YAML) +3. Map ADF activity dependencies to DABs task dependencies +4. Generate the `databricks.yml` root configuration with target environments +5. Copy/generate notebooks to `src/notebooks/` +6. Generate setup scripts for infrastructure (volumes, secrets, connections) +7. Generate skeleton test files + +**Output:** +- Complete DABs project ready for `databricks bundle validate` and `databricks bundle deploy` + +**Key decisions:** +- One Databricks Job per ADF pipeline. Activities within a pipeline become tasks within the job. +- ADF dependency chains (`dependsOn`) are mapped to DABs task dependencies. +- Environment-specific values (catalog, schema, warehouse ID) are parameterized as bundle variables. +- Three default targets: dev, staging, prod. Each can have different variable values. +- Setup scripts are generated but not run automatically — the user must review and run them. + +## Design Principles + +### Deterministic-first, agentic fallback + +The majority of ADF activities (Copy, DatabricksNotebook, ForEach, IfCondition, etc.) have well-defined Databricks equivalents. Building deterministic translators for these ensures: +- **Speed** — no LLM calls needed for 70-80% of activities +- **Reliability** — same input always produces same output +- **Testability** — translators are unit-testable Python functions +- **Cost** — no token consumption for the bulk of translation + +Agentic skills handle the remaining 20-30% that require interpretation or generation. This is where LLM reasoning adds value — complex data flows, exotic activity types, expression conversion. + +### Phased execution with checkpoints + +Each phase produces a persistent artifact (`inventory.json`, `translation_report.json`, DABs project). This allows: +- **Resumability** — if a phase fails, restart from that phase +- **Inspection** — review intermediate artifacts before proceeding +- **Modularity** — run phases independently or skip phases +- **Debugging** — trace issues back to specific phases + +### IR as decoupling layer + +The Databricks IR (intermediate representation) sits between ADF semantics and DABs output. This decouples: +- **Parsing** — understanding ADF JSON structure +- **Semantic mapping** — translating ADF concepts to Databricks concepts +- **Serialization** — writing DABs YAML and notebooks + +This means the prepare phase could target different output formats (Terraform, raw API calls, etc.) without changing the translation logic. + +## ADF Concepts to Databricks Mapping + +| ADF Concept | Databricks Equivalent | +|---|---| +| Pipeline | Lakeflow Job | +| Activity | Job Task | +| Activity dependency (`dependsOn`) | Task dependency (`depends_on`) | +| Pipeline parameter | Job parameter | +| Pipeline variable | Task value | +| Linked service | Unity Catalog connection or secret scope | +| Dataset | Unity Catalog table/volume path | +| Data flow | DLT pipeline or PySpark notebook | +| Trigger (schedule) | Job schedule (`quartz_cron_expression`) | +| Trigger (tumbling window) | Job schedule (periodic) | +| Trigger (event) | File arrival trigger | +| Integration runtime | Job cluster or serverless compute | + +## Error Handling + +- **Parse errors** — logged to `parse_errors.json`, skipped in inventory +- **Translation failures** — marked as `failed` in translation report, get placeholder tasks in DABs +- **Agentic failures** — saved with error details, can be retried with additional context +- **Unsupported activities** — warned at ingest, get placeholder tasks with TODO comments in DABs diff --git a/skills/prepare/SKILL.md b/skills/prepare/SKILL.md new file mode 100644 index 0000000..2ba7353 --- /dev/null +++ b/skills/prepare/SKILL.md @@ -0,0 +1,163 @@ +--- +name: prepare +description: > + Generate Databricks Declarative Automation Bundles (DABs) from translated IR, + including job definitions, notebooks, and setup scripts. +triggers: + - "prepare bundles" + - "generate DABs" + - "create bundles" + - "prepare deployment" + - "generate bundles" + - "build DABs" +--- + +# Prepare Databricks Declarative Automation Bundles + +Generate deployment-ready Databricks Declarative Automation Bundles (DABs) from the translated intermediate representation, including job definitions, notebooks, and infrastructure setup scripts. + +## Context + +This is phase 3 of the flowx migration workflow. It consumes the `translation_report.json` produced by the `translate` skill and generates a complete DABs project that can be validated and deployed with the Databricks CLI. + +The output is a standard DABs project with: +- `databricks.yml` — the bundle configuration +- `resources/` — job and pipeline YAML definitions +- `src/notebooks/` — generated and helper notebooks +- `setup/` — infrastructure setup scripts (volumes, secrets, connections) + +## Workflow + +Follow these steps in order: + +### Step 1 — Locate the translation report + +Read `translation_report.json` from the translate phase. If the path is not in conversation context, ask the user: + +> Where is the translation_report.json from the translate phase? (default: `./orchestra_output/translate/translation_report.json`) + +Validate the file exists and all required translations have status `translated`. + +### Step 2 — Gather deployment parameters + +Ask the user for the following (provide defaults): + +| Parameter | Description | Default | +|---|---|---| +| Target catalog | Unity Catalog catalog for tables/volumes | `main` | +| Target schema | Schema within the catalog | `default` | +| Output directory | Where to write the DABs project | `./dab_output/` | +| Bundle name | Name for the DABs project | derived from first pipeline name | +| Target environments | Deployment targets to configure | `dev, staging, prod` | +| Warehouse ID | SQL warehouse for SQL tasks (optional) | prompt if SQL tasks exist | + +### Step 3 — Run bundle generation + +Execute the DAB writer: + +```bash +python3 /src/flowx/bundler/dab_writer.py \ + --report \ + --output-dir \ + --catalog \ + --schema \ + --bundle-name \ + --targets +``` + +Where: +- `` is the root of the flowx plugin +- `` is the path to `translation_report.json` +- Other parameters are from step 2 + +### Step 4 — Present the generated file tree + +Show the user what was generated: + +``` +dab_output/ + databricks.yml + resources/ + etl_main_job.yml + etl_secondary_job.yml + transform_dlt_pipeline.yml + src/ + notebooks/ + copy_from_blob.py + lookup_config.py + web_activity_call.py + set_variable_helper.py + setup/ + create_volumes.py + create_secrets.py + register_connections.py + tests/ + test_etl_main.py +``` + +### Step 5 — Explain setup tasks + +If the `setup/` directory was generated, explain what each script does: + +**create_volumes.py** — Creates Unity Catalog volumes required by the migrated jobs. These volumes replace Azure Blob Storage or ADLS references from ADF. Run this once per environment. + +**create_secrets.py** — Creates Databricks secret scopes and secrets for connection credentials that were in ADF linked services. Review the secret values and populate them manually or via your secrets management system. + +**register_connections.py** — Registers Unity Catalog connections for external data sources (SQL Server, REST APIs, etc.) that were referenced in ADF linked services. + +Emphasize that the user should review these scripts before running them, especially `create_secrets.py` which will need actual credential values. + +### Step 6 — Explain the generated bundle structure + +Briefly describe: +- **databricks.yml** — The root bundle config with workspace, target environments (dev/staging/prod), and variable definitions. Variables are parameterized for environment-specific values (catalog, schema, warehouse). +- **resources/*.yml** — One YAML file per Databricks Lakeflow Job (one per ADF pipeline). Each job contains tasks mapped from ADF activities, with dependencies matching the original ADF dependency chains. +- **src/notebooks/*.py** — Python notebooks for activities that translate to notebook_task. These contain the actual data movement or transformation logic. +- **tests/*.py** — Skeleton test files for validating the migrated jobs. + +### Step 7 — Suggest next steps + +Present the following next steps: + +``` +Next Steps +========== +1. Review generated files: + cd + cat databricks.yml + +2. Validate the bundle: + databricks bundle validate --target dev + +3. Run setup scripts (if generated): + databricks bundle run setup_volumes --target dev + +4. Deploy to dev: + databricks bundle deploy --target dev + +5. Test the deployed jobs: + databricks bundle run --target dev + +6. Promote to staging/prod: + databricks bundle deploy --target staging + databricks bundle deploy --target prod +``` + +Recommend running `databricks bundle validate` first to catch any configuration issues before deployment. + +## Examples + +- "Prepare the bundles" +- "Generate DABs for the translated pipelines" +- "Create deployment bundles targeting catalog 'analytics' and schema 'bronze'" +- "Build the DABs project in ./output/my_migration/" + +## Output Artifacts + +| File | Description | +|---|---| +| `databricks.yml` | Root bundle configuration | +| `resources/*.yml` | Job and pipeline YAML definitions | +| `src/notebooks/*.py` | Generated notebooks | +| `setup/*.py` | Infrastructure setup scripts | +| `tests/*.py` | Skeleton test files | diff --git a/skills/translate/SKILL.md b/skills/translate/SKILL.md new file mode 100644 index 0000000..283c9b9 --- /dev/null +++ b/skills/translate/SKILL.md @@ -0,0 +1,207 @@ +--- +name: translate +description: > + Translate parsed ADF pipeline AST into Databricks IR (intermediate representation). + Runs deterministic translators for known activity types, then invokes agentic skills + from adf-to-databricks-plugin for gaps. +triggers: + - "translate ADF" + - "convert ADF" + - "translate pipelines" + - "convert pipelines" + - "run translation" +--- + +# Translate ADF to Databricks IR + +Convert the parsed ADF inventory into Databricks intermediate representation (IR) using deterministic translators for known types and agentic fallback for complex/unknown types. + +## Context + +This is phase 2 of the flowx migration workflow. It consumes the `inventory.json` produced by the `ingest` skill and produces a `translation_report.json` that the `prepare` skill uses to generate Databricks Declarative Automation Bundles. + +The translation follows a **deterministic-first** strategy: +1. Activities with known, well-defined mappings are translated by built-in Python translators (fast, reliable, no LLM needed) +2. Activities that require interpretation, complex expression conversion, or lack a direct mapping are handled by agentic skills from the `adf-to-databricks-plugin` (LLM-assisted) + +This approach maximizes reliability while covering the long tail of ADF activity types. + +## Workflow + +Follow these steps in order: + +### Step 1 — Locate the inventory + +Read `inventory.json` from the ingest phase. If the path is not already in conversation context, ask the user: + +> Where is the inventory.json from the ingest phase? (default: `./orchestra_output/ingest/inventory.json`) + +Validate the file exists and is well-formed. + +### Step 2 — Run deterministic translation + +Execute the translation engine on all deterministic activities: + +```bash +python3 /src/flowx/translator/engine.py \ + --inventory \ + --source-dir \ + --output-dir +``` + +Where: +- `` is the root of the flowx plugin +- `` is the path to `inventory.json` +- `` is the original ADF JSON directory (from the ingest phase) +- `` is where to write translation output (default: `./orchestra_output/translate/`) + +This produces: +- `translation_report.json` — results for deterministic activities + placeholders for agentic gaps +- `ir/` directory — the Databricks IR for each translated activity +- `notebooks/` directory — any generated helper notebooks + +### Step 3 — Read the translation report + +Read `translation_report.json`. It has this structure: + +```json +{ + "inventory_path": "/path/to/inventory.json", + "generated_at": "2026-04-07T12:30:00Z", + "translations": [ + { + "pipeline": "ETL_Main", + "activity": "CopyFromBlob", + "type": "Copy", + "strategy": "deterministic", + "status": "translated", + "ir": { + "task_key": "copy_from_blob", + "task_type": "notebook_task", + "notebook_path": "notebooks/copy_from_blob.py", + "parameters": { "source": "abfss://...", "target": "..." } + } + }, + { + "pipeline": "ETL_Main", + "activity": "TransformData", + "type": "ExecuteDataFlow", + "strategy": "agentic", + "status": "pending", + "raw_activity_json": { "...": "..." }, + "target_skill": "adf-to-databricks:adf-dataflow-converter" + } + ], + "summary": { + "total": 47, + "deterministic_translated": 35, + "agentic_pending": 10, + "failed": 2 + } +} +``` + +### Step 4 — Handle agentic gaps + +For each translation with `"status": "pending"` and `"strategy": "agentic"`, invoke the appropriate skill from the `adf-to-databricks-plugin`. Route by activity type: + +**ExecuteDataFlow activities:** +Invoke `adf-to-databricks:adf-dataflow-converter` with the raw activity JSON and associated data flow definition. Provide context: +- The raw `typeProperties` from the ADF activity +- The data flow JSON definition (if available in the source directory under `dataflow/`) +- The linked service configurations for source/sink connections +- Target catalog and schema for the DLT pipeline or PySpark notebook output + +**Control flow activities (Switch, Until, Wait, Filter, AppendVariable):** +Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: +- The full pipeline JSON containing the activity +- Any nested activities within the control flow +- Variable definitions from the pipeline +- The desired Databricks task type mapping + +**Stored procedures and external calls (SqlServerStoredProcedure, AzureFunction, WebHook, Custom):** +Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: +- The linked service configuration for the target system +- Connection details and authentication method +- Any parameters or request bodies + +**Complex expressions:** +If any activity (deterministic or agentic) contains ADF expressions that the deterministic translator could not resolve, invoke `adf-to-databricks:adf-expression-translator` with: +- The raw expression string (e.g., `@pipeline().parameters.inputPath`) +- The expression context (pipeline parameters, variables, activity outputs) +- The target format (Python f-string, Spark SQL, task parameter reference) + +**Trigger definitions:** +Invoke `adf-to-databricks:adf-trigger-converter` with: +- The trigger JSON definition +- The associated pipeline references +- Target: Databricks job schedule configuration (quartz_cron_expression, periodic, or file_arrival) + +### Step 5 — Collect agentic results + +Each agentic skill invocation produces a translation result. Collect all results into `/agentic_results/`: +- Save each result as `__.json` +- Include the generated IR, any notebooks, and metadata + +### Step 6 — Merge agentic results + +Run the merge step to combine deterministic and agentic translations: + +```bash +python3 /src/flowx/translator/engine.py \ + --merge-agentic \ + --report \ + --agentic-results +``` + +This updates `translation_report.json` with the agentic results merged in, changing their status from `pending` to `translated` (or `failed` if the agentic skill could not produce a result). + +### Step 7 — Present translation summary + +Display a summary to the user: + +``` +Translation Summary +=================== +Total activities: 47 +Deterministic translated: 35 (74.5%) +Agentic translated: 8 (17.0%) +Failed: 4 ( 8.5%) + +Overall coverage: 91.5% + +Failed translations: + - ETL_Main / RunSSIS (ExecuteSSISPackage) — no translator available + - ETL_Main / CustomTask (Custom) — agentic skill returned error + ... + +Generated artifacts: + - translation_report.json + - ir/ (43 files) + - notebooks/ (12 files) +``` + +If coverage is below 100%, explain the options for failed translations: +1. Manual notebook creation for unsupported types +2. Retry agentic translation with additional context +3. Skip the activity and add a placeholder task in the DAB + +## Reference + +See `references/activity-mapping.md` for the complete mapping between ADF activity types and translation strategies. + +## Examples + +- "Translate the ADF pipelines" +- "Convert ADF to Databricks" +- "Run the translation on the inventory from the ingest step" +- "Translate the parsed pipelines using deterministic + agentic" + +## Output Artifacts + +| File | Description | +|---|---| +| `translation_report.json` | Full translation report with IR for all activities | +| `ir/*.json` | Databricks IR for each translated activity | +| `notebooks/*.py` | Generated helper notebooks | +| `agentic_results/*.json` | Raw results from agentic skill invocations | diff --git a/skills/translate/references/activity-mapping.md b/skills/translate/references/activity-mapping.md new file mode 100644 index 0000000..137d980 --- /dev/null +++ b/skills/translate/references/activity-mapping.md @@ -0,0 +1,174 @@ +# ADF Activity Type to Databricks Translation Mapping + +This reference defines the mapping between Azure Data Factory activity types and their translation strategy in flowx. + +## Strategy Definitions + +- **Deterministic** — Handled by a built-in Python translator module. Fast, reliable, no LLM required. These mappings are well-defined and produce consistent output. +- **Agentic** — Handled by an LLM-assisted skill from the `adf-to-databricks-plugin`. Required when the ADF activity has complex semantics, requires interpretation, or lacks a direct Databricks equivalent. +- **Unsupported** — No automated translation path. Requires manual intervention. + +## Activity Mapping Table + +| ADF Activity Type | Strategy | Translator / Skill | Databricks Target | +|---|---|---|---| +| Copy | Deterministic | `copy.py` | `notebook_task` (Auto Loader / COPY INTO / JDBC) or `pipeline_task` (DLT) | +| DatabricksNotebook | Deterministic | `notebook.py` | `notebook_task` | +| DatabricksSparkJar | Deterministic | `spark_jar.py` | `spark_jar_task` | +| DatabricksSparkPython | Deterministic | `spark_python.py` | `spark_python_task` | +| ForEach | Deterministic | `for_each.py` | `for_each_task` | +| IfCondition | Deterministic | `if_condition.py` | `condition_task` | +| SetVariable | Deterministic | `set_variable.py` | `notebook_task` (task values) | +| Lookup | Deterministic | `lookup.py` | `notebook_task` | +| WebActivity | Deterministic | `web_activity.py` | `notebook_task` | +| Delete | Deterministic | `delete.py` | `notebook_task` (`dbutils.fs.rm`) | +| ExecutePipeline | Deterministic | `execute_pipeline.py` | `run_job_task` | +| DatabricksJob | Deterministic | `databricks_job.py` | `run_job_task` | +| Switch | Deterministic | `switch.py` | chained `condition_task`s | +| Wait | Deterministic | `wait.py` | `notebook_task` (`time.sleep`) | +| ExecuteDataFlow | Agentic | `adf-to-databricks:adf-dataflow-converter` | DLT pipeline or PySpark notebook | +| Until | Agentic | `adf-to-databricks:adf-pipeline-converter` | while-loop notebook | +| Filter | Deterministic | `filter.py` | `notebook_task` (filter array + task values) | +| AppendVariable | Deterministic | `append_variable.py` | `notebook_task` (append to array task value) | +| SqlServerStoredProcedure | Agentic | `adf-to-databricks:adf-pipeline-converter` | SQL notebook | +| AzureFunction | Agentic | `adf-to-databricks:adf-pipeline-converter` | webhook/REST notebook | +| WebHook | Agentic | `adf-to-databricks:adf-pipeline-converter` | REST notebook | +| Custom | Agentic | `adf-to-databricks:adf-pipeline-converter` | custom notebook | +| ExecuteSSISPackage | Agentic | `adf-to-databricks:adf-pipeline-converter` | PySpark notebook | +| AzureMLExecutePipeline | Agentic | `adf-to-databricks:adf-pipeline-converter` | MLflow notebook | +| Triggers (Schedule) | Agentic | `adf-to-databricks:adf-trigger-converter` | `quartz_cron_expression` | +| Triggers (Tumbling Window) | Agentic | `adf-to-databricks:adf-trigger-converter` | periodic schedule | +| Triggers (Blob Event) | Agentic | `adf-to-databricks:adf-trigger-converter` | `file_arrival` trigger | + +## Deterministic Translator Details + +### Copy (`copy.py`) + +Translates ADF Copy activities based on source/sink types: +- **Blob/ADLS to Delta** — Auto Loader (`cloudFiles`) notebook or COPY INTO +- **SQL to Delta** — JDBC read notebook +- **Delta to Delta** — DLT pipeline with `pipeline_task` +- Handles type mapping, column mapping, and partitioning from ADF typeProperties + +### DatabricksNotebook (`notebook.py`) + +Direct 1:1 mapping. Extracts: +- `notebookPath` from typeProperties +- `baseParameters` mapped to task parameters +- Linked service cluster config mapped to job cluster or existing cluster reference + +### DatabricksSparkJar (`spark_jar.py`) + +Maps to `spark_jar_task` with: +- `mainClassName` and `parameters` from typeProperties +- JAR library references from linked service or activity settings + +### DatabricksSparkPython (`spark_python.py`) + +Maps to `spark_python_task` with: +- `pythonFile` path and `parameters` from typeProperties +- Library dependencies + +### ForEach (`for_each.py`) + +Maps to `for_each_task` with: +- `items` expression translated to task parameter or task values reference +- `isSequential` mapped to concurrency setting +- Inner activities translated recursively + +### IfCondition (`if_condition.py`) + +Maps to `condition_task` with: +- `expression` translated to a condition expression +- `ifTrueActivities` and `ifFalseActivities` translated recursively as nested tasks + +### SetVariable (`set_variable.py`) + +Maps to a lightweight notebook that sets task values: +- Variable name becomes task value key +- Variable value expression becomes the task value + +### Lookup (`lookup.py`) + +Maps to a notebook that reads data and returns results via task values: +- Source dataset determines the read method (SQL query, file read, etc.) +- `firstRowOnly` setting determines output shape + +### WebActivity (`web_activity.py`) + +Maps to a notebook that makes HTTP requests: +- URL, method, headers, body from typeProperties +- Authentication from linked service +- Response captured as task value + +### Delete (`delete.py`) + +Maps to a notebook using `dbutils.fs.rm`: +- Dataset path determines the target +- `recursive` flag from typeProperties + +### ExecutePipeline (`execute_pipeline.py`) + +Maps to `run_job_task`: +- Referenced pipeline name mapped to target job name +- `parameters` mapped to job parameters +- `waitOnCompletion` maps to task dependency behavior + +### DatabricksJob (`databricks_job.py`) + +Maps to `run_job_task`: +- Existing Databricks job reference preserved +- Parameters forwarded + +### Switch (`switch.py`) + +Maps to chained `condition_task` nodes: +- `typeProperties.on.value` expression evaluated against each case value +- Each case becomes an equality check in a nested condition chain +- Default activities fire when no case matches (the final `if_false` branch) +- Child activities within each case are translated recursively + +### Wait (`wait.py`) + +Maps to a `notebook_task` with a `time.sleep()` call: +- `typeProperties.waitTimeInSeconds` becomes the sleep duration +- Generated notebook accepts a `wait_seconds` widget parameter for runtime override + +### Filter (`filter.py`) + +Maps to a `notebook_task` that filters an array: +- `typeProperties.items.value` expression provides the input array +- `typeProperties.condition.value` expression provides the filter predicate +- Generated notebook evaluates the array, applies the condition, and stores the filtered result as a task value via `dbutils.jobs.taskValues.set()` + +### AppendVariable (`append_variable.py`) + +Maps to a `notebook_task` that appends a value to an array variable: +- `typeProperties.variableName` identifies the target array variable +- `typeProperties.value` expression provides the value to append +- Threads context like SetVariable, registering the variable mapping +- Generated notebook reads the current array from a task value, appends the new value, and writes the updated array back via `dbutils.jobs.taskValues.set()` + +## Agentic Translation Notes + +Agentic translations are handled by skills from the `adf-to-databricks-plugin` (`birbalin25/adf-to-databricks-plugin`). These skills use LLM reasoning to: + +1. Interpret complex ADF semantics that lack direct Databricks equivalents +2. Convert ADF expressions to Python/SQL equivalents +3. Generate purpose-built notebooks for activities without task-level mappings +4. Handle data flow visual transformations (joins, pivots, derived columns, etc.) +5. Map trigger schedules accounting for timezone and windowing semantics + +The agentic approach trades speed for coverage — it can handle the long tail of ADF activity types that would be impractical to build deterministic translators for. + +## Expression Function Coverage + +Flowx deterministically translates 73 of 84 ADF expression functions to Python notebook code. The remaining 11 functions require agentic translation: + +- `dataUri`, `dataUriToBinary`, `dataUriToString`, `decodeDataUri` — Data URI encoding/decoding (rare in practice) +- `uriComponentToBinary` — URI component to binary conversion +- `xml`, `xpath` — XML parsing and XPath evaluation (complex DOM handling) +- `convertFromUtc`, `convertTimeZone`, `convertToUtc` — Timezone conversions using Windows timezone names +- `ticks` — .NET DateTime ticks (100-nanosecond intervals since 0001-01-01) + +See [expression-functions.md](expression-functions.md) for the complete function mapping reference. diff --git a/skills/translate/references/expression-functions.md b/skills/translate/references/expression-functions.md new file mode 100644 index 0000000..90a270a --- /dev/null +++ b/skills/translate/references/expression-functions.md @@ -0,0 +1,154 @@ +# ADF Expression Function Translation Reference + +This reference documents how all 84 ADF expression functions are translated by `flowx.parser.expression_parser`. + +## Translation Categories + +- **notebook_code** — Translated to Python code embedded in the generated notebook body. +- **dab_ref** — Mapped to a DAB dynamic value reference (rare for functions; primarily for `@pipeline()`, `@activity()`, `@variables()`, `@item()`). +- **agentic** — Too complex for deterministic translation; returns `None` and requires LLM-assisted translation. + +## String Functions (12) — notebook_code + +| ADF Function | Python Translation | Notes | +|---|---|---| +| `concat(a, b, ...)` | `str(a) + str(b) + ...` | Variadic; handled by both dedicated resolver and dispatch table | +| `endsWith(text, search)` | `str(text).endswith(str(search))` | | +| `guid()` | `str(uuid4())` | | +| `guid('N')` | `str(uuid4()).replace('-', '')` | No-dash format | +| `indexOf(text, search)` | `str(text).lower().find(str(search).lower())` | ADF is case-insensitive | +| `lastIndexOf(text, search)` | `str(text).lower().rfind(str(search).lower())` | ADF is case-insensitive | +| `replace(text, old, new)` | `str(text).replace(str(old), str(new))` | | +| `split(text, delim)` | `str(text).split(str(delim))` | | +| `startsWith(text, search)` | `str(text).lower().startswith(str(search).lower())` | ADF is case-insensitive | +| `substring(text, start, length)` | `str(text)[int(start):int(start)+int(length)]` | | +| `toLower(text)` | `str(text).lower()` | | +| `toUpper(text)` | `str(text).upper()` | | +| `trim(text)` | `str(text).strip()` | | + +## Collection Functions (10) — notebook_code + +| ADF Function | Python Translation | +|---|---| +| `contains(collection, value)` | `(value in collection)` | +| `empty(collection)` | `(len(collection) == 0)` | +| `first(collection)` | `collection[0]` | +| `intersection(c1, c2, ...)` | `list(set(c1) & set(c2) & ...)` | +| `join(array, delim)` | `str(delim).join(str(x) for x in array)` | +| `last(collection)` | `collection[-1]` | +| `length(collection)` | `len(collection)` | +| `skip(collection, count)` | `collection[int(count):]` | +| `take(collection, count)` | `collection[:int(count)]` | +| `union(c1, c2, ...)` | `list(set(c1) \| set(c2) \| ...)` | + +## Logical Functions (9) — notebook_code + +| ADF Function | Python Translation | +|---|---| +| `and(a, b)` | `(a and b)` | +| `equals(a, b)` | `(a == b)` | +| `greater(a, b)` | `(a > b)` | +| `greaterOrEquals(a, b)` | `(a >= b)` | +| `if(expr, trueVal, falseVal)` | `(trueVal if expr else falseVal)` | +| `less(a, b)` | `(a < b)` | +| `lessOrEquals(a, b)` | `(a <= b)` | +| `not(expr)` | `(not expr)` | +| `or(a, b)` | `(a or b)` | + +## Conversion Functions (24) — notebook_code / agentic + +| ADF Function | Python Translation | Status | +|---|---|---| +| `array(value)` | `[value]` | notebook_code | +| `base64(value)` | `base64.b64encode(str(value).encode()).decode()` | notebook_code | +| `base64ToBinary(value)` | `base64.b64decode(value)` | notebook_code | +| `base64ToString(value)` | `base64.b64decode(value).decode()` | notebook_code | +| `binary(value)` | `str(value).encode()` | notebook_code | +| `bool(value)` | `bool(value)` | notebook_code | +| `coalesce(a, b, ...)` | `next((x for x in [a, b, ...] if x is not None), None)` | notebook_code | +| `createArray(a, b, ...)` | `[a, b, ...]` | notebook_code | +| `dataUri(value)` | — | **agentic** (rare, complex encoding) | +| `dataUriToBinary(value)` | — | **agentic** | +| `dataUriToString(value)` | — | **agentic** | +| `decodeBase64(value)` | `base64.b64decode(value).decode()` | notebook_code (alias of base64ToString) | +| `decodeDataUri(value)` | — | **agentic** | +| `decodeUriComponent(value)` | `urllib.parse.unquote(value)` | notebook_code | +| `encodeUriComponent(value)` | `urllib.parse.quote(str(value), safe='')` | notebook_code | +| `float(value)` | `float(value)` | notebook_code | +| `int(value)` | `int(value)` | notebook_code | +| `json(value)` | `json.loads(value)` | notebook_code | +| `string(value)` | `str(value)` | notebook_code | +| `uriComponent(value)` | `urllib.parse.quote(str(value), safe='')` | notebook_code (alias of encodeUriComponent) | +| `uriComponentToBinary(value)` | — | **agentic** | +| `uriComponentToString(value)` | `urllib.parse.unquote(value)` | notebook_code (alias of decodeUriComponent) | +| `xml(value)` | — | **agentic** (XML handling complex) | +| `xpath(xml, expr)` | — | **agentic** (XPath complex) | + +## Math Functions (9) — notebook_code + +| ADF Function | Python Translation | +|---|---| +| `add(a, b)` | `(a + b)` | +| `div(a, b)` | `(a // b)` (integer division, matching ADF semantics) | +| `max(a, b, ...)` | `max(a, b, ...)` | +| `min(a, b, ...)` | `min(a, b, ...)` | +| `mod(a, b)` | `(a % b)` | +| `mul(a, b)` | `(a * b)` | +| `rand(min, max)` | `random.randint(min, max-1)` | +| `range(start, count)` | `list(range(start, start + count))` | +| `sub(a, b)` | `(a - b)` | + +## Date/Time Functions (20) — notebook_code / agentic + +All date functions use `from datetime import datetime, timezone, timedelta`. + +| ADF Function | Python Translation | Status | +|---|---|---| +| `addDays(ts, days, fmt?)` | `(datetime.fromisoformat(ts) + timedelta(days=days)).strftime(fmt)` | notebook_code | +| `addHours(ts, hours, fmt?)` | `(datetime.fromisoformat(ts) + timedelta(hours=hours)).strftime(fmt)` | notebook_code | +| `addMinutes(ts, minutes, fmt?)` | `(datetime.fromisoformat(ts) + timedelta(minutes=minutes)).strftime(fmt)` | notebook_code | +| `addSeconds(ts, seconds, fmt?)` | `(datetime.fromisoformat(ts) + timedelta(seconds=seconds)).strftime(fmt)` | notebook_code | +| `addToTime(ts, interval, unit, fmt?)` | `(datetime.fromisoformat(ts) + timedelta(**{unit}=interval)).strftime(fmt)` | notebook_code | +| `convertFromUtc(ts, tz, fmt?)` | — | **agentic** (timezone handling complex) | +| `convertTimeZone(ts, srcTz, destTz, fmt?)` | — | **agentic** (timezone handling complex) | +| `convertToUtc(ts, srcTz, fmt?)` | — | **agentic** (timezone handling complex) | +| `dayOfMonth(ts)` | `datetime.fromisoformat(ts).day` | notebook_code | +| `dayOfWeek(ts)` | `datetime.fromisoformat(ts).isoweekday() % 7` | notebook_code (ADF: 0=Sunday) | +| `dayOfYear(ts)` | `datetime.fromisoformat(ts).timetuple().tm_yday` | notebook_code | +| `formatDateTime(ts, fmt?)` | `datetime.fromisoformat(ts).strftime(converted_fmt)` | notebook_code | +| `getFutureTime(interval, unit, fmt?)` | `(datetime.now(timezone.utc) + timedelta(...)).strftime(fmt)` | notebook_code | +| `getPastTime(interval, unit, fmt?)` | `(datetime.now(timezone.utc) - timedelta(...)).strftime(fmt)` | notebook_code | +| `startOfDay(ts, fmt?)` | `datetime.fromisoformat(ts).replace(hour=0,...).strftime(fmt)` | notebook_code | +| `startOfHour(ts, fmt?)` | `datetime.fromisoformat(ts).replace(minute=0,...).strftime(fmt)` | notebook_code | +| `startOfMonth(ts, fmt?)` | `datetime.fromisoformat(ts).replace(day=1,...).strftime(fmt)` | notebook_code | +| `subtractFromTime(ts, interval, unit, fmt?)` | `(datetime.fromisoformat(ts) - timedelta(...)).strftime(fmt)` | notebook_code | +| `ticks(ts)` | — | **agentic** (.NET ticks conversion complex) | +| `utcNow(fmt?)` | `datetime.now(timezone.utc).strftime(fmt)` | notebook_code (dedicated handler) | + +## Summary + +| Category | Total | notebook_code | agentic | +|---|---|---|---| +| String | 12 | 12 | 0 | +| Collection | 10 | 10 | 0 | +| Logical | 9 | 9 | 0 | +| Conversion | 24 | 17 | 7 | +| Math | 9 | 9 | 0 | +| Date/Time | 20 | 16 | 4 | +| **Total** | **84** | **73** | **11** | + +## Agentic Functions (11 total) + +These functions return `None` from `resolve_expression()` and require LLM-assisted translation: + +1. `dataUri` — Data URI encoding (rare in ADF pipelines) +2. `dataUriToBinary` — Data URI to binary conversion +3. `dataUriToString` — Data URI to string conversion +4. `decodeDataUri` — Data URI decoding +5. `uriComponentToBinary` — URI component to binary +6. `xml` — XML parsing (complex DOM handling) +7. `xpath` — XPath evaluation (requires XML context) +8. `convertFromUtc` — UTC to timezone conversion (Windows timezone names) +9. `convertTimeZone` — Timezone conversion (Windows timezone names) +10. `convertToUtc` — Timezone to UTC conversion (Windows timezone names) +11. `ticks` — .NET DateTime ticks (100-nanosecond intervals since 0001-01-01) diff --git a/src/AGENTS.md b/src/AGENTS.md new file mode 100644 index 0000000..2198208 --- /dev/null +++ b/src/AGENTS.md @@ -0,0 +1,103 @@ +# Source Code Conventions + + This file documents *code-style* expectations for everything under +`src/flowx/`. When in doubt, match the existing code in the +neighbouring module. This is a companion to the top-level [AGENTS.md](../AGENTS.md) + file which documents the plugin architecture. + +## Naming + +- **No abbreviated identifiers.** Use full descriptive names: `notebook` + not `nb`, `destination` not `dest`, `context` not `ctx`, + `type_properties` not `tp`, `linked_service` not `ls`, `keyword` not + `kw`. Single-letter loop counters (`i`, `j`, `k`) are fine; `e` for an + exception is fine; `df` for a Spark DataFrame in *generated user-facing + notebook code* is fine. +- **Module-private names start with `_`.** Anything imported by another + module must not have a leading underscore -- promote it to public (or + move it to a more appropriate module) instead of importing a private + name. +- **Constants in `UPPER_SNAKE`** (e.g. `JDBC_SOURCE_TYPES`); modules and + functions in `lower_snake`; classes in `PascalCase`. + +## Docstrings + +- First line is **third-person present indicative** describing what the + function does: ``"""Converts X to Y."""`` rather than ``"""Convert + X..."""`` or ``"""This function converts..."""``. +- Keep docstrings **brief** -- usually one sentence. Add extra detail + only when behaviour is non-obvious (edge cases, surprising + invariants). Don't restate the type signature or repeat parameter + names; ``Args:`` / ``Returns:`` / ``Raises:`` blocks are optional and + should appear only when the type is genuinely ambiguous. +- Don't write multi-paragraph design rationale in docstrings; that + belongs in commit messages or pull-request descriptions. + +## Comments + +- The bar is: **could a new developer trace through this codebase and + understand the line without the comment?** If yes, delete the comment. +- Comments explain **why** -- a constraint, a non-obvious invariant, a + reference to an external spec, a prior bug being avoided. They never + restate **what** the code does. +- Prefer a self-documenting variable or function name over a comment. +- One-line comments only; multi-line block comments are a smell. + +## Imports + +- **Top-level only.** Inline imports inside function bodies are allowed + only when avoiding a real circular import. When you do need one, add a + one-line comment naming the cycle. +- **No pointless aliases.** Don't write ``import json as _json`` or + ``from x import Y as _Y`` unless an alias is genuinely required to + avoid a name collision. +- **No cross-module private imports.** If module A needs a name from + module B, that name lives at the top level of B without a leading + underscore. Lazy or circular cases should use a public re-export. + +## Mutability + +- **Models are immutable by convention.** All dataclasses use + ``@dataclass(slots=True, kw_only=True)``. Frozen dataclasses are + preferred when the type carries no internal mutation. +- **Collections accumulated across helpers should be returned, not + mutated.** Helpers that "fold up" results (e.g. + :class:`PreparedArtifacts`) take an immutable accumulator and return a + new one rather than mutating shared lists. +- ``TranslationContext`` is **never modified in place** -- always return + a new instance via ``context.with_*(...)``. + +## Function shape + +- Keep functions short. When a function grows past ~40 lines or 3 + levels of nesting, extract a private helper. +- Use **guard clauses** (early ``return``/``continue``) instead of + building deep ``if`` pyramids. +- Prefer comprehensions and generator expressions over explicit + ``for``-append loops when the comprehension fits on one screen line. + +## Duplication + +- Two near-identical 4-6 line blocks inside a single module: extract a + module-private helper. +- The same block across modules: extract to a public helper in the most + semantically appropriate module (often + ``flowx/utils.py`` or + ``flowx/preparer/activity_preparers/helpers.py``). + +## Tests + +- Every translator and preparer module has a corresponding ``tests/unit/`` + test file. When adding a new helper, add a unit test for it. +- Tests run with ``make test``; integration tests run with + ``make integration`` (require ADF fixtures). +- Behavioural changes must be verified by checking that bundle YAML + output for the verdi-test fixtures is byte-identical (or, if it + differs, that the diff matches the intended change). + +## Tooling + +- ``make fmt`` runs ``ruff format``, ``ruff check --fix``, and ``mypy``. +- Line length is 120 characters. +- Python 3.12+ syntax is OK (e.g. ``dict[str, Any]`` over + ``Dict[str, Any]``, ``X | None`` over ``Optional[X]``). diff --git a/src/orchestra/__init__.py b/src/orchestra/__init__.py new file mode 100644 index 0000000..16c449e --- /dev/null +++ b/src/orchestra/__init__.py @@ -0,0 +1,3 @@ +"""Flowx - ADF to Databricks translation plugin for Claude Code.""" + +__version__ = "0.1.0" diff --git a/src/orchestra/bundler/__init__.py b/src/orchestra/bundler/__init__.py new file mode 100644 index 0000000..40b61a6 --- /dev/null +++ b/src/orchestra/bundler/__init__.py @@ -0,0 +1,11 @@ +"""Bundler layer: write PreparedWorkflows as Databricks Declarative Automation Bundles.""" + +from flowx.bundler.dab_writer import write_bundle +from flowx.bundler.notebook_writer import write_notebooks +from flowx.bundler.setup_generator import generate_setup_tasks + +__all__ = [ + "generate_setup_tasks", + "write_bundle", + "write_notebooks", +] diff --git a/src/orchestra/bundler/dab_writer.py b/src/orchestra/bundler/dab_writer.py new file mode 100644 index 0000000..29da7d5 --- /dev/null +++ b/src/orchestra/bundler/dab_writer.py @@ -0,0 +1,1020 @@ +"""Writes a PreparedWorkflow to Databricks Declarative Automation Bundle files.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import yaml + +from flowx.bundler.inner_job_params import normalize_value +from flowx.bundler.notebook_writer import write_notebooks +from flowx.bundler.prereqs_writer import ManualParameter, build_prereqs, render_setup_md +from flowx.bundler.setup_generator import generate_setup_tasks +from flowx.models.dab import DabNotebook +from flowx.models.ir import ( + Activity, + AppendVariableActivity, + CopyActivity, + DeleteActivity, + Dependency, + ExecutePipelineActivity, + FilterActivity, + ForEachActivity, + IfConditionActivity, + LookupActivity, + MotifActivity, + NotebookActivity, + Pipeline, + PlaceholderActivity, + RunJobActivity, + SetVariableActivity, + SparkJarActivity, + SparkPythonActivity, + SwitchActivity, + SwitchCase, + UnsupportedActivity, + WaitActivity, + WebActivity, +) +from flowx.preparer.workflow_preparer import PreparedWorkflow, prepare_workflow +from flowx.utils import normalize_task_key + + +class _BundleYamlDumper(yaml.SafeDumper): + """YAML dumper that leaves keys unquoted and only quotes values when needed.""" + + +# Module-level warnings collector — reset per write_bundle call. +_bundle_warnings: list[str] = [] + +# Cross-bundle ExecutePipeline refs seen while translating: variable_name → +# target pipeline name. Reset per write_bundle call and surfaced via the +# bundle's ``variables`` block + SETUP.md. +_cross_bundle_variables: dict[str, str] = {} + +_WIDGET_REFERENCE = re.compile(r"""dbutils\.widgets\.get\(\s*["']([^"']+)["']\s*\)""") + + +def write_bundle( + workflow: PreparedWorkflow, + output_dir: Path, + catalog: str = "main", + schema: str = "default", + bundle_name: str | None = None, +) -> list[Path]: + """Writes all DAB files to output_dir. + + Args: + workflow: The PreparedWorkflow to serialize. + output_dir: Root directory for the bundle output. + catalog: Default target catalog name. + schema: Default target schema name. + bundle_name: Optional bundle name (defaults to workflow name). + + Returns: + List of absolute paths to all created files. + """ + # Reset module-level accumulators so successive ``write_bundle`` calls + # (CLI loops, library users, integration tests) don't carry warnings or + # cross-bundle variables from one bundle into the next. + _bundle_warnings.clear() + _cross_bundle_variables.clear() + + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + created_files: list[Path] = [] + resource_key = normalize_task_key(workflow.name) + effective_name = bundle_name or resource_key + + # Bind clusters across the parent workflow and any inner workflows up + # front so we can decide whether the bundle needs cluster-related + # tunables in ``databricks.yml`` at all. Binding is idempotent, so the + # subsequent ``_build_job_resource`` calls re-checking the same tasks is + # harmless. + _bind_cluster_to_notebook_tasks(workflow.tasks) + for inner in workflow.inner_workflows: + _bind_cluster_to_notebook_tasks(inner.tasks) + bundle_uses_classic_cluster = _any_task_uses_classic_cluster(workflow.tasks) or any( + _any_task_uses_classic_cluster(inner.tasks) for inner in workflow.inner_workflows + ) + + # 1. Write databricks.yml. When at least one task runs on classic + # compute, defaults for spark_version / node_type_id come from the + # ADF linked service configs on the tasks so the emitted cluster + # matches the source-of-truth runtime. When every task is + # serverless, those variables are omitted entirely. + databricks_yml_path = output_dir / "databricks.yml" + inferred_spark_version, inferred_node_type_id = _infer_bundle_cluster_defaults(workflow) + databricks_yml_dict = _build_databricks_yml( + effective_name, + catalog, + schema, + spark_version=inferred_spark_version, + node_type_id=inferred_node_type_id, + include_cluster_variables=bundle_uses_classic_cluster, + ) + databricks_yml_path.write_text( + yaml.dump( + databricks_yml_dict, + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + Dumper=_BundleYamlDumper, + ), + encoding="utf-8", + ) + created_files.append(databricks_yml_path.resolve()) + + # 2. Write job resource YAML. Strip broken base_parameters from + # existing-notebook tasks before serialising — these are surfaced + # in SETUP.md (§Existing-notebook parameter handling) further down + # and shouldn't ship in the YAML as malformed widget values. + manual_parameters: list[ManualParameter] = _extract_manual_parameters_from_existing_notebook_tasks(workflow.tasks) + for inner in workflow.inner_workflows: + manual_parameters.extend(_extract_manual_parameters_from_existing_notebook_tasks(inner.tasks)) + + resources_dir = output_dir / "resources" + resources_dir.mkdir(parents=True, exist_ok=True) + job_yml_path = resources_dir / f"{resource_key}.yml" + job_resource = _build_job_resource(workflow, resource_key) + job_yml_path.write_text( + yaml.dump( + job_resource, default_flow_style=False, sort_keys=False, allow_unicode=True, Dumper=_BundleYamlDumper + ), + encoding="utf-8", + ) + created_files.append(job_yml_path.resolve()) + + # Write inner workflows as additional resource files. Inner tasks reuse + # notebooks that live in the parent workflow's notebooks list, so pass + # those in so the inner job's widget auto-augmentation can see them. + for inner in workflow.inner_workflows: + inner_key = normalize_task_key(inner.name) + inner_yml_path = resources_dir / f"{inner_key}.yml" + inner_resource = _build_job_resource(inner, inner_key, extra_notebooks_for_augment=workflow.notebooks) + inner_yml_path.write_text( + yaml.dump( + inner_resource, + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + Dumper=_BundleYamlDumper, + ), + encoding="utf-8", + ) + created_files.append(inner_yml_path.resolve()) + + # 3. Write generated notebooks + src_dir = output_dir / "src" + if workflow.notebooks: + created_files.extend(write_notebooks(workflow.notebooks, src_dir)) + + # 4. Generate and write setup notebooks (create-scope, create-volume, etc.). + # These are the *executable* provisioning artifacts; SETUP.md (below) + # is the human-readable companion. + setup_notebooks: list[DabNotebook] = generate_setup_tasks( + secrets=workflow.secrets, + setup_tasks=workflow.setup_tasks, + catalog=catalog, + schema=schema, + ) + if setup_notebooks: + created_files.extend(write_notebooks(setup_notebooks, src_dir)) + + # Collect notebooks from inner workflows + for inner in workflow.inner_workflows: + if inner.notebooks: + created_files.extend(write_notebooks(inner.notebooks, src_dir)) + inner_setup = generate_setup_tasks( + secrets=inner.secrets, + setup_tasks=inner.setup_tasks, + catalog=catalog, + schema=schema, + ) + if inner_setup: + created_files.extend(write_notebooks(inner_setup, src_dir)) + + # 5. Build SETUP.md — a root-level, human-readable summary of every + # external step the user must take before ``bundle run``. This is + # additive to the setup/ notebooks above; the setup notebooks are + # the executable path, SETUP.md is the checklist. + all_notebooks = list(workflow.notebooks) + for inner in workflow.inner_workflows: + all_notebooks.extend(inner.notebooks) + all_tasks = list(workflow.tasks) + for inner in workflow.inner_workflows: + all_tasks.extend(inner.tasks) + known_bundle_jobs = {resource_key} | {normalize_task_key(inner.name) for inner in workflow.inner_workflows} + # ``manual_parameters`` was collected above (before YAML emission) so + # the broken values are also stripped from the on-disk YAML. + prereqs = build_prereqs( + notebooks=all_notebooks, + tasks=all_tasks, + known_bundle_jobs=known_bundle_jobs, + cross_bundle_variables=dict(_cross_bundle_variables), + manual_parameters=manual_parameters, + ) + setup_path = output_dir / "SETUP.md" + setup_path.write_text(render_setup_md(prereqs, bundle_name=effective_name), encoding="utf-8") + created_files.append(setup_path.resolve()) + + # 5. Write warnings file if any warnings were collected + if _bundle_warnings: + warnings_path = output_dir / "WARNINGS.md" + lines = [ + "# Translation Warnings\n", + "", + "The following items require manual review or modification:\n", + "", + ] + lines.extend(_bundle_warnings) + lines.append("") + warnings_path.write_text("\n".join(lines), encoding="utf-8") + created_files.append(warnings_path.resolve()) + + return created_files + + +def main() -> None: + """CLI entry point for DAB bundle generation.""" + parser = argparse.ArgumentParser( + description="Generate a Databricks Declarative Automation Bundle from a translation report.", + ) + parser.add_argument( + "--report", + type=Path, + required=True, + help="Path to the translation report or pipeline IR JSON produced by the translate phase.", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("./orchestra_output/bundle"), + help="Output directory for the DAB bundle (default: ./orchestra_output/bundle).", + ) + parser.add_argument( + "--catalog", + type=str, + default="main", + help="Target Unity Catalog name (default: main).", + ) + parser.add_argument( + "--schema", + type=str, + default="default", + help="Target schema name (default: default).", + ) + parser.add_argument( + "--bundle-name", + type=str, + default=None, + help="Override the bundle name (defaults to the workflow name).", + ) + args = parser.parse_args() + + if not args.report.exists(): + print(f"Error: Report file not found: {args.report}", file=sys.stderr) + sys.exit(1) + + print(f"Loading translation report: {args.report}") + workflows = _load_report(args.report) + + if not workflows: + print("No translated pipelines found in the report.", file=sys.stderr) + sys.exit(1) + + all_created: list[Path] = [] + for index, workflow in enumerate(workflows): + if len(workflows) > 1: + workflow_dir = args.output_dir / normalize_task_key(workflow.name) + else: + workflow_dir = args.output_dir + + effective_bundle_name = args.bundle_name if len(workflows) == 1 else None + created = write_bundle( + workflow=workflow, + output_dir=workflow_dir, + catalog=args.catalog, + schema=args.schema, + bundle_name=effective_bundle_name, + ) + all_created.extend(created) + print(f" [{index + 1}/{len(workflows)}] {workflow.name}: {len(created)} files") + + print(f"\nBundle generation complete: {len(all_created)} files written to {args.output_dir}") + print("\nNext steps:") + print(" 1. Review the generated notebooks in src/") + print(" 2. Run the setup notebooks to create secrets and volumes") + print(" 3. Validate the bundle: databricks bundle validate") + print(" 4. Deploy: databricks bundle deploy -t dev") + + +def _warn(task_key: str, message: str) -> None: + """Record a translation warning for the current bundle.""" + _bundle_warnings.append(f"- **{task_key}**: {message}") + + +_DEFAULT_SPARK_VERSION = "15.4.x-scala2.12" +_DEFAULT_NODE_TYPE_ID = "Standard_DS3_v2" + + +def _infer_bundle_cluster_defaults(workflow: PreparedWorkflow) -> tuple[str, str]: + """Derive ``spark_version`` and ``node_type_id`` defaults from task clusters. + + Args: + workflow: The prepared workflow being written. + + Returns: + ``(spark_version, node_type_id)`` strings. + """ + from collections import Counter + + spark_versions = [hint["spark_version"] for hint in workflow.cluster_hints if hint.get("spark_version")] + node_types = [hint["node_type_id"] for hint in workflow.cluster_hints if hint.get("node_type_id")] + + spark_version = Counter(spark_versions).most_common(1)[0][0] if spark_versions else _DEFAULT_SPARK_VERSION + node_type_id = Counter(node_types).most_common(1)[0][0] if node_types else _DEFAULT_NODE_TYPE_ID + return spark_version, node_type_id + + +def _build_databricks_yml( + bundle_name: str, + catalog: str, + schema: str, + *, + spark_version: str = _DEFAULT_SPARK_VERSION, + node_type_id: str = _DEFAULT_NODE_TYPE_ID, + include_cluster_variables: bool = True, +) -> dict[str, Any]: + """Builds the root ``databricks.yml`` configuration as a dict. + + Args: + bundle_name: Name for the bundle. + catalog: Default target catalog. + schema: Default target schema. + spark_version: DBR version for the default job_cluster. Callers + typically derive this from :func:`_infer_bundle_cluster_defaults`. + node_type_id: Instance type for the default job_cluster. + include_cluster_variables: When True, declares ``spark_version`` and + ``node_type_id`` variables for the default job_cluster. Set to + False when no task in the bundle uses classic compute (every + generated notebook runs on serverless), so the bundle stays + free of unused tunables. + + Returns: + Dict ready for YAML serialization. + """ + variables: dict[str, Any] = { + "catalog": { + "description": "Target catalog", + "default": catalog, + }, + "schema": { + "description": "Target schema", + "default": schema, + }, + } + if include_cluster_variables: + variables["node_type_id"] = { + "description": ( + "Instance type for the default job_cluster — override per cloud " + "(e.g. i3.xlarge on AWS, n1-standard-4 on GCP)." + ), + "default": node_type_id, + } + variables["spark_version"] = { + "description": "Databricks Runtime for the default job_cluster.", + "default": spark_version, + } + # Declare a variable for each cross-bundle ExecutePipeline reference so + # `${var.X_job_id}` resolves and `bundle validate` passes. Users fill in + # the numeric job ID per SETUP.md. + for variable_name, target_pipeline in sorted(_cross_bundle_variables.items()): + variables[variable_name] = { + "description": ( + f"Numeric job ID for pipeline '{target_pipeline}' (defined in a sibling bundle). " + f'Populate via `databricks bundle deploy --var "{variable_name}="` or set ' + "the default here." + ), + } + return { + "bundle": { + "name": bundle_name, + }, + "variables": variables, + "include": [ + "resources/*.yml", + ], + "targets": { + "dev": { + "mode": "development", + }, + "staging": { + "mode": "production", + }, + "prod": { + "mode": "production", + }, + }, + } + + +_DEFAULT_JOB_CLUSTER_KEY = "default_cluster" + + +def _build_default_job_clusters() -> list[dict[str, Any]]: + """Return a job_clusters stanza that binds notebook tasks to a real cluster.""" + return [ + { + "job_cluster_key": _DEFAULT_JOB_CLUSTER_KEY, + "new_cluster": { + "spark_version": "${var.spark_version}", + "node_type_id": "${var.node_type_id}", + "num_workers": 1, + "data_security_mode": "SINGLE_USER", + }, + } + ] + + +# Patterns that signal a base_parameter value couldn't be evaluated cleanly. +# When any task references an *existing* notebook (absolute workspace path), +# flowx can't inject the runtime computation, so these end up as manual +# work for the user. +_HYBRID_ADF_FN_RE = re.compile(r"@[a-zA-Z][a-zA-Z0-9]*\(") +_PYTHON_CODE_HINTS = ("dbutils.widgets.get(", "datetime.now(", "datetime.fromisoformat(") + + +def _value_needs_manual_handling(value: Any) -> bool: + """Return True when *value* is too dynamic for DAB to substitute at deploy time.""" + if not isinstance(value, str): + return False + if _HYBRID_ADF_FN_RE.search(value): + return True + return any(hint in value for hint in _PYTHON_CODE_HINTS) + + +def _extract_manual_parameters_from_existing_notebook_tasks( + tasks: list[dict[str, Any]], +) -> list[ManualParameter]: + """Finds base_parameters flowx couldn't evaluate for existing-notebook tasks.""" + manual_parameters: list[ManualParameter] = [] + for task in _iter_tasks_recursively(tasks): + notebook_task = task.get("notebook_task") or {} + notebook_path = notebook_task.get("notebook_path", "") + base_params = notebook_task.get("base_parameters") + # Bundle-relative paths (``../src/...``) can have their notebook + # bodies patched to inline the runtime computation; absolute paths + # belong to the user's existing notebooks and must be surfaced. + if not notebook_path.startswith("/") or not isinstance(base_params, dict): + continue + keys_to_drop: list[str] = [] + for key, value in base_params.items(): + if not _value_needs_manual_handling(value): + continue + manual_parameters.append( + ManualParameter( + task_key=task.get("task_key", ""), + widget_name=key, + notebook_path=notebook_path, + raw_expression=str(value), + ) + ) + keys_to_drop.append(key) + for key in keys_to_drop: + del base_params[key] + if not base_params: + notebook_task.pop("base_parameters", None) + return manual_parameters + + +def _iter_tasks_recursively(tasks: list[dict[str, Any]]) -> Iterator[dict[str, Any]]: + """Yields every task in *tasks*, descending into ``for_each_task.task``.""" + for task in tasks: + yield task + for_each = task.get("for_each_task") or {} + inner = for_each.get("task") + if isinstance(inner, dict): + yield from _iter_tasks_recursively([inner]) + + +_CLUSTER_BINDING_KEYS = ("existing_cluster_id", "new_cluster", "job_cluster_key") + + +def _any_task_uses_classic_cluster(tasks: list[dict[str, Any]]) -> bool: + """Return True if any task (recursively) is bound to a job_cluster_key.""" + return any(task.get("job_cluster_key") for task in _iter_tasks_recursively(tasks)) + + +def _bind_cluster_to_notebook_tasks(tasks: list[dict[str, Any]]) -> None: + """Attaches the default job_cluster_key to existing-notebook tasks.""" + for task in _iter_tasks_recursively(tasks): + notebook_task = task.get("notebook_task") + if notebook_task is None: + continue + notebook_path = notebook_task.get("notebook_path", "") + if notebook_path.startswith("../src/"): + continue + if any(key in task for key in _CLUSTER_BINDING_KEYS): + continue + task["job_cluster_key"] = _DEFAULT_JOB_CLUSTER_KEY + + +def _rewrite_post_branch_dependencies(tasks: list[dict[str, Any]]) -> None: + """Rewrites ``depends_on`` edges that target a condition_task to target its branches. + + Args: + tasks: Top-level task list, mutated in place. + """ + condition_keys = {task["task_key"] for task in tasks if "condition_task" in task} + if not condition_keys: + return + + direct_outcome_children: dict[str, list[str]] = {key: [] for key in condition_keys} + for task in tasks: + for dep in task.get("depends_on") or []: + if dep.get("task_key") in condition_keys and dep.get("outcome") in ("true", "false"): + direct_outcome_children[dep["task_key"]].append(task["task_key"]) + + def expand_terminals(condition_key: str, seen: set[str]) -> list[str]: + """Return branch-terminal task keys for a condition, transitively.""" + terminals: list[str] = [] + for child_key in direct_outcome_children.get(condition_key, []): + if child_key in seen: + continue + seen.add(child_key) + if child_key in condition_keys: + terminals.extend(expand_terminals(child_key, seen)) + else: + terminals.append(child_key) + return terminals + + for task in tasks: + depends_on = task.get("depends_on") or [] + if not depends_on: + continue + rewritten: list[dict[str, Any]] = [] + touched_condition = False + for dep in depends_on: + dep_task_key = dep.get("task_key") + if dep_task_key in condition_keys and "outcome" not in dep: + replacement_keys = expand_terminals(dep_task_key, set()) + if replacement_keys: + rewritten.extend({"task_key": branch_key} for branch_key in replacement_keys) + touched_condition = True + else: + rewritten.append(dep) + else: + rewritten.append(dep) + if touched_condition: + # Drop duplicates — a diamond-shaped join could hit the same + # terminal via more than one branch. + seen_keys: set[str] = set() + deduped: list[dict[str, Any]] = [] + for dep in rewritten: + key = dep.get("task_key", "") + if key in seen_keys: + continue + seen_keys.add(key) + deduped.append(dep) + task["depends_on"] = deduped + task.setdefault("run_if", "AT_LEAST_ONE_SUCCESS") + + +_TASK_VALUE_REF = re.compile(r"\{\{tasks\.([^.]+)\.values\.[^}]+\}\}") + + +def _strip_dangling_task_value_refs(tasks: list[dict[str, Any]], all_task_keys: set[str]) -> None: + """Replaces ``{{tasks.X.values.Y}}`` refs whose ``X`` is not in the bundle. + + Args: + tasks: Top-level tasks for one job (mutated in place). + all_task_keys: Task keys that do exist in this job (including those + inside ``for_each_task.task`` bodies). + """ + + def visit(task: dict[str, Any]) -> None: + notebook_task = task.get("notebook_task") or {} + base_parameters = notebook_task.get("base_parameters") or {} + for widget_name, value in list(base_parameters.items()): + if not isinstance(value, str): + continue + match = _TASK_VALUE_REF.search(value) + if match and match.group(1) not in all_task_keys: + base_parameters[widget_name] = "" + for_each = task.get("for_each_task") + if for_each and isinstance(for_each.get("task"), dict): + visit(for_each["task"]) + + for task in tasks: + visit(task) + + +def _collect_all_task_keys(tasks: list[dict[str, Any]]) -> set[str]: + """Collects every task_key reachable from the job's top-level task list.""" + keys: set[str] = set() + for task in tasks: + keys.add(task.get("task_key", "")) + for_each = task.get("for_each_task") + if for_each and isinstance(for_each.get("task"), dict): + keys.update(_collect_all_task_keys([for_each["task"]])) + keys.discard("") + return keys + + +def _augment_base_parameters(tasks: list[dict[str, Any]], notebooks: list[DabNotebook]) -> None: + """Ensure every widget a notebook reads is declared in its base_parameters. + + Args: + tasks: Top-level task dicts (mutated in place). + notebooks: Generated notebooks to scan. + """ + notebook_by_relpath = {notebook.relative_path: notebook for notebook in notebooks} + + def visit(task: dict[str, Any]) -> None: + notebook_task = task.get("notebook_task") + if notebook_task: + notebook_path = notebook_task.get("notebook_path", "") + relative = notebook_path[len("../src/") :] if notebook_path.startswith("../src/") else "" + notebook = notebook_by_relpath.get(relative) + if notebook: + widgets = set(_WIDGET_REFERENCE.findall(notebook.content)) + base_parameters = notebook_task.setdefault("base_parameters", {}) + for widget_name in sorted(widgets): + base_parameters.setdefault(widget_name, "") + for_each = task.get("for_each_task") + if for_each and isinstance(for_each.get("task"), dict): + visit(for_each["task"]) + + for task in tasks: + visit(task) + + +def _build_job_resource( + workflow: PreparedWorkflow, + resource_key: str, + *, + attach_clusters: bool = True, + extra_notebooks_for_augment: list[DabNotebook] | None = None, +) -> dict[str, Any]: + """Builds a job resource dict for a single workflow. + + Args: + workflow: The prepared workflow to serialize. + resource_key: The sanitised resource key for this job. + attach_clusters: When ``True`` (default) emits a ``job_clusters`` block + and binds every notebook task to it. Set to ``False`` for inner + jobs that are invoked via ``run_job_task`` from another bundle + job — they inherit compute from the caller. + + Returns: + Dict ready for YAML serialization. + """ + _rewrite_post_branch_dependencies(workflow.tasks) + # For inner jobs (invoked via run_job_task), notebooks live in the parent + # workflow's notebooks list — pass them in so widget auto-augment can + # still find the bound notebook and populate base_parameters. + augment_scope = list(workflow.notebooks) + list(extra_notebooks_for_augment or []) + _augment_base_parameters(workflow.tasks, augment_scope) + # Task values don't cross ``run_job_task`` boundaries; any such + # reference in this job resolves to an empty string at runtime. Emit + # the empty string now so SETUP.md §4 flags it. + _strip_dangling_task_value_refs(workflow.tasks, _collect_all_task_keys(workflow.tasks)) + + job_def: dict[str, Any] = { + "name": workflow.name, + "tasks": workflow.tasks, + } + + if attach_clusters: + _bind_cluster_to_notebook_tasks(workflow.tasks) + # Only emit the ``job_clusters`` block when at least one task is + # actually bound to it. When every task runs on serverless (the + # generated-notebook case), the job stays cluster-free and inherits + # the workspace's serverless defaults. + if _any_task_uses_classic_cluster(workflow.tasks): + job_def["job_clusters"] = _build_default_job_clusters() + + if workflow.parameters: + job_def["parameters"] = workflow.parameters + + return { + "resources": { + "jobs": { + resource_key: job_def, + }, + }, + } + + +def _normalize_base_parameters( + params: dict[str, Any], + *, + task_key: str = "", +) -> dict[str, str]: + """Normalise raw ADF expression dicts in base_parameters to resolved strings. + + Args: + params: Raw base_parameters dict from the IR. + task_key: Task key for warning attribution. + + Returns: + Dict with all values resolved to strings. + """ + resolved: dict[str, str] = {} + for key, value in params.items(): + normalized = normalize_value(value) + if "dbutils.widgets.get" in normalized or "dbutils.jobs.taskValues" in normalized: + _warn( + task_key, + f"Parameter `{key}` contains a computed expression that cannot be " + f"expressed as a DAB dynamic value reference. The task's notebook " + f"or entry point must handle this parameter at runtime. " + f"Value: `{normalized}`", + ) + resolved[key] = normalized + return resolved + + +def _load_report(report_path: Path) -> list[PreparedWorkflow]: + """Loads a translation report and reconstruct PreparedWorkflow objects. + + Args: + report_path: Path to the translation report JSON file. + + Returns: + List of PreparedWorkflow objects, one per pipeline. + """ + with open(report_path, encoding="utf-8") as report_file: + report = json.load(report_file) + + workflows: list[PreparedWorkflow] = [] + + if "tasks" in report and "name" in report: + workflow = _pipeline_dict_to_workflow(report) + workflows.append(workflow) + return workflows + + if "translations" in report: + # Aggregated translation_report.json format: ``translations`` is a + # flat list of ``{pipeline, ir, status, ...}`` entries. Group by + # pipeline name and route each group through the same + # ``_pipeline_dict_to_workflow`` machinery as the single-pipeline IR + # format, so secret discovery / setup tasks / control-flow handling + # all match. + pipelines: dict[str, list[dict]] = {} + for translation in report.get("translations", []): + pipeline_name = translation.get("pipeline", "unknown") + if translation.get("status") != "translated": + continue + ir = translation.get("ir") or {} + if not ir: + continue + pipelines.setdefault(pipeline_name, []).append(ir) + + for pipeline_name, task_irs in pipelines.items(): + workflow = _pipeline_dict_to_workflow({"name": pipeline_name, "tasks": task_irs}) + workflows.append(workflow) + return workflows + + # Empty or unrecognised report shape — nothing to do. + return workflows + + +def _pipeline_dict_to_workflow(pipeline_dict: dict[str, Any]) -> PreparedWorkflow: + """Converts a serialised pipeline IR dict to a PreparedWorkflow. + + Rehydrates every task into a typed Activity via :func:`_reconstruct_ir` + and routes through :func:`prepare_workflow` so the JSON-reload path + shares one code path with the in-process translator. This guarantees + feature parity for secrets, setup tasks, manual parameters, + expression resolution, and motif handling without duplicating the + per-activity preparer logic. + """ + activities = [_reconstruct_ir(task_ir) for task_ir in pipeline_dict.get("tasks", [])] + + parameters: list[dict[str, Any]] = [] + for param in pipeline_dict.get("parameters") or []: + entry: dict[str, Any] = {"name": param["name"]} + if "default" in param and param["default"] is not None: + entry["default"] = normalize_value(str(param["default"])) + parameters.append(entry) + + pipeline = Pipeline( + name=pipeline_dict.get("name", "unknown"), + tasks=activities, + parameters=parameters or None, + ) + + workflow = prepare_workflow(pipeline) + if parameters: + workflow.parameters.extend(parameters) + return workflow + + +def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: + """Rehydrates a typed Activity from its serialised IR dict. + + Recurses into control-flow inner activities (ForEach, IfCondition, + Switch). Unknown ``type`` strings fall back to PlaceholderActivity + so the rest of the pipeline can still be prepared. + """ + task_type = task_ir.get("type", "") + base = _common_activity_kwargs(task_ir) + + if task_type == "LookupActivity": + return LookupActivity( + **base, + source_type=task_ir.get("source_type"), + source_properties=task_ir.get("source_properties"), + first_row_only=task_ir.get("first_row_only", True), + source_query=task_ir.get("source_query"), + ) + if task_type == "CopyActivity": + return CopyActivity( + **base, + source_type=task_ir.get("source_type"), + sink_type=task_ir.get("sink_type"), + source_properties=task_ir.get("source_properties"), + sink_properties=task_ir.get("sink_properties"), + sink_dataset_type=task_ir.get("sink_dataset_type"), + sink_format=task_ir.get("sink_format"), + sink_resolved_path=task_ir.get("sink_resolved_path"), + column_mapping=task_ir.get("column_mapping"), + ) + if task_type == "WebActivity": + return WebActivity( + **base, + url=task_ir.get("url", ""), + method=task_ir.get("method", "GET"), + body=task_ir.get("body"), + headers=task_ir.get("headers"), + authentication=task_ir.get("authentication"), + ) + if task_type == "SetVariableActivity": + return SetVariableActivity( + **base, + variable_name=task_ir.get("variable_name", ""), + variable_value=task_ir.get("variable_value", ""), + value_kind=task_ir.get("value_kind", "literal"), + notebook_code=task_ir.get("notebook_code"), + notebook_imports=task_ir.get("notebook_imports", []), + ) + if task_type == "WaitActivity": + return WaitActivity( + **base, + wait_time_seconds=task_ir.get("wait_time_seconds", 0), + ) + if task_type == "DeleteActivity": + return DeleteActivity( + **base, + dataset_name=task_ir.get("dataset_name", ""), + folder_path=task_ir.get("folder_path"), + recursive=task_ir.get("recursive", True), + ) + if task_type == "FilterActivity": + return FilterActivity( + **base, + items_expression=task_ir.get("items_expression", ""), + condition_expression=task_ir.get("condition_expression", ""), + condition_code=task_ir.get("condition_code"), + condition_imports=list(task_ir.get("condition_imports") or []), + ) + if task_type == "AppendVariableActivity": + return AppendVariableActivity( + **base, + variable_name=task_ir.get("variable_name", ""), + append_value=task_ir.get("append_value", ""), + value_kind=task_ir.get("value_kind", "literal"), + notebook_code=task_ir.get("notebook_code"), + notebook_imports=task_ir.get("notebook_imports", []), + ) + if task_type == "NotebookActivity": + return NotebookActivity( + **base, + notebook_path=task_ir.get("notebook_path", ""), + base_parameters=task_ir.get("base_parameters"), + ) + if task_type == "SparkJarActivity": + return SparkJarActivity( + **base, + main_class_name=task_ir.get("main_class_name", ""), + parameters=task_ir.get("parameters"), + libraries=task_ir.get("libraries"), + ) + if task_type == "SparkPythonActivity": + return SparkPythonActivity( + **base, + python_file=task_ir.get("python_file", ""), + parameters=task_ir.get("parameters"), + ) + if task_type == "ExecutePipelineActivity": + return ExecutePipelineActivity( + **base, + pipeline_name=task_ir.get("pipeline_name", ""), + parameters=task_ir.get("parameters"), + wait_on_completion=task_ir.get("wait_on_completion", True), + ) + if task_type == "RunJobActivity": + return RunJobActivity( + **base, + job_name=task_ir.get("job_name", ""), + existing_job_id=task_ir.get("existing_job_id"), + job_parameters=task_ir.get("job_parameters") or task_ir.get("parameters"), + ) + if task_type == "ForEachActivity": + return ForEachActivity( + **base, + items_expression=task_ir.get("items_expression", ""), + inner_activities=[_reconstruct_ir(child) for child in task_ir.get("inner_activities") or []], + concurrency=task_ir.get("concurrency"), + ) + if task_type == "IfConditionActivity": + return IfConditionActivity( + **base, + op=task_ir.get("op", "EQUAL_TO"), + left=task_ir.get("left", ""), + right=task_ir.get("right", ""), + if_true_activities=[_reconstruct_ir(child) for child in task_ir.get("if_true_activities") or []], + if_false_activities=[_reconstruct_ir(child) for child in task_ir.get("if_false_activities") or []], + ) + if task_type == "SwitchActivity": + return SwitchActivity( + **base, + on_expression=task_ir.get("on_expression", ""), + cases=[ + SwitchCase( + value=case.get("value", ""), + activities=[_reconstruct_ir(child) for child in case.get("activities") or []], + ) + for case in task_ir.get("cases") or [] + ], + default_activities=[_reconstruct_ir(child) for child in task_ir.get("default_activities") or []], + ) + if task_type == "MotifActivity": + return MotifActivity( + **base, + motif_id=task_ir.get("motif_id", "unknown"), + display_name=task_ir.get("display_name", base["name"]), + databricks_replacement=task_ir.get("databricks_replacement", "notebook"), + matched_activity_names=list(task_ir.get("matched_activity_names", [])), + source_type_hint=task_ir.get("source_type_hint"), + confidence_notes=list(task_ir.get("confidence_notes", [])), + original_activities=[], + notebook_template=task_ir.get("notebook_template"), + motif_config=task_ir.get("motif_config") or {}, + ) + if task_type == "UnsupportedActivity": + return UnsupportedActivity( + **base, + original_type=task_ir.get("original_type", "unknown"), + reason=task_ir.get("reason"), + ) + if task_type == "PlaceholderActivity": + return PlaceholderActivity( + **base, + original_type=task_ir.get("original_type", task_type), + notebook_path=task_ir.get("notebook_path", "/UNSUPPORTED_ADF_ACTIVITY"), + comment=task_ir.get("comment"), + ) + return PlaceholderActivity( + **base, + original_type=task_type or "unknown", + comment=f"Unknown activity type {task_type!r}; produced as placeholder during JSON-reload.", + ) + + +def _common_activity_kwargs(task_ir: dict[str, Any]) -> dict[str, Any]: + """Extracts the base Activity fields shared by every IR class.""" + task_key = task_ir.get("task_key", "") + return { + "name": task_ir.get("name", task_key), + "task_key": task_key, + "description": task_ir.get("description"), + "timeout_seconds": task_ir.get("timeout_seconds"), + "max_retries": task_ir.get("max_retries"), + "min_retry_interval_millis": task_ir.get("min_retry_interval_millis"), + "depends_on": _reconstruct_dependencies(task_ir.get("depends_on")), + "cluster": task_ir.get("cluster"), + "required_parameters": dict(task_ir.get("required_parameters") or {}), + } + + +def _reconstruct_dependencies(raw: list[dict[str, Any]] | None) -> list[Dependency] | None: + if not raw: + return None + return [Dependency(task_key=dep.get("task_key", ""), outcome=dep.get("outcome")) for dep in raw] + + +# Notebook content generators + + +if __name__ == "__main__": + main() diff --git a/src/orchestra/bundler/inner_job_params.py b/src/orchestra/bundler/inner_job_params.py new file mode 100644 index 0000000..20e4e58 --- /dev/null +++ b/src/orchestra/bundler/inner_job_params.py @@ -0,0 +1,363 @@ +"""Collects and normalize parameters for ForEach inner jobs.""" + +from __future__ import annotations + +import re +from typing import Any + +# The leading @ is optional -- ADF only puts it on the outermost expression; +# inner references like concat('x', pipeline().parameters.Y) are bare. +_ITEM_BARE_RE = re.compile(r"@?item\(\s*\)", re.IGNORECASE) +_ITEM_FIELD_RE = re.compile(r"@?item\(\s*\)\.(\w+)", re.IGNORECASE) +_PIPELINE_PARAM_RE = re.compile(r"@?pipeline\(\s*\)\.parameters\.(\w+)", re.IGNORECASE) +_PIPELINE_RUNID_RE = re.compile(r"@?pipeline\(\s*\)\.RunId", re.IGNORECASE) +_PIPELINE_TRIGGER_TIME_RE = re.compile(r"@?pipeline\(\s*\)\.TriggerTime", re.IGNORECASE) +_PIPELINE_NAME_RE = re.compile(r"@?pipeline\(\s*\)\.Pipeline", re.IGNORECASE) +_PIPELINE_GROUPID_RE = re.compile(r"@?pipeline\(\s*\)\.GroupId", re.IGNORECASE) +_ACTIVITY_OUTPUT_RE = re.compile( + r"@?activity\(\s*'([^']+)'\s*\)\.output(?:\.firstRow)?\.(\w+)", + re.IGNORECASE, +) +_VARIABLES_RE = re.compile(r"@?variables\(\s*'([^']+)'\s*\)", re.IGNORECASE) +_UTCNOW_RE = re.compile(r"@?utcNow\(\s*\)", re.IGNORECASE) +_DAB_JOB_PARAM_RE = re.compile(r"\{\{job\.parameters\.(\w+)\}\}") +_DAB_INPUT_FIELD_RE = re.compile(r"\{\{input\.(\w+)\}\}") + +_CONCAT_RE = re.compile(r"@?concat\((.+)\)$", re.IGNORECASE | re.DOTALL) + +# ADF type-conversion wrappers: @string(expr), @int(expr), @bool(expr), etc. +# These are no-ops in DAB string parameter context -- strip them. +_TYPE_CAST_RE = re.compile( + r"^@?(string|int|float|bool|decimal|json|xml|base64|binary|uriComponent|" + r"ticks|dataUri|dataUriToBinary|dataUriToString|uriComponentToString)\((.+)\)$", + re.IGNORECASE | re.DOTALL, +) + + +def collect_inner_job_params( + tasks: list[dict[str, Any]], + *, + raw_ir_tasks: list[dict[str, Any]] | None = None, +) -> tuple[list[dict[str, Any]], dict[str, str]]: + """Scans task dicts for parameter references and return declarations + pass-through map. + + Args: + tasks: The inner job's DAB task dicts (may be nested via condition_task). + raw_ir_tasks: Optional raw IR dicts (before DAB conversion) to scan + for references in fields that are consumed during conversion + (e.g. WebActivity ``url``, ``body``). + + Returns: + Tuple of: + - ``parameters``: list of ``{"name": ..., "default": ...}`` dicts for + the inner job definition. + - ``job_parameters``: dict mapping param name -> parent expression, + suitable for the ``run_job_task.job_parameters`` block. ``item`` + always maps to ``"{{input}}"``, pipeline/variable params map to + ``"{{job.parameters.}}"``. + """ + param_names: set[str] = set() + item_field_names: set[str] = set() + + _scan_tasks(tasks, param_names, item_field_names=item_field_names) + + if raw_ir_tasks: + _scan_ir_tasks(raw_ir_tasks, param_names, item_field_names=item_field_names) + + parameters: list[dict[str, Any]] = [] + for name in sorted(param_names): + param: dict[str, Any] = {"name": name} + if name != "item": + param["default"] = "" + parameters.append(param) + + # "item" (bare @item()) maps to {{input}} (the full iteration value); + # item field names (@item().field) map to {{input.}}; + # pipeline params / variables map to {{job.parameters.}}. + job_parameters: dict[str, str] = {} + for name in sorted(param_names): + if name == "item": + job_parameters[name] = "{{input}}" + elif name in item_field_names: + job_parameters[name] = "{{input." + name + "}}" + else: + job_parameters[name] = "{{job.parameters." + name + "}}" + + return parameters, job_parameters + + +def normalize_inner_task_params(tasks: list[dict[str, Any]]) -> None: + """Normalise ADF expressions in task dicts for an inner job context. + + Args: + tasks: The inner job's task dicts. + """ + for task in tasks: + notebook_task = task.get("notebook_task") + if notebook_task and "base_parameters" in notebook_task: + notebook_task["base_parameters"] = { + key: normalize_value(value) for key, value in notebook_task["base_parameters"].items() + } + + condition_task = task.get("condition_task") + if condition_task: + if "left" in condition_task: + condition_task["left"] = normalize_value(condition_task["left"]) + if "right" in condition_task: + condition_task["right"] = normalize_value(condition_task["right"]) + normalize_inner_task_params(condition_task.get("if_true", [])) + normalize_inner_task_params(condition_task.get("if_false", [])) + + for_each_task = task.get("for_each_task", {}) + body = for_each_task.get("task") + if body: + normalize_inner_task_params([body]) + + +def _scan_tasks( + tasks: list[dict[str, Any]], + param_names: set[str], + *, + item_field_names: set[str] | None = None, +) -> None: + """Recursively scan task dicts for ADF parameter references. + + Args: + tasks: List of task dicts to scan. + param_names: Accumulator set of discovered parameter names. + item_field_names: Optional accumulator for field names from item().field refs. + """ + for task in tasks: + notebook_task = task.get("notebook_task", {}) + params = notebook_task.get("base_parameters", {}) + for value in params.values(): + _extract_refs(value, param_names, item_field_names=item_field_names) + + run_job_task = task.get("run_job_task", {}) + for value in run_job_task.get("job_parameters", {}).values(): + _extract_refs(value, param_names, item_field_names=item_field_names) + + condition_task = task.get("condition_task", {}) + if condition_task: + _extract_refs(condition_task.get("left", ""), param_names, item_field_names=item_field_names) + _extract_refs(condition_task.get("right", ""), param_names, item_field_names=item_field_names) + _scan_tasks(condition_task.get("if_true", []), param_names, item_field_names=item_field_names) + _scan_tasks(condition_task.get("if_false", []), param_names, item_field_names=item_field_names) + + for_each_task = task.get("for_each_task", {}) + body = for_each_task.get("task") + if body: + _scan_tasks([body], param_names, item_field_names=item_field_names) + + +def _scan_ir_tasks( + ir_tasks: list[dict[str, Any]], + param_names: set[str], + *, + item_field_names: set[str] | None = None, +) -> None: + """Scans raw IR task dicts for parameter references in all fields. + + Args: + ir_tasks: Raw serialised IR task dicts. + param_names: Accumulator set of discovered parameter names. + item_field_names: Optional accumulator for field names from item().field refs. + """ + field_name_kwargs = {"item_field_names": item_field_names} + for task_dict in ir_tasks: + _extract_refs(task_dict.get("url", ""), param_names, **field_name_kwargs) + _extract_refs(task_dict.get("body"), param_names, **field_name_kwargs) + if isinstance(task_dict.get("headers"), dict): + for value in task_dict["headers"].values(): + _extract_refs(value, param_names, **field_name_kwargs) + + base_parameters = task_dict.get("base_parameters") + if isinstance(base_parameters, dict): + for value in base_parameters.values(): + _extract_refs(value, param_names, **field_name_kwargs) + + _extract_refs(task_dict.get("on_expression", ""), param_names, **field_name_kwargs) + for case in task_dict.get("cases", []): + _scan_ir_tasks(case.get("activities", []), param_names, **field_name_kwargs) + _scan_ir_tasks(task_dict.get("default_activities", []), param_names, **field_name_kwargs) + + _extract_refs(task_dict.get("op", ""), param_names, **field_name_kwargs) + _extract_refs(task_dict.get("left", ""), param_names, **field_name_kwargs) + _extract_refs(task_dict.get("right", ""), param_names, **field_name_kwargs) + _scan_ir_tasks(task_dict.get("if_true_activities", []), param_names, **field_name_kwargs) + _scan_ir_tasks(task_dict.get("if_false_activities", []), param_names, **field_name_kwargs) + + _scan_ir_tasks(task_dict.get("inner_activities", []), param_names, **field_name_kwargs) + + +def _extract_refs( + value: Any, + param_names: set[str], + *, + item_field_names: set[str] | None = None, +) -> None: + """Extracts parameter names from a single value that may be a string or ADF expression dict.""" + text = "" + if isinstance(value, str): + text = value + elif isinstance(value, dict): + text = value.get("value", "") if value.get("type") == "Expression" else "" + + if not text: + return + + for match in _PIPELINE_PARAM_RE.finditer(text): + param_names.add(match.group(1)) + + for match in _VARIABLES_RE.finditer(text): + param_names.add(match.group(1)) + + for match in _ITEM_FIELD_RE.finditer(text): + field_name = match.group(1) + param_names.add(field_name) + if item_field_names is not None: + item_field_names.add(field_name) + + # Only add bare "item" if there's an item() that isn't part of item().field + bare_text = _ITEM_FIELD_RE.sub("", text) + if _ITEM_BARE_RE.search(bare_text): + param_names.add("item") + + for match in _DAB_INPUT_FIELD_RE.finditer(text): + field_name = match.group(1) + param_names.add(field_name) + if item_field_names is not None: + item_field_names.add(field_name) + + for match in _DAB_JOB_PARAM_RE.finditer(text): + param_names.add(match.group(1)) + + +def normalize_value(value: Any) -> str: + """Normalize a single parameter value from ADF expression to DAB reference. + + Args: + value: A string or ``{type: Expression, value: ...}`` dict. + + Returns: + A normalized string with ``{{job.parameters.*}}`` references. + """ + if isinstance(value, dict) and value.get("type") == "Expression": + text = value.get("value", "") + elif isinstance(value, str): + text = value + else: + return str(value) + + # Strip ADF type-conversion wrappers (@string, @int, etc.) first so the + # inner expression can be resolved by subsequent steps. + match = _TYPE_CAST_RE.match(text.strip()) + if match: + text = match.group(2) + # Re-add @ prefix if the inner expression is a function call or + # reference that needs it for pattern matching. + if not text.startswith("@") and not text.startswith("{{"): + text = "@" + text + + text = _replace_refs(text) + text = _resolve_concat(text) + + return text + + +def _replace_refs(text: str) -> str: + """Replaces ADF references with {{job.parameters.*}} or {{job.*}} refs. + + Args: + text: Expression text potentially containing ADF references. + + Returns: + Text with references replaced. + """ + text = _DAB_INPUT_FIELD_RE.sub(r"{{job.parameters.\1}}", text) + text = _ITEM_FIELD_RE.sub(r"{{job.parameters.\1}}", text) + text = _ITEM_BARE_RE.sub("{{job.parameters.item}}", text) + text = _PIPELINE_PARAM_RE.sub(r"{{job.parameters.\1}}", text) + text = _PIPELINE_RUNID_RE.sub("{{job.run_id}}", text) + text = _PIPELINE_TRIGGER_TIME_RE.sub("{{job.start_time.iso_datetime}}", text) + text = _PIPELINE_NAME_RE.sub("{{job.name}}", text) + text = _PIPELINE_GROUPID_RE.sub("{{job.run_id}}", text) + text = _ACTIVITY_OUTPUT_RE.sub(r"{{tasks.\1.values.\2}}", text) + text = _VARIABLES_RE.sub(r"{{job.parameters.\1}}", text) + text = _UTCNOW_RE.sub("{{job.start_time.iso_datetime}}", text) + + return text + + +def _resolve_concat(text: str) -> str: + """Resolves ``@concat(arg1, arg2, ...)`` to a plain concatenated string. + + Args: + text: Expression that may be a @concat(...) call. + + Returns: + Resolved string, or original text if not resolvable. + """ + match = _CONCAT_RE.match(text.strip()) + if not match: + return text + + args_str = match.group(1) + parts = _split_concat_args(args_str) + if parts is None: + return text + + resolved: list[str] = [] + for part in parts: + part = part.strip() + if part.startswith("'") and part.endswith("'"): + resolved.append(part[1:-1]) + elif part.startswith("{{"): + resolved.append(part) + else: + return text + + return "".join(resolved) + + +def _split_concat_args(args_str: str) -> list[str] | None: + """Splits concat arguments respecting nested parentheses and quotes. + + Args: + args_str: The argument string inside concat(...). + + Returns: + List of argument strings, or None if parsing fails. + """ + parts: list[str] = [] + depth = 0 + current: list[str] = [] + in_quote = False + + for char in args_str: + if char == "'" and depth == 0: + in_quote = not in_quote + current.append(char) + elif in_quote: + current.append(char) + elif char == "(": + depth += 1 + current.append(char) + elif char == ")": + if depth == 0: + return None + depth -= 1 + current.append(char) + elif char == "," and depth == 0: + parts.append("".join(current).strip()) + current = [] + else: + current.append(char) + + if current: + parts.append("".join(current).strip()) + + if depth != 0 or in_quote: + return None + + return parts diff --git a/src/orchestra/bundler/notebook_writer.py b/src/orchestra/bundler/notebook_writer.py new file mode 100644 index 0000000..7817b7a --- /dev/null +++ b/src/orchestra/bundler/notebook_writer.py @@ -0,0 +1,22 @@ +"""Utility to write DabNotebook objects to files on disk.""" + +from __future__ import annotations + +from pathlib import Path + +from flowx.models.dab import DabNotebook + + +def write_notebooks(notebooks: list[DabNotebook], output_dir: Path) -> list[Path]: + """Writes each notebook to ``output_dir/`` and returns the absolute paths.""" + created: list[Path] = [] + for notebook in notebooks: + destination = output_dir / notebook.relative_path + destination.parent.mkdir(parents=True, exist_ok=True) + if notebook.binary_content is not None: + destination.write_bytes(notebook.binary_content) + else: + content = notebook.content if notebook.content.endswith("\n") else notebook.content + "\n" + destination.write_text(content, encoding="utf-8") + created.append(destination.resolve()) + return created diff --git a/src/orchestra/bundler/prereqs_writer.py b/src/orchestra/bundler/prereqs_writer.py new file mode 100644 index 0000000..934b8bc --- /dev/null +++ b/src/orchestra/bundler/prereqs_writer.py @@ -0,0 +1,538 @@ +"""Generates a SETUP.md file listing steps required before a bundle can run.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from flowx.models.dab import DabNotebook + +# Regexes used to mine the generated artifacts for external dependencies. +# Kept as compiled patterns so :func:`build_prereqs` is cheap to call. +_SECRET_REFERENCE = re.compile( + r"""dbutils\.secrets\.get\(\s*scope\s*=\s*["']([^"']+)["']\s*,\s*key\s*=\s*["']([^"']+)["']""", +) +_WORKSPACE_PATH_HEADER = re.compile( + r"\*\*Source workspace path\*\*:\s*`([^`]+)`", +) +_NOT_IMPLEMENTED_STUB = re.compile(r"\braise\s+NotImplementedError\b") +_WIDGET_REFERENCE = re.compile(r"""dbutils\.widgets\.get\(\s*["']([^"']+)["']\s*\)""") + + +@dataclass(slots=True, kw_only=True) +class MissingNotebook: + """A notebook that must be authored or imported before the bundle can run. + + Attributes: + task_key: DAB task key that references the notebook. + workspace_path: Original ADF/workspace path (empty if unknown). + bundle_path: Relative path within the bundle where the stub lives. + widget_names: Widget names the task passes via ``base_parameters`` + — useful for the author of the replacement notebook. + """ + + task_key: str + workspace_path: str + bundle_path: str + widget_names: list[str] = field(default_factory=list) + + +@dataclass(slots=True, kw_only=True) +class CrossBundleReference: + """A ``run_job_task`` pointing at a job defined in a different bundle. + + Attributes: + task_key: DAB task key of the caller. + target_pipeline: Name of the referenced pipeline/job. + """ + + task_key: str + target_pipeline: str + + +@dataclass(slots=True, kw_only=True) +class EmptyParameter: + """A task widget the notebook reads but the translator couldn't populate. + + Attributes: + task_key: DAB task key. + widget_name: Widget name with an empty default. + """ + + task_key: str + widget_name: str + + +@dataclass(slots=True, kw_only=True) +class ManualParameter: + """A base_parameter that the user must compute inside an existing notebook. + + Attributes: + task_key: DAB task key. + widget_name: The base_parameter name. + notebook_path: Workspace path of the existing notebook. + raw_expression: The original ADF expression (verbatim) so the user + knows what runtime value they need to compute. + """ + + task_key: str + widget_name: str + notebook_path: str + raw_expression: str + + +@dataclass(slots=True, kw_only=True) +class NetworkEndpoint: + """A piece of network connectivity the bundle's notebooks expect. + + Attributes: + kind: One of ``"jdbc"`` (database server), ``"http"`` (REST endpoint), + or ``"storage"`` (object storage / UC volume). + target: A short identifier for the endpoint — the JDBC scope name, + the URL host, or the storage account. + notes: Free-form guidance shown next to the endpoint in SETUP.md. + """ + + kind: str + target: str + notes: str = "" + + +@dataclass(slots=True, kw_only=True) +class Prereqs: + """Collected external dependencies for one bundle.""" + + secrets: dict[str, set[str]] = field(default_factory=dict) # scope -> {keys} + missing_notebooks: list[MissingNotebook] = field(default_factory=list) + cross_bundle_refs: list[CrossBundleReference] = field(default_factory=list) + empty_parameters: list[EmptyParameter] = field(default_factory=list) + compute_notes: list[str] = field(default_factory=list) + network_endpoints: list[NetworkEndpoint] = field(default_factory=list) + manual_parameters: list[ManualParameter] = field(default_factory=list) + + def is_empty(self) -> bool: + """Return ``True`` when nothing needs to happen before ``bundle run``.""" + return ( + not self.secrets + and not self.missing_notebooks + and not self.cross_bundle_refs + and not self.empty_parameters + and not self.compute_notes + and not self.network_endpoints + and not self.manual_parameters + ) + + +def _walk_tasks(tasks: list[dict[str, Any]]): + """Yields every task dict including nested ``for_each_task.task`` bodies.""" + for task in tasks: + yield task + for_each = task.get("for_each_task") + if for_each and isinstance(for_each.get("task"), dict): + yield from _walk_tasks([for_each["task"]]) + + +def scan_notebooks_for_secrets(notebooks: list[DabNotebook]) -> dict[str, set[str]]: + """Extracts all ``dbutils.secrets.get(scope=..., key=...)`` references. + + Args: + notebooks: Generated notebooks in the bundle. + + Returns: + Mapping of scope name to the set of keys referenced within that scope. + """ + scopes: dict[str, set[str]] = {} + for notebook in notebooks: + for scope_name, key in _SECRET_REFERENCE.findall(notebook.content): + scopes.setdefault(scope_name, set()).add(key) + return scopes + + +def collect_missing_notebooks( + notebooks: list[DabNotebook], + tasks: list[dict[str, Any]], +) -> list[MissingNotebook]: + """Identify notebook stubs the user must replace before running the bundle. + + Args: + notebooks: Generated notebooks in the bundle. + tasks: Top-level task dicts (used to correlate widgets and paths). + + Returns: + List of :class:`MissingNotebook` entries sorted by task_key. + """ + task_by_path: dict[str, dict[str, Any]] = {} + for task in _walk_tasks(tasks): + notebook_task = task.get("notebook_task") or {} + path = notebook_task.get("notebook_path", "") + if path: + task_by_path[path] = task + + missing: list[MissingNotebook] = [] + for notebook in notebooks: + if not _NOT_IMPLEMENTED_STUB.search(notebook.content): + continue + + workspace_match = _WORKSPACE_PATH_HEADER.search(notebook.content) + workspace_path = workspace_match.group(1) if workspace_match else "" + + bundle_relative_notebook_path = f"../src/{notebook.relative_path}" + task = task_by_path.get(bundle_relative_notebook_path) + task_key = task.get("task_key", notebook.relative_path) if task else notebook.relative_path + base_parameters: dict[str, str] = {} + if task and "notebook_task" in task: + base_parameters = task["notebook_task"].get("base_parameters") or {} + widget_names = sorted(base_parameters.keys()) + + missing.append( + MissingNotebook( + task_key=task_key, + workspace_path=workspace_path, + bundle_path=f"src/{notebook.relative_path}", + widget_names=widget_names, + ) + ) + + missing.sort(key=lambda n: n.task_key) + return missing + + +def collect_cross_bundle_refs(tasks: list[dict[str, Any]], known_bundle_jobs: set[str]) -> list[CrossBundleReference]: + """Finds ``run_job_task`` entries pointing outside this bundle. + + Args: + tasks: Top-level task dicts. + known_bundle_jobs: Resource keys of jobs defined in this bundle. + + Returns: + List of :class:`CrossBundleReference` entries. + """ + pattern = re.compile(r"\$\{resources\.jobs\.([^.]+)\.id\}") + refs: list[CrossBundleReference] = [] + for task in _walk_tasks(tasks): + run_job = task.get("run_job_task") + if not run_job: + continue + job_id_reference = run_job.get("job_id", "") + match = pattern.search(str(job_id_reference)) + if not match: + continue + target = match.group(1) + if target in known_bundle_jobs: + continue + refs.append(CrossBundleReference(task_key=task.get("task_key", ""), target_pipeline=target)) + return refs + + +def collect_empty_parameters(tasks: list[dict[str, Any]]) -> list[EmptyParameter]: + """Finds base_parameters whose values are empty strings. + + Args: + tasks: All top-level task dicts in the bundle. + + Returns: + One :class:`EmptyParameter` per empty widget, sorted for stability. + """ + empty: list[EmptyParameter] = [] + for task in _walk_tasks(tasks): + notebook_task = task.get("notebook_task") or {} + for widget_name, value in (notebook_task.get("base_parameters") or {}).items(): + if isinstance(value, str) and value == "": + empty.append(EmptyParameter(task_key=task.get("task_key", ""), widget_name=widget_name)) + empty.sort(key=lambda parameter: (parameter.task_key, parameter.widget_name)) + return empty + + +def collect_network_endpoints(notebooks: list[DabNotebook]) -> list[NetworkEndpoint]: + """Scans generated notebook content for network-dependent endpoints. + + Args: + notebooks: All generated notebooks in the bundle. + + Returns: + Sorted, deduplicated list of :class:`NetworkEndpoint` records. + """ + seen: set[tuple[str, str]] = set() + endpoints: list[NetworkEndpoint] = [] + + jdbc_scope_re = re.compile( + r"""dbutils\.secrets\.get\(\s*scope\s*=\s*["']([^"']+)["']\s*,\s*key\s*=\s*["']jdbc-url["']""", + ) + requests_re = re.compile(r"\brequests\.(?:get|post|put|patch|delete|request)\(") + https_url_re = re.compile(r"https?://[A-Za-z0-9.\-]+(?::\d+)?/?") + storage_url_re = re.compile(r"(?:abfss|wasbs)://[A-Za-z0-9_.\-@/]+") + + for notebook in notebooks: + content = notebook.content + for scope_name in jdbc_scope_re.findall(content): + key = ("jdbc", scope_name) + if key not in seen: + seen.add(key) + endpoints.append( + NetworkEndpoint( + kind="jdbc", + target=scope_name, + notes=( + "JDBC database read by the generated notebook. " + "Confirm the workspace has network reach to the database " + "(VNet peering, private endpoint, or firewall allowlist) " + "before running the job." + ), + ) + ) + + if requests_re.search(content): + for url in https_url_re.findall(content): + # Trim placeholders like ``https://example.com/...`` that we + # emit in fallback bodies — they are not real endpoints. + if "example.com" in url: + continue + key = ("http", url) + if key not in seen: + seen.add(key) + endpoints.append( + NetworkEndpoint( + kind="http", + target=url, + notes="HTTP/S endpoint reached via `requests`. Verify outbound HTTPS is allowed.", + ) + ) + + for storage_url in storage_url_re.findall(content): + key = ("storage", storage_url) + if key not in seen: + seen.add(key) + endpoints.append( + NetworkEndpoint( + kind="storage", + target=storage_url, + notes="Cloud storage path. Best reached through a Unity Catalog external volume.", + ) + ) + + endpoints.sort(key=lambda endpoint: (endpoint.kind, endpoint.target)) + return endpoints + + +def build_prereqs( + *, + notebooks: list[DabNotebook], + tasks: list[dict[str, Any]], + known_bundle_jobs: set[str], + cross_bundle_variables: dict[str, str] | None = None, + compute_notes: list[str] | None = None, + manual_parameters: list[ManualParameter] | None = None, +) -> Prereqs: + """Assemble a :class:`Prereqs` from the bundle's generated artifacts. + + Args: + notebooks: All generated notebooks (including inner-workflow notebooks). + tasks: All task dicts in the bundle (including inner-workflow tasks). + known_bundle_jobs: Resource keys for every job defined in this bundle. + cross_bundle_variables: Map of bundle-variable name → target pipeline + name for every ExecutePipeline reference the bundler translated + as ``${var.}``. The user must supply a numeric job ID for + each one before running. + compute_notes: Free-form compute/configuration notes to surface. + + Returns: + A :class:`Prereqs` aggregating everything the user must do before + ``databricks bundle run``. + """ + cross_bundle = [ + CrossBundleReference(task_key=variable_name, target_pipeline=target_pipeline) + for variable_name, target_pipeline in sorted((cross_bundle_variables or {}).items()) + ] + # Also fold in any residual ${resources.jobs.X.id} refs we see directly in + # the tasks (in case upstream still emits them). + cross_bundle.extend(collect_cross_bundle_refs(tasks, known_bundle_jobs)) + + return Prereqs( + secrets=scan_notebooks_for_secrets(notebooks), + missing_notebooks=collect_missing_notebooks(notebooks, tasks), + cross_bundle_refs=cross_bundle, + empty_parameters=collect_empty_parameters(tasks), + compute_notes=list(compute_notes or []), + network_endpoints=collect_network_endpoints(notebooks), + manual_parameters=list(manual_parameters or []), + ) + + +def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: + """Renders a :class:`Prereqs` into a human-readable ``SETUP.md``. + + Args: + prereqs: Collected dependencies for this bundle. + bundle_name: Bundle name (used in the header). + + Returns: + Markdown source as a string. Always begins with a header so the + file is consistent even when nothing needs to happen. + """ + lines: list[str] = [ + f"# Setup for bundle `{bundle_name}`", + "", + ] + + if prereqs.is_empty(): + lines.extend( + [ + "This bundle has no external prerequisites. Deploy it and run the job:", + "", + "```bash", + "databricks bundle validate", + "databricks bundle deploy -t dev", + "```", + "", + ] + ) + return "\n".join(lines) + + lines.append( + "Complete every step below before running the bundle. " + "Deployment itself is **not** listed — it is the step that comes *after* everything here." + ) + lines.append("") + + if prereqs.secrets: + lines.append("## Secret scopes and values") + lines.append("") + lines.append( + "The generated notebooks read credentials via `dbutils.secrets.get(...)`. " + "You have two equivalent ways to provision the scopes and keys:" + ) + lines.append("") + lines.append( + "**Option A — run `src/setup/create_secrets.py`** in the target workspace. " + "It creates every scope and populates each key with a `PLACEHOLDER` value " + "that you then replace with a real credential." + ) + lines.append("") + lines.append("**Option B — use the CLI directly:**") + lines.append("") + lines.append("```bash") + for scope_name in sorted(prereqs.secrets): + lines.append(f"databricks secrets create-scope {scope_name}") + for key in sorted(prereqs.secrets[scope_name]): + lines.append(f"databricks secrets put-secret {scope_name} {key}") + lines.append("```") + lines.append("") + lines.append( + '`put-secret` opens an editor by default; pass `--json \'{"string_value": "…"}\'` ' + "for a non-interactive flow." + ) + lines.append("") + + if prereqs.missing_notebooks: + lines.append("## Notebooks to author") + lines.append("") + lines.append( + "The following notebooks are stubs that raise `NotImplementedError`. " + "Flowx could not download the source (either no workspace path was " + "supplied in ADF, or the path did not resolve against the " + "authenticated workspace). Replace each stub with the real logic." + ) + lines.append("") + lines.append("| Task | Workspace path (ADF) | Stub in bundle | Widgets available |") + lines.append("|---|---|---|---|") + for missing_notebook in prereqs.missing_notebooks: + source_cell = f"`{missing_notebook.workspace_path}`" if missing_notebook.workspace_path else "*(none)*" + widgets_cell = ( + ", ".join(f"`{name}`" for name in missing_notebook.widget_names) + if missing_notebook.widget_names + else "*(none)*" + ) + lines.append( + f"| `{missing_notebook.task_key}` | {source_cell} | `{missing_notebook.bundle_path}` | {widgets_cell} |" + ) + lines.append("") + lines.append( + "If the workspace path exists in a reachable Databricks workspace, you can " + "have Flowx re-ingest it by running `databricks workspace export` and " + "placing the result at the indicated bundle path." + ) + lines.append("") + + if prereqs.cross_bundle_refs: + lines.append("## Cross-bundle job references") + lines.append("") + lines.append( + "Each row below describes a `run_job_task` that invokes a job **not** " + "defined in this bundle. Flowx emitted a bundle variable for each " + "one (`${var.}`) so `databricks bundle validate` passes. " + "Before running, populate the variable with the numeric job ID the " + "target pipeline was deployed under — either set a `default:` in " + '`databricks.yml` or pass `--var "="` at deploy time.' + ) + lines.append("") + lines.append("| Variable | Target pipeline |") + lines.append("|---|---|") + for ref in prereqs.cross_bundle_refs: + lines.append(f"| `{ref.task_key}` | `{ref.target_pipeline}` |") + lines.append("") + + if prereqs.empty_parameters: + lines.append("## Unresolved task parameters") + lines.append("") + lines.append( + "The translator left the base_parameters below with empty-string " + "defaults — either the source ADF activity carried a value that " + "couldn't be resolved, or it depended on pipeline state (variables, " + "activity outputs) that doesn't cross the bundle boundary. Review " + "each entry and either set a real default in the job YAML or pass " + "a value via `databricks bundle run ... --params '{:}'`." + ) + lines.append("") + lines.append("| Task | Widget |") + lines.append("|---|---|") + for empty_parameter in prereqs.empty_parameters: + lines.append(f"| `{empty_parameter.task_key}` | `{empty_parameter.widget_name}` |") + lines.append("") + + if prereqs.compute_notes: + lines.append("## Compute configuration") + lines.append("") + for note in prereqs.compute_notes: + lines.append(f"- {note}") + lines.append("") + + if prereqs.manual_parameters: + lines.append("## Existing-notebook parameter handling") + lines.append("") + lines.append( + "The translator could not evaluate the following base_parameters into DAB-compatible " + "values. These come from ADF expressions that need runtime context (e.g. `utcnow()`, " + "`activity().output...`) which DAB does not evaluate. The tasks below already point " + "at existing workspace notebooks, so flowx cannot inject the computation; **update " + "the listed notebooks to compute each value in-line** (the original ADF expression is " + "shown so you know what runtime value to produce)." + ) + lines.append("") + lines.append("| Task | Existing notebook | Widget | Original ADF expression |") + lines.append("|---|---|---|---|") + for manual_parameter in prereqs.manual_parameters: + lines.append( + f"| `{manual_parameter.task_key}` | `{manual_parameter.notebook_path}` " + f"| `{manual_parameter.widget_name}` | `{manual_parameter.raw_expression}` |" + ) + lines.append("") + + if prereqs.network_endpoints: + lines.append("## Networking") + lines.append("") + lines.append( + "The generated notebooks reach the following endpoints. Confirm the workspace's " + "network profile permits each one before running the bundle. Private endpoints, " + "VNet peering, or storage credentials may be required." + ) + lines.append("") + lines.append("| Type | Target | Notes |") + lines.append("|---|---|---|") + kind_label = {"jdbc": "Database (JDBC)", "http": "HTTP/S", "storage": "Cloud storage"} + for endpoint in prereqs.network_endpoints: + label = kind_label.get(endpoint.kind, endpoint.kind) + lines.append(f"| {label} | `{endpoint.target}` | {endpoint.notes} |") + lines.append("") + + return "\n".join(lines) diff --git a/src/orchestra/bundler/setup_generator.py b/src/orchestra/bundler/setup_generator.py new file mode 100644 index 0000000..e637cf5 --- /dev/null +++ b/src/orchestra/bundler/setup_generator.py @@ -0,0 +1,308 @@ +"""Generates setup notebooks for creating Databricks resources needed by translated jobs.""" + +from __future__ import annotations + +import textwrap + +from flowx.models.dab import DabNotebook, SecretInstruction, SetupTask + + +def generate_setup_tasks( + secrets: list[SecretInstruction], + setup_tasks: list[SetupTask], + catalog: str, + schema: str, +) -> list[DabNotebook]: + """Generates setup notebooks for provisioning required Databricks resources. + + Args: + secrets: Secret instructions collected from all prepared activities. + setup_tasks: Additional setup tasks (volume creation, connections, etc.). + catalog: Target Unity Catalog name. + schema: Target schema name within the catalog. + + Returns: + List of DabNotebook objects for the setup notebooks. + """ + notebooks: list[DabNotebook] = [] + + if secrets: + notebook = _generate_secrets_setup_notebook(secrets) + notebooks.append(notebook) + + volume_tasks = [t for t in setup_tasks if t.type == "volume"] + if volume_tasks: + notebook = _generate_volume_setup_notebook(volume_tasks, catalog, schema) + notebooks.append(notebook) + + connection_tasks = [t for t in setup_tasks if t.type == "connection"] + if connection_tasks: + notebook = _generate_connection_setup_notebook(connection_tasks, catalog) + notebooks.append(notebook) + + return notebooks + + +def _generate_secrets_setup_notebook(secrets: list[SecretInstruction]) -> DabNotebook: + """Generates a notebook that creates secret scopes and populates secrets. + + Args: + secrets: Secret instructions to provision. + + Returns: + A DabNotebook for the secrets setup notebook. + """ + header = textwrap.dedent("""\ + # Databricks notebook source + # MAGIC %md + # MAGIC # Setup: Create Secret Scopes and Secrets + # MAGIC + # MAGIC *Auto-generated by Flowx. Run this notebook once before deploying the job.* + # MAGIC + # MAGIC This notebook creates the Databricks secret scopes and placeholder secrets + # MAGIC required by the translated pipelines. After running, update each secret + # MAGIC with the real credential value using the Databricks CLI: + # MAGIC + # MAGIC ```bash + # MAGIC databricks secrets put-secret + # MAGIC ``` + """) + + separator = "\n# COMMAND ----------\n\n" + + scopes: dict[str, list[SecretInstruction]] = {} + for s in secrets: + scopes.setdefault(s.scope, []).append(s) + + body_parts: list[str] = [] + for scope_name, scope_secrets in sorted(scopes.items()): + lines: list[str] = [f"# Create scope: {scope_name}"] + lines.append("try:") + lines.append(f' dbutils.secrets.createScope(scope="{scope_name}")') + lines.append(f' print("Created scope: {scope_name}")') + lines.append("except Exception as e:") + lines.append(' if "RESOURCE_ALREADY_EXISTS" in str(e):') + lines.append(f' print("Scope already exists: {scope_name}")') + lines.append(" else:") + lines.append(" raise") + lines.append("") + + for secret in scope_secrets: + lines.append(f"# {secret.value_source}") + lines.append(f'dbutils.secrets.put(scope="{scope_name}", key="{secret.key}", string_value="PLACEHOLDER")') + lines.append(f'print("Created secret: {scope_name}/{secret.key}")') + lines.append("") + + body_parts.append("\n".join(lines)) + + body = separator.join(body_parts) + + summary = separator + "# MAGIC %md\n" + summary += "# MAGIC ## Next Steps\n" + summary += "# MAGIC\n" + summary += f"# MAGIC Created **{len(scopes)}** scope(s) and **{len(secrets)}** secret(s).\n" + summary += "# MAGIC\n" + summary += "# MAGIC Replace each PLACEHOLDER value with the real credential:\n" + summary += "# MAGIC ```bash\n" + for scope_name, scope_secrets in sorted(scopes.items()): + for secret in scope_secrets: + summary += f"# MAGIC databricks secrets put-secret {scope_name} {secret.key}\n" + summary += "# MAGIC ```\n" + + content = header + separator + body + summary + return DabNotebook( + relative_path="setup/create_secrets.py", + content=content, + ) + + +def _generate_volume_setup_notebook( + volume_tasks: list[SetupTask], + catalog: str, + schema: str, +) -> DabNotebook: + """Generates a notebook that creates Unity Catalog volumes. + + Args: + volume_tasks: Volume creation setup tasks. Each task's ``config`` + carries ``volume_name``, ``volume_type`` (``MANAGED`` / + ``EXTERNAL``), ``location`` (``abfss://`` / ``s3://`` / + ``gs://`` URL), ``location_type`` (ADF dataset location type), + and optional ``storage_account``. + catalog: Target Unity Catalog name. + schema: Target schema name. + + Returns: + A DabNotebook for the volume setup notebook. + """ + header = textwrap.dedent("""\ + # Databricks notebook source + # MAGIC %md + # MAGIC # Setup: Create Unity Catalog Volumes + # MAGIC + # MAGIC *Auto-generated by Flowx. Run this notebook once before deploying the job.* + # MAGIC + # MAGIC For each external volume below, this notebook attempts to create the + # MAGIC underlying Storage Credential and External Location first. Both require + # MAGIC `CREATE STORAGE CREDENTIAL` / `CREATE EXTERNAL LOCATION` privileges and + # MAGIC the appropriate cloud-side configuration: + # MAGIC + # MAGIC - **Azure** — an Access Connector for Azure Databricks with + # MAGIC Storage Blob Data Contributor on the storage account. + # MAGIC - **AWS** — an IAM role with read/write on the bucket trusted by the + # MAGIC Databricks IAM principal. + # MAGIC - **GCP** — a service account with `roles/storage.admin` (or finer) + # MAGIC on the bucket. + # MAGIC + # MAGIC Replace the `PLACEHOLDER` values in the credential cells before running. + """) + + separator = "\n# COMMAND ----------\n\n" + + # Group external-location and credential creation by URL so we don't + # emit duplicate DDL when several sinks share a container. + seen_locations: set[str] = set() + seen_credentials: set[str] = set() + + body_parts: list[str] = [] + for task in volume_tasks: + config = task.config + volume_name = config.get("volume_name", "unknown") + volume_type = (config.get("volume_type") or "MANAGED").upper() + location = config.get("location", "") + location_type = config.get("location_type", "") or "" + + fqn = f"{catalog}.{schema}.{volume_name}" + lines: list[str] = [f"# Create volume: {volume_name}"] + + if volume_type == "EXTERNAL" and location: + credential_name = _credential_name_for(volume_name) + external_location_name = _external_location_name_for(volume_name) + + if credential_name not in seen_credentials: + seen_credentials.add(credential_name) + lines.append(_render_storage_credential_ddl(credential_name, location_type)) + + if location not in seen_locations: + seen_locations.add(location) + lines.append(_render_external_location_ddl(external_location_name, location, credential_name)) + + lines.append(f"spark.sql(\"CREATE EXTERNAL VOLUME IF NOT EXISTS {fqn} LOCATION '{location}'\")") + else: + lines.append(f'spark.sql("CREATE VOLUME IF NOT EXISTS {fqn}")') + + lines.append(f'print("Created volume: {fqn}")') + body_parts.append("\n".join(lines)) + + content = header + separator + separator.join(body_parts) + "\n" + return DabNotebook( + relative_path="setup/create_volumes.py", + content=content, + ) + + +def _credential_name_for(volume_name: str) -> str: + return f"orchestra_{volume_name}_credential" + + +def _external_location_name_for(volume_name: str) -> str: + return f"orchestra_{volume_name}_location" + + +def _render_storage_credential_ddl(credential_name: str, location_type: str) -> str: + """Emits a CREATE STORAGE CREDENTIAL block keyed off the source cloud.""" + if location_type in ("AzureBlobStorageLocation", "AzureBlobFSLocation"): + return ( + "# Azure: requires an Access Connector for Azure Databricks granted\n" + "# Storage Blob Data Contributor on the target account.\n" + 'spark.sql("""\n' + f" CREATE STORAGE CREDENTIAL IF NOT EXISTS {credential_name}\n" + " WITH AZURE_MANAGED_IDENTITY (\n" + " 'PLACEHOLDER_ACCESS_CONNECTOR_RESOURCE_ID'\n" + " )\n" + f" COMMENT 'Auto-generated by Flowx for volume {credential_name}'\n" + '""")' + ) + if location_type == "AmazonS3Location": + return ( + "# AWS: requires an IAM role trusted by the Databricks principal.\n" + 'spark.sql("""\n' + f" CREATE STORAGE CREDENTIAL IF NOT EXISTS {credential_name}\n" + " WITH IAM_ROLE 'arn:aws:iam::PLACEHOLDER_ACCOUNT_ID:role/PLACEHOLDER_ROLE'\n" + f" COMMENT 'Auto-generated by Flowx for volume {credential_name}'\n" + '""")' + ) + if location_type == "GoogleCloudStorageLocation": + return ( + "# GCP: requires a Databricks service account email.\n" + 'spark.sql("""\n' + f" CREATE STORAGE CREDENTIAL IF NOT EXISTS {credential_name}\n" + " WITH GCP_SERVICE_ACCOUNT 'PLACEHOLDER_SERVICE_ACCOUNT_EMAIL'\n" + f" COMMENT 'Auto-generated by Flowx for volume {credential_name}'\n" + '""")' + ) + return ( + f"# Storage credential for {credential_name} could not be auto-typed (unknown\n" + f"# location_type={location_type!r}). Create it manually before this cell.\n" + f"# See https://docs.databricks.com/aws/en/connect/unity-catalog/storage-credentials" + ) + + +def _render_external_location_ddl(name: str, url: str, credential: str) -> str: + return ( + 'spark.sql("""\n' + f" CREATE EXTERNAL LOCATION IF NOT EXISTS {name}\n" + f" URL '{url}'\n" + f" WITH (STORAGE CREDENTIAL {credential})\n" + f" COMMENT 'Auto-generated by Flowx'\n" + '""")' + ) + + +def _generate_connection_setup_notebook( + connection_tasks: list[SetupTask], + catalog: str, +) -> DabNotebook: + """Generates a notebook that creates external connections. + + Args: + connection_tasks: Connection creation setup tasks. + catalog: Target Unity Catalog name. + + Returns: + A DabNotebook for the connection setup notebook. + """ + header = textwrap.dedent("""\ + # Databricks notebook source + # MAGIC %md + # MAGIC # Setup: Create External Connections + # MAGIC + # MAGIC *Auto-generated by Flowx. Run this notebook once before deploying the job.* + # MAGIC + # MAGIC Update the placeholder connection details below with real values before running. + """) + + separator = "\n# COMMAND ----------\n\n" + + body_parts: list[str] = [] + for task in connection_tasks: + config = task.config + conn_name = config.get("connection_name", "unknown") + conn_type = config.get("connection_type", "MYSQL") + host = config.get("host", "PLACEHOLDER_HOST") + port = config.get("port", "3306") + + lines: list[str] = [f"# Create connection: {conn_name}"] + lines.append('spark.sql("""') + lines.append(f" CREATE CONNECTION IF NOT EXISTS {conn_name}") + lines.append(f" TYPE {conn_type}") + lines.append(f" OPTIONS (host '{host}', port '{port}')") + lines.append('""")') + lines.append(f'print("Created connection: {conn_name}")') + body_parts.append("\n".join(lines)) + + content = header + separator + separator.join(body_parts) + "\n" + return DabNotebook( + relative_path="setup/create_connections.py", + content=content, + ) diff --git a/src/orchestra/models/__init__.py b/src/orchestra/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/orchestra/models/adf_ast.py b/src/orchestra/models/adf_ast.py new file mode 100644 index 0000000..e25c90e --- /dev/null +++ b/src/orchestra/models/adf_ast.py @@ -0,0 +1,294 @@ +"""Typed ADF AST nodes.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + + +class TranslationStrategy(Enum): + """Classification of how an ADF activity should be translated.""" + + DETERMINISTIC = "deterministic" + AGENTIC = "agentic" + UNSUPPORTED = "unsupported" + + +# --------------------------------------------------------------------------- +# Activity-level AST nodes +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class AdfDependency: + """Dependency edge between two ADF activities. + + Attributes: + activity: Name of the upstream activity. + dependency_conditions: Required outcome(s) (e.g. ``["Succeeded"]``). + """ + + activity: str + dependency_conditions: list[str] = field(default_factory=lambda: ["Succeeded"]) + + +@dataclass(slots=True, kw_only=True) +class AdfPolicy: + """Retry / timeout policy attached to an ADF activity. + + Attributes: + timeout: Timeout string in ADF format (``"d.hh:mm:ss"`` or ``"hh:mm:ss"``). + retry: Maximum number of retries. + retry_interval_in_seconds: Delay between retries in seconds. + secure_input: Whether the activity input is masked in logs. + secure_output: Whether the activity output is masked in logs. + """ + + timeout: str | None = None + retry: int | None = None + retry_interval_in_seconds: int | None = None + secure_input: bool = False + secure_output: bool = False + + +@dataclass(slots=True, kw_only=True) +class AdfParameter: + """Pipeline-level parameter definition. + + Attributes: + type: ADF parameter type (``"String"``, ``"Int"``, ``"Bool"``, etc.). + default_value: Optional default value for the parameter. + """ + + type: str = "String" + default_value: Any = None + + +@dataclass(slots=True, kw_only=True) +class AdfVariable: + """Pipeline-level variable definition. + + Attributes: + type: ADF variable type. + default_value: Optional initial value. + """ + + type: str = "String" + default_value: Any = None + + +# --------------------------------------------------------------------------- +# Reference nodes +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class AdfDatasetReference: + """Reference to an ADF dataset used as an activity input or output. + + Attributes: + reference_name: Logical name of the dataset. + type: Reference type (always ``"DatasetReference"``). + parameters: Runtime parameters passed to the dataset, if any. + """ + + reference_name: str + type: str = "DatasetReference" + parameters: dict[str, Any] | None = None + + +@dataclass(slots=True, kw_only=True) +class AdfLinkedServiceReference: + """Reference to an ADF linked service. + + Attributes: + reference_name: Logical name of the linked service. + type: Reference type (always ``"LinkedServiceReference"``). + """ + + reference_name: str + type: str = "LinkedServiceReference" + + +# --------------------------------------------------------------------------- +# Activity node +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class AdfActivity: + """Single ADF activity node. + + Attributes: + name: Activity display name. + type: ADF activity type string (e.g. ``"Copy"``, ``"DatabricksNotebook"``). + depends_on: Upstream dependency edges. + policy: Retry / timeout policy. + type_properties: Raw ``typeProperties`` bag from the ADF JSON. + inputs: Dataset references consumed by the activity. + outputs: Dataset references produced by the activity. + linked_service_name: Linked service reference used by the activity. + if_true_activities: Activities to run when an IfCondition evaluates to true. + if_false_activities: Activities to run when an IfCondition evaluates to false. + activities: Child activities for ForEach / Until containers. + """ + + name: str + type: str + depends_on: list[AdfDependency] | None = None + policy: AdfPolicy | None = None + type_properties: dict[str, Any] | None = None + inputs: list[AdfDatasetReference] | None = None + outputs: list[AdfDatasetReference] | None = None + linked_service_name: AdfLinkedServiceReference | None = None + if_true_activities: list[AdfActivity] | None = None + if_false_activities: list[AdfActivity] | None = None + activities: list[AdfActivity] | None = None # ForEach, Until + + +# --------------------------------------------------------------------------- +# Pipeline node +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class AdfPipeline: + """Top-level ADF pipeline definition. + + Attributes: + name: Pipeline display name. + activities: Ordered list of activities that make up the pipeline. + parameters: Pipeline parameter declarations, keyed by name. + variables: Pipeline variable declarations, keyed by name. + annotations: Free-form annotation strings attached to the pipeline. + folder: Organisational folder path within the ADF workspace. + """ + + name: str + activities: list[AdfActivity] + parameters: dict[str, AdfParameter] | None = None + variables: dict[str, AdfVariable] | None = None + annotations: list[str] | None = None + folder: str | None = None + + +# --------------------------------------------------------------------------- +# Supporting definition nodes +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class AdfDataset: + """ADF dataset definition. + + Attributes: + name: Dataset display name. + type: Dataset type (e.g. ``"AzureSqlTable"``, ``"DelimitedText"``). + properties: Full properties bag from the ADF JSON. + linked_service_name: Name of the linked service backing this dataset. + """ + + name: str + type: str + properties: dict[str, Any] + linked_service_name: str | None = None + + +@dataclass(slots=True, kw_only=True) +class AdfLinkedService: + """ADF linked service definition. + + Attributes: + name: Linked service display name. + type: Service type (e.g. ``"AzureBlobStorage"``, ``"AzureSqlDatabase"``). + properties: Full properties bag from the ADF JSON. + """ + + name: str + type: str + properties: dict[str, Any] + + +@dataclass(slots=True, kw_only=True) +class AdfTrigger: + """ADF trigger definition. + + Attributes: + name: Trigger display name. + type: Trigger type (e.g. ``"ScheduleTrigger"``). + properties: Full properties bag from the ADF JSON. + pipelines: List of pipeline references activated by this trigger. + """ + + name: str + type: str + properties: dict[str, Any] + pipelines: list[dict[str, Any]] | None = None + + +# --------------------------------------------------------------------------- +# Aggregate containers +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class AdfDefinitions: + """Complete set of ADF definitions loaded from JSON files. + + Attributes: + pipelines: All pipeline definitions. + datasets: Dataset definitions keyed by name. + linked_services: Linked service definitions keyed by name. + triggers: Trigger definitions. + """ + + pipelines: list[AdfPipeline] + datasets: dict[str, AdfDataset] = field(default_factory=dict) + linked_services: dict[str, AdfLinkedService] = field(default_factory=dict) + triggers: list[AdfTrigger] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Inventory / classification +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class InventoryItem: + """Single row in the translation inventory. + + Attributes: + pipeline_name: Owning pipeline name. + activity_name: Activity display name. + activity_type: ADF activity type string. + strategy: Determined translation strategy. + agentic_skill: Skill identifier when strategy is ``AGENTIC``. + depends_on: Upstream activity names. + """ + + pipeline_name: str + activity_name: str + activity_type: str + strategy: TranslationStrategy + agentic_skill: str | None = None + depends_on: list[str] | None = None + + +@dataclass(slots=True, kw_only=True) +class Inventory: + """Aggregated translation inventory for all discovered pipelines. + + Attributes: + items: Individual inventory rows. + deterministic_count: Number of deterministically translatable activities. + agentic_count: Number of activities requiring agentic translation. + unsupported_count: Number of unsupported activities. + pipeline_count: Total number of pipelines inventoried. + """ + + items: list[InventoryItem] + deterministic_count: int = 0 + agentic_count: int = 0 + unsupported_count: int = 0 + pipeline_count: int = 0 diff --git a/src/orchestra/models/dab.py b/src/orchestra/models/dab.py new file mode 100644 index 0000000..23bd002 --- /dev/null +++ b/src/orchestra/models/dab.py @@ -0,0 +1,149 @@ +"""DAB output models.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +# --------------------------------------------------------------------------- +# Notebook / code artefacts +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class DabNotebook: + """Generated notebook to include in the bundle. + + Attributes: + relative_path: Path relative to the bundle root (e.g. ``"src/copy_data.py"``). + content: Full notebook source content. + language: Notebook language (``"python"``, ``"sql"``, ``"scala"``, ``"r"``). + binary_content: Raw bytes for binary files (e.g. JARs). When set, + the notebook writer writes these bytes instead of ``content``. + """ + + relative_path: str + content: str = "" + language: str = "python" + binary_content: bytes | None = None + + +# --------------------------------------------------------------------------- +# Job / task definitions +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class DabJob: + """Databricks workflow job definition. + + Attributes: + resource_key: Unique key used in the ``databricks.yml`` resources block. + name: Human-readable job name. + tasks: Ordered list of task configuration dictionaries. + schedule: Optional cron schedule configuration. + tags: Key-value tags applied to the job. + """ + + resource_key: str + name: str + tasks: list[dict[str, Any]] = field(default_factory=list) + schedule: dict[str, Any] | None = None + tags: dict[str, str] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Pipeline (Lakeflow Declarative Pipeline) +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class DabPipeline: + """Lakeflow Declarative Pipeline definition. + + Attributes: + resource_key: Unique key used in the ``databricks.yml`` resources block. + name: Human-readable pipeline name. + catalog: Unity Catalog catalog for the pipeline output. + schema: Unity Catalog schema for the pipeline output. + notebooks: Notebook paths that make up the pipeline. + """ + + resource_key: str + name: str + catalog: str | None = None + schema: str | None = None + notebooks: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Variables / secrets / setup +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class DabVariable: + """Bundle variable requiring user input at deploy time. + + Attributes: + description: Human-readable description shown during ``databricks bundle deploy``. + default: Default value, or ``None`` if the variable is required. + """ + + description: str + default: str | None = None + + +@dataclass(slots=True, kw_only=True) +class SecretInstruction: + """Instruction for creating a Databricks secret. + + Attributes: + scope: Secret scope name. + key: Secret key within the scope. + value_source: Description of where the value should come from + (e.g. ``"Azure Key Vault: my-kv/secret-name"``). + """ + + scope: str + key: str + value_source: str + + +@dataclass(slots=True, kw_only=True) +class SetupTask: + """One-time setup task to run before deployment. + + Attributes: + type: Task category (``"volume"``, ``"secret"``, or ``"connection"``). + config: Configuration dictionary specific to the task type. + """ + + type: str # "volume", "secret", "connection" + config: dict[str, Any] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Top-level bundle +# --------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class DabBundle: + """Complete DAB bundle ready for serialization. + + Attributes: + name: Bundle name (used as the project directory name). + jobs: Workflow job definitions. + notebooks: Generated notebook files. + pipelines: Lakeflow Declarative Pipeline definitions. + setup_notebooks: Notebooks for one-time setup tasks. + variables: Bundle variables requiring user input at deploy time. + """ + + name: str + jobs: list[DabJob] = field(default_factory=list) + notebooks: list[DabNotebook] = field(default_factory=list) + pipelines: list[DabPipeline] = field(default_factory=list) + setup_notebooks: list[DabNotebook] = field(default_factory=list) + variables: dict[str, DabVariable] = field(default_factory=dict) diff --git a/src/orchestra/models/ir.py b/src/orchestra/models/ir.py new file mode 100644 index 0000000..c4e1cf4 --- /dev/null +++ b/src/orchestra/models/ir.py @@ -0,0 +1,560 @@ +"""Translation IR -- intermediate representation after translation.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Any, TypeAlias + + +@dataclass(slots=True, kw_only=True) +class ExpressionResult: + """Result of resolving an ADF expression.""" + + kind: str # "literal", "dab_ref", "notebook_code" + value: str + imports: list[str] = field(default_factory=list) + required_parameters: dict[str, str] = field(default_factory=dict) + + +@dataclass(slots=True, kw_only=True) +class Dependency: + """Dependency on an upstream task. + + Attributes: + task_key: Task key of the upstream activity. + outcome: Required outcome (e.g. ``"Succeeded"``) for this edge. + """ + + task_key: str + outcome: str | None = None + + +@dataclass(slots=True, kw_only=True) +class Activity: + """Base class for all translated pipeline activities. + + Attributes: + name: Logical activity name from ADF. + task_key: Unique task key within the workflow. + description: Human-readable description. + timeout_seconds: Maximum execution time in seconds. + max_retries: Retry limit on failure. + min_retry_interval_millis: Minimum delay between retries (ms). + depends_on: Upstream task dependencies. + cluster: Cluster configuration for the task, if any. + """ + + name: str + task_key: str + description: str | None = None + timeout_seconds: int | None = None + max_retries: int | None = None + min_retry_interval_millis: int | None = None + depends_on: list[Dependency] | None = None + cluster: dict[str, Any] | None = None + # Widget-name → DAB-ref mapping for every `dbutils.widgets.get()` call + # that shows up in any notebook_code the translator produced for this + # activity. Preparers thread these into ``base_parameters`` so DAB + # resolves the refs at job runtime. + required_parameters: dict[str, str] = field(default_factory=dict) + + +@dataclass(slots=True, kw_only=True) +class NotebookActivity(Activity): + """Databricks notebook activity. + + Attributes: + notebook_path: Workspace path to the notebook. + base_parameters: Parameters passed to the notebook at runtime. + linked_service_definition: Raw linked-service dictionary for cluster config. + """ + + notebook_path: str + base_parameters: dict[str, str] | None = None + linked_service_definition: dict[str, Any] | None = None + + +@dataclass(slots=True, kw_only=True) +class CopyActivity(Activity): + """Copy data activity. + + Attributes: + source_type: Source dataset type string. + sink_type: Sink dataset type string. + source_properties: Parsed source format/connection options. + sink_properties: Parsed sink format/connection options. + sink_dataset_type: ADF dataset ``type`` of the sink (e.g. ``DelimitedText``, + ``Parquet``, ``Json``, ``AzureSqlTable``). Captured from the activity's + output dataset so the code generator can write to the actual target + format instead of always defaulting to Delta. + sink_format: Spark format string derived from ``sink_dataset_type`` + (``csv``, ``parquet``, ``json``, ``delta``, ...). ``None`` if the + target is a table, not a file. + sink_resolved_path: Resolved abfss:// or table location for the sink, + mirroring ``source_properties.resolved_path`` for consistency. + column_mapping: Column-level source-to-sink mappings. + """ + + source_type: str | None = None + sink_type: str | None = None + source_properties: dict[str, Any] | None = None + sink_properties: dict[str, Any] | None = None + sink_dataset_type: str | None = None + sink_format: str | None = None + sink_resolved_path: str | None = None + column_mapping: list[dict[str, str]] | None = None + + +@dataclass(slots=True, kw_only=True) +class ForEachActivity(Activity): + """ForEach loop activity. + + Attributes: + items_expression: ADF expression driving the iteration. + inner_activities: Translated activities executed for each item. + concurrency: Maximum parallel iterations (maps to Databricks + ``for_each_task.concurrency``). + """ + + items_expression: str + inner_activities: list[Activity] = field(default_factory=list) + concurrency: int | None = None + + +@dataclass(slots=True, kw_only=True) +class IfConditionActivity(Activity): + """If condition branching activity. + + Attributes: + op: Comparison operator name. + left: Left-hand operand expression. + right: Right-hand operand expression. + if_true_activities: Activities for the true branch. + if_false_activities: Activities for the false branch. + """ + + op: str + left: str + right: str + if_true_activities: list[Activity] = field(default_factory=list) + if_false_activities: list[Activity] = field(default_factory=list) + + +@dataclass(slots=True, kw_only=True) +class SetVariableActivity(Activity): + """Sets variable activity. + + Attributes: + variable_name: Name of the variable being set. + variable_value: Expression string that evaluates to the value. + value_kind: Kind of the resolved expression ("literal", "dab_ref", "notebook_code"). + notebook_code: Python code for notebook_code kind values. + notebook_imports: Import statements needed for notebook_code. + """ + + variable_name: str + variable_value: str + value_kind: str = "literal" # "literal", "dab_ref", "notebook_code" + notebook_code: str | None = None + notebook_imports: list[str] = field(default_factory=list) + + +@dataclass(slots=True, kw_only=True) +class LookupActivity(Activity): + """Lookup activity. + + Attributes: + source_type: Type of the lookup dataset. + source_properties: Parsed source format/connection options. + first_row_only: When True, only the first row is returned. + source_query: Optional SQL query or stored-procedure call. + """ + + source_type: str | None = None + source_properties: dict[str, Any] | None = None + first_row_only: bool = True + source_query: str | None = None + + +@dataclass(slots=True, kw_only=True) +class WebActivity(Activity): + """Web / HTTP activity. + + Attributes: + url: Target URL. + method: HTTP method (GET, POST, etc.). + body: Request body payload. + headers: HTTP headers. + authentication: Parsed authentication configuration. + disable_cert_validation: Skip TLS verification when True. + http_request_timeout_seconds: Request-level timeout. + """ + + url: str + method: str + body: Any = None + headers: dict[str, str] | None = None + authentication: dict[str, Any] | None = None + disable_cert_validation: bool = False + http_request_timeout_seconds: int | None = None + + +@dataclass(slots=True, kw_only=True) +class DeleteActivity(Activity): + """Deletes files / folders activity. + + Attributes: + dataset_name: Reference name of the target dataset. + folder_path: Folder path to delete within the dataset. + recursive: Remove contents recursively when True. + """ + + dataset_name: str + folder_path: str | None = None + recursive: bool = True + + +@dataclass(slots=True, kw_only=True) +class ExecutePipelineActivity(Activity): + """Execute (nested) pipeline activity. + + Attributes: + pipeline_name: Name of the child pipeline to invoke. + parameters: Parameters passed to the child pipeline. + wait_on_completion: Block until the child pipeline finishes. + """ + + pipeline_name: str + parameters: dict[str, Any] | None = None + wait_on_completion: bool = True + + +@dataclass(slots=True, kw_only=True) +class RunJobActivity(Activity): + """Runs an existing Databricks job. + + Attributes: + job_name: Name of the job to run. + existing_job_id: ID of an existing job, if known. + job_parameters: Parameters passed to the job at runtime. + """ + + job_name: str + existing_job_id: str | None = None + job_parameters: dict[str, Any] | None = None + + +@dataclass(slots=True, kw_only=True) +class SparkJarActivity(Activity): + """Spark JAR activity. + + Attributes: + main_class_name: Fully qualified main class within the JAR. + parameters: Arguments passed to the main class. + libraries: Library descriptors (JARs, wheels, etc.). + """ + + main_class_name: str + parameters: list[str] | None = None + libraries: list[dict[str, Any]] | None = None + + +@dataclass(slots=True, kw_only=True) +class SparkPythonActivity(Activity): + """Spark Python activity. + + Attributes: + python_file: Path to the Python file to execute. + parameters: Arguments passed to the script. + """ + + python_file: str + parameters: list[str] | None = None + + +@dataclass(slots=True, kw_only=True) +class SwitchCase: + """A single case branch within a SwitchActivity. + + Attributes: + value: The literal value to compare against the switch expression. + activities: Activities to execute when this case matches. + """ + + value: str + activities: list[Activity] = field(default_factory=list) + + +@dataclass(slots=True, kw_only=True) +class SwitchActivity(Activity): + """Switch (multi-branch) activity. + + Attributes: + on_expression: The ADF expression to evaluate. + cases: Ordered list of case branches. + default_activities: Activities to run when no case matches. + """ + + on_expression: str + cases: list[SwitchCase] = field(default_factory=list) + default_activities: list[Activity] = field(default_factory=list) + + +@dataclass(slots=True, kw_only=True) +class WaitActivity(Activity): + """Wait / sleep activity. + + Attributes: + wait_time_seconds: Duration to wait in seconds. + """ + + wait_time_seconds: int + + +@dataclass(slots=True, kw_only=True) +class FilterActivity(Activity): + """Filters activity. + + Attributes: + items_expression: ADF expression for the input array. + condition_expression: Original ADF condition expression (preserved + for documentation; never executed at runtime). + condition_code: Python expression that evaluates to a bool against + a per-iteration ``item`` dict. ``None`` when the translator + could not safely pre-resolve the condition; the code generator + emits a TODO placeholder notebook in that case. + condition_imports: Imports the ``condition_code`` expression + requires (e.g. ``datetime``). + """ + + items_expression: str + condition_expression: str + condition_code: str | None = None + condition_imports: list[str] = field(default_factory=list) + + +@dataclass(slots=True, kw_only=True) +class AppendVariableActivity(Activity): + """Appends variable activity. + + Attributes: + variable_name: Name of the array variable. + append_value: Expression string that evaluates to the value to append. + value_kind: Kind of the resolved expression ("literal", "dab_ref", "notebook_code"). + notebook_code: Python code for notebook_code kind values. + notebook_imports: Import statements needed for notebook_code. + """ + + variable_name: str + append_value: str + value_kind: str = "literal" # "literal", "dab_ref", "notebook_code" + notebook_code: str | None = None + notebook_imports: list[str] = field(default_factory=list) + + +@dataclass(slots=True, kw_only=True) +class UnsupportedActivity(Activity): + """Sentinel for activities that could not be translated. + + Attributes: + original_type: The ADF activity type that was not supported. + reason: Human-readable explanation of why translation failed. + """ + + original_type: str + reason: str | None = None + + +@dataclass(slots=True, kw_only=True) +class PlaceholderActivity(Activity): + """Placeholder notebook for activities that require manual intervention. + + Attributes: + original_type: The ADF activity type being replaced. + notebook_path: Workspace path to the placeholder notebook. + comment: Guidance for the user on what to implement. + """ + + original_type: str + notebook_path: str = "/UNSUPPORTED_ADF_ACTIVITY" + comment: str | None = None + + +@dataclass(slots=True, kw_only=True) +class MotifActivity(Activity): + """Activity produced by collapsing a detected motif pattern. + + Attributes: + motif_id: Identifier of the matched motif definition. + display_name: Human-readable motif name. + databricks_replacement: Target Databricks construct + (e.g. ``"auto_loader"``, ``"dlt_apply_changes"``). + matched_activity_names: Original ADF activity names that were collapsed. + source_type_hint: Inferred source type (``"files"``, ``"database"``, + ``"rest_api"``) or ``None``. + confidence_notes: Detector notes explaining the match rationale. + original_activities: The original translated Activity IR nodes that + were replaced, preserved for reference and fallback. + notebook_template: Name of the code generator template, if any. + """ + + motif_id: str + display_name: str + databricks_replacement: str + matched_activity_names: list[str] + source_type_hint: str | None = None + confidence_notes: list[str] = field(default_factory=list) + original_activities: list[Activity] = field(default_factory=list) + notebook_template: str | None = None + # Small dict of motif-specific settings extracted from the collapsed + # activities — e.g. ``{"lookup_query": ..., "lookup_scope": ...}`` for + # ``for_each_ingestion``. Used by the notebook generator so the motif + # can fetch its input list itself instead of requiring an ``items`` + # widget that has no upstream writer. + motif_config: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True, kw_only=True) +class Pipeline: + """Top-level workflow container produced by the translator. + + Attributes: + name: Logical pipeline name. + parameters: Pipeline parameter definitions. + schedule: Serialized schedule definition, if any. + tasks: Ordered list of translated activities. + tags: System and user-defined tags. + not_translatable: Entries describing properties that could not be translated. + """ + + name: str + parameters: list[dict[str, Any]] | None = None + schedule: dict[str, Any] | None = None + tasks: list[Activity] = field(default_factory=list) + tags: dict[str, str] = field(default_factory=dict) + not_translatable: list[dict[str, Any]] = field(default_factory=list) + + +@dataclass(frozen=True, slots=True) +class TranslationContext: + """Immutable snapshot of translation state threaded through each visitor call. + + Attributes: + activity_cache: Read-only mapping of activity names to translated activities. + registry: Read-only mapping of activity type strings to translator callables. + variable_cache: Read-only mapping of variable names to the task keys + of the tasks that set them. + """ + + activity_cache: MappingProxyType[str, Activity] = field(default_factory=lambda: MappingProxyType({})) + registry: MappingProxyType[str, Any] = field(default_factory=lambda: MappingProxyType({})) + variable_cache: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) + variable_value_cache: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) + + def with_activity(self, name: str, activity: Activity) -> TranslationContext: + """Return a new context with *activity* added to the cache. + + Args: + name: Activity name used as the cache key. + activity: Translated activity to store. + + Returns: + New ``TranslationContext`` containing the updated activity cache. + """ + return TranslationContext( + activity_cache=MappingProxyType({**self.activity_cache, name: activity}), + registry=self.registry, + variable_cache=self.variable_cache, + variable_value_cache=self.variable_value_cache, + ) + + def get_activity(self, activity_name: str) -> Activity | None: + """Look up a previously translated activity by name. + + Args: + activity_name: Activity name. + + Returns: + Cached ``Activity`` or ``None`` if not yet visited. + """ + return self.activity_cache.get(activity_name) + + def with_variable( + self, + variable_name: str, + task_key: str, + *, + dab_ref_value: str | None = None, + ) -> TranslationContext: + """Return a new context with a variable mapping added. + + Args: + variable_name: Variable name. + task_key: Task key of the task that sets this variable. + dab_ref_value: When the variable's value resolves to a DAB + dynamic value reference (e.g. ``{{job.start_time.iso_datetime}}``), + store it so downstream ``@variables()`` calls can inline the + ref instead of routing through the task value. + + Returns: + New ``TranslationContext`` containing the updated caches. + """ + new_variable_value_cache = self.variable_value_cache + if dab_ref_value is not None: + new_variable_value_cache = MappingProxyType({**self.variable_value_cache, variable_name: dab_ref_value}) + return TranslationContext( + activity_cache=self.activity_cache, + registry=self.registry, + variable_cache=MappingProxyType({**self.variable_cache, variable_name: task_key}), + variable_value_cache=new_variable_value_cache, + ) + + def get_variable_task_key(self, variable_name: str) -> str | None: + """Look up the task key that sets a variable.""" + return self.variable_cache.get(variable_name) + + def get_variable_dab_ref(self, variable_name: str) -> str | None: + """Look up the inlined DAB ref value for a variable, if available.""" + return self.variable_value_cache.get(variable_name) + + +TranslationResult: TypeAlias = Activity | UnsupportedActivity + + +@dataclass(slots=True, kw_only=True) +class AgenticGap: + """Describes an activity that requires agentic translation. + + Attributes: + activity_name: Display name of the activity. + activity_type: ADF activity type string. + recommended_skill: Skill identifier to use for translation. + raw_definition: Original ADF JSON definition for the activity. + """ + + activity_name: str + activity_type: str + recommended_skill: str | None = None + raw_definition: dict[str, Any] | None = None + + +@dataclass(slots=True, kw_only=True) +class TranslationReport: + """Summary produced after translating an entire ADF pipeline. + + Attributes: + pipeline: The translated pipeline IR. + deterministic_count: Activities translated deterministically. + agentic_count: Activities requiring agentic translation. + unsupported_count: Activities that could not be translated. + gaps: List of agentic gaps identified during translation. + warnings: Human-readable warning messages emitted during translation. + """ + + pipeline: Pipeline + deterministic_count: int = 0 + agentic_count: int = 0 + unsupported_count: int = 0 + gaps: list[AgenticGap] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) diff --git a/src/orchestra/models/motifs.py b/src/orchestra/models/motifs.py new file mode 100644 index 0000000..2e998b8 --- /dev/null +++ b/src/orchestra/models/motifs.py @@ -0,0 +1,196 @@ +"""Motif definitions for common ADF pipeline patterns.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(frozen=True, slots=True) +class MotifDefinition: + """Immutable definition of a recognised ADF motif pattern. + + Attributes: + motif_id: Unique short identifier (e.g. ``"incremental_load_watermark"``). + display_name: Human-readable name shown in reports. + description: Paragraph explaining the ADF pattern and its Databricks equivalent. + expected_activity_types: ADF activity types that participate in this motif. + databricks_replacement: Short label for the target Databricks construct + (e.g. ``"auto_loader"``, ``"dlt_pipeline"``, ``"structured_streaming"``). + notebook_template: Name of the Jinja2 or code-generator template used to + produce the replacement notebook. ``None`` when no template exists yet. + """ + + motif_id: str + display_name: str + description: str + expected_activity_types: tuple[str, ...] + databricks_replacement: str + notebook_template: str | None = None + + +@dataclass(slots=True, kw_only=True) +class DetectedMotif: + """Result of matching a motif in a specific pipeline. + + Attributes: + definition: The motif definition that was matched. + matched_activities: Activity names claimed by this match. + source_type_hint: Inferred source category (``"files"``, ``"database"``, + ``"rest_api"``) or ``None``. + confidence_notes: Human-readable notes explaining *why* the detector + concluded this motif matches (useful for review). + """ + + definition: MotifDefinition + matched_activities: list[str] + source_type_hint: str | None = None + confidence_notes: list[str] = field(default_factory=list) + + +MOTIF_INCREMENTAL_LOAD_WATERMARK = MotifDefinition( + motif_id="incremental_load_watermark", + display_name="Incremental Load (Watermark)", + description=( + "Two Lookup activities fetch the old and new watermark values, a Copy " + "activity loads the delta between them, and a StoredProcedure updates " + "the watermark table. Translates to Auto Loader with checkpoint-based " + "incremental ingestion or a Spark Structured Streaming job." + ), + expected_activity_types=("Lookup", "Copy", "SqlServerStoredProcedure"), + databricks_replacement="auto_loader", + notebook_template="incremental_watermark.py", +) + +MOTIF_CDC_CHANGE_TRACKING = MotifDefinition( + motif_id="cdc_change_tracking", + display_name="CDC (SQL Server Change Tracking)", + description=( + "Similar to watermark but relies on SQL Server Change Tracking " + "(CHANGETABLE / SYS_CHANGE_VERSION). Translates to a Spark " + "Structured Streaming job reading the change feed or a DLT " + "pipeline with APPLY CHANGES." + ), + expected_activity_types=("Lookup", "Copy", "SqlServerStoredProcedure"), + databricks_replacement="dlt_apply_changes", + notebook_template="cdc_change_tracking.py", +) + +MOTIF_METADATA_DRIVEN_BULK_COPY = MotifDefinition( + motif_id="metadata_driven_bulk_copy", + display_name="Metadata-Driven Bulk Copy", + description=( + "A Lookup reads a control/metadata table listing source tables, then " + "a ForEach iterates over the list and copies each table. Translates " + "to a parameterised Databricks job with a for_each_task or a DLT " + "pipeline ingesting multiple sources." + ), + expected_activity_types=("Lookup", "ForEach", "Copy"), + databricks_replacement="for_each_ingestion", + notebook_template="metadata_bulk_copy.py", +) + +MOTIF_FILE_LANDING_ZONE_PROCESSING = MotifDefinition( + motif_id="file_landing_zone_processing", + display_name="File Landing Zone Processing", + description=( + "GetMetadata lists files in a landing zone, an optional Filter narrows " + "the list, ForEach iterates and copies each file, and Delete cleans up " + "processed files. Translates to Auto Loader with file notification " + "triggers." + ), + expected_activity_types=("GetMetadata", "Filter", "ForEach", "Copy", "Delete"), + databricks_replacement="auto_loader_file_notification", + notebook_template="file_landing_zone.py", +) + +MOTIF_REST_API_PAGINATION = MotifDefinition( + motif_id="rest_api_pagination", + display_name="REST API Pagination", + description=( + "A WebActivity fetches an authentication token, a SetVariable " + "initialises a pagination cursor, and an Until loop fetches pages " + "via Copy or WebActivity until exhausted. Translates to a Python " + "notebook with requests-based pagination." + ), + expected_activity_types=("WebActivity", "SetVariable", "Until", "Copy"), + databricks_replacement="python_rest_ingestion", + notebook_template="rest_api_pagination.py", +) + +MOTIF_PARENT_CHILD_ORCHESTRATION = MotifDefinition( + motif_id="parent_child_orchestration", + display_name="Parent-Child Orchestration", + description=( + "A Lookup provides a list of work items and a ForEach iterates, " + "calling ExecutePipeline for each. Translates to a Databricks " + "for_each_task with run_job_task calling the child job." + ), + expected_activity_types=("Lookup", "ForEach", "ExecutePipeline"), + databricks_replacement="for_each_run_job", + notebook_template=None, +) + +MOTIF_FILE_EXISTENCE_VALIDATION = MotifDefinition( + motif_id="file_existence_validation", + display_name="File Existence Validation", + description=( + "GetMetadata checks whether a file or folder exists, and an " + "IfCondition gates downstream logic on the result. Translates " + "to a condition_task checking a file-existence notebook." + ), + expected_activity_types=("GetMetadata", "IfCondition"), + databricks_replacement="condition_task", + notebook_template="file_existence_check.py", +) + +MOTIF_SCD_TYPE_2 = MotifDefinition( + motif_id="scd_type_2", + display_name="SCD Type 2", + description=( + "A Copy loads data into a staging table and an ExecuteDataFlow " + "applies SCD Type 2 merge logic (Lookup + AlterRow + Union). " + "Translates to a DLT pipeline with APPLY CHANGES INTO." + ), + expected_activity_types=("Copy", "ExecuteDataFlow"), + databricks_replacement="dlt_apply_changes", + notebook_template="scd_type_2.py", +) + +MOTIF_STAGED_LOAD_SYNAPSE = MotifDefinition( + motif_id="staged_load_synapse", + display_name="Staged Load (Synapse)", + description=( + "A Copy activity loads data via PolyBase/COPY command staging, " + "followed by a StoredProcedure for post-load transforms. " + "Translates to a direct Spark write to Delta with post-processing." + ), + expected_activity_types=("Copy", "SqlServerStoredProcedure"), + databricks_replacement="spark_delta_write", + notebook_template="staged_load.py", +) + +MOTIF_COPY_AND_NOTIFY = MotifDefinition( + motif_id="copy_and_notify", + display_name="Copy and Notify", + description=( + "A Copy activity followed by WebActivity calls for success/failure " + "notifications (Logic Apps, Slack, email). Translates to a notebook " + "task with built-in notification via job email/webhook settings." + ), + expected_activity_types=("Copy", "WebActivity"), + databricks_replacement="notebook_with_notification", + notebook_template="copy_and_notify.py", +) + +ALL_MOTIFS: tuple[MotifDefinition, ...] = ( + MOTIF_INCREMENTAL_LOAD_WATERMARK, + MOTIF_CDC_CHANGE_TRACKING, + MOTIF_METADATA_DRIVEN_BULK_COPY, + MOTIF_FILE_LANDING_ZONE_PROCESSING, + MOTIF_REST_API_PAGINATION, + MOTIF_PARENT_CHILD_ORCHESTRATION, + MOTIF_FILE_EXISTENCE_VALIDATION, + MOTIF_SCD_TYPE_2, + MOTIF_STAGED_LOAD_SYNAPSE, + MOTIF_COPY_AND_NOTIFY, +) diff --git a/src/orchestra/models/source_types.py b/src/orchestra/models/source_types.py new file mode 100644 index 0000000..8d815c1 --- /dev/null +++ b/src/orchestra/models/source_types.py @@ -0,0 +1,50 @@ +"""Canonical source-type taxonomy used across the translator and preparer.""" + +from __future__ import annotations + +# Database-style sources reachable via JDBC. Every entry here implies the +# generated notebook will read with ``spark.read.format("jdbc")`` and +# require ``jdbc-url`` / ``jdbc-password`` (and optionally ``jdbc-user``) +# secrets. +JDBC_SOURCE_TYPES: frozenset[str] = frozenset( + { + "AzureSqlSource", + "AzureSqlDatabaseSource", + "SqlServerSource", + "OracleSource", + "PostgreSqlSource", + "MySqlSource", + "SqlSource", + "CosmosDbSqlApiSource", + "SqlDWSource", + } +) + + +# File-based sources that resolve to an object store location. These +# trigger UC volume / external-location provisioning and use Auto Loader +# (``cloudFiles``) for ingestion. +FILE_SOURCE_TYPES: frozenset[str] = frozenset( + { + "BlobSource", + "AzureBlobFSSource", + "AzureBlobStorageSource", + "AzureDataLakeStoreSource", + "AmazonS3Source", + "FileSystemSource", + "SftpSource", + "HttpSource", + "DelimitedTextSource", + "JsonSource", + "ParquetSource", + "AvroSource", + "OrcSource", + } +) + + +# Paginated REST API sources -- handled by a generic ``requests``-based +# pagination loop in the generated copy notebook. ADF ``HttpSource`` +# is *not* in this set: it downloads a single file (CSV / JSON / +# Parquet) over HTTP and is handled as a FILE source via Auto Loader. +REST_SOURCE_TYPES: frozenset[str] = frozenset({"RestSource"}) diff --git a/src/orchestra/motifs/__init__.py b/src/orchestra/motifs/__init__.py new file mode 100644 index 0000000..d9831b5 --- /dev/null +++ b/src/orchestra/motifs/__init__.py @@ -0,0 +1,6 @@ +"""Motif detection and collapsing for ADF pipeline patterns.""" + +from flowx.motifs.collapser import collapse_motifs +from flowx.motifs.detector import detect_motifs + +__all__ = ["detect_motifs", "collapse_motifs"] diff --git a/src/orchestra/motifs/collapser.py b/src/orchestra/motifs/collapser.py new file mode 100644 index 0000000..e532e01 --- /dev/null +++ b/src/orchestra/motifs/collapser.py @@ -0,0 +1,202 @@ +"""Collapse detected motifs into MotifActivity IR nodes.""" + +from __future__ import annotations + +import logging +from typing import Any + +from flowx.models.ir import Activity, CopyActivity, Dependency, LookupActivity, MotifActivity, Pipeline +from flowx.models.motifs import DetectedMotif + +logger = logging.getLogger(__name__) + + +def collapse_motifs( + pipeline: Pipeline, + motifs: list[DetectedMotif], +) -> Pipeline: + """Replaces matched activity groups with MotifActivity nodes. + + Args: + pipeline: The translated pipeline IR. + motifs: Detected motif matches from the detector. + + Returns: + A new Pipeline with motif activities collapsed. The original + pipeline is not mutated. + """ + if not motifs: + return pipeline + + claimed_names: set[str] = set() + for motif in motifs: + claimed_names.update(motif.matched_activities) + + tasks_by_name: dict[str, Activity] = {task.name: task for task in pipeline.tasks} + new_tasks: list[Activity] = [] + # Maps a *sanitised* task_key of a collapsed activity to the + # MotifActivity's task_key so ``_rewire_dependencies`` can match + # against ``Dependency.task_key`` (which is also sanitised). Keying + # by raw activity name here would silently fail to rewire any edge + # whose source had spaces or other characters in its name. + motif_task_keys: dict[str, str] = {} + + inserted_motifs: set[str] = set() + for task in pipeline.tasks: + if task.name in claimed_names: + detected = _find_motif_for_activity(task.name, motifs) + if detected is None: + new_tasks.append(task) + continue + + motif_id = detected.definition.motif_id + if motif_id in inserted_motifs: + continue + inserted_motifs.add(motif_id) + + motif_activity = _build_motif_activity(detected, tasks_by_name) + new_tasks.append(motif_activity) + + for matched_name in detected.matched_activities: + matched_task = tasks_by_name.get(matched_name) + if matched_task is not None: + motif_task_keys[matched_task.task_key] = motif_activity.task_key + else: + new_tasks.append(task) + + _rewire_dependencies(new_tasks, motif_task_keys) + + return Pipeline( + name=pipeline.name, + parameters=pipeline.parameters, + schedule=pipeline.schedule, + tasks=new_tasks, + tags=pipeline.tags, + not_translatable=pipeline.not_translatable, + ) + + +def _find_motif_for_activity( + activity_name: str, + motifs: list[DetectedMotif], +) -> DetectedMotif | None: + """Finds the motif that claimed a given activity.""" + for motif in motifs: + if activity_name in motif.matched_activities: + return motif + return None + + +def _build_motif_activity( + motif: DetectedMotif, + tasks_by_name: dict[str, Activity], +) -> MotifActivity: + """Builds a MotifActivity from a detected motif and the original tasks.""" + definition = motif.definition + + task_key = f"motif_{definition.motif_id}" + display_name = definition.display_name + + original_activities = [tasks_by_name[name] for name in motif.matched_activities if name in tasks_by_name] + + # Use the sanitised task_keys (not raw activity names) for the + # internal-dependency check; ``Dependency.task_key`` is sanitised by + # the translator, so comparing against raw names would mis-classify + # any internal dep whose source name contained spaces / hyphens. + matched_task_keys = {activity.task_key for activity in original_activities} + external_deps = _collect_external_dependencies(original_activities, matched_task_keys) + + return MotifActivity( + name=display_name, + task_key=task_key, + description=( + f"Collapsed motif: {definition.display_name}. " + f"Replaces {len(motif.matched_activities)} ADF activities with " + f"{definition.databricks_replacement}." + ), + depends_on=external_deps, + motif_id=definition.motif_id, + display_name=display_name, + databricks_replacement=definition.databricks_replacement, + matched_activity_names=list(motif.matched_activities), + source_type_hint=motif.source_type_hint, + confidence_notes=list(motif.confidence_notes), + original_activities=original_activities, + notebook_template=definition.notebook_template, + motif_config=_build_motif_config(definition.databricks_replacement, original_activities, task_key), + ) + + +def _build_motif_config( + databricks_replacement: str, + original_activities: list[Activity], + motif_task_key: str, +) -> dict[str, Any]: + """Extracts motif-specific settings from the activities being collapsed.""" + if databricks_replacement != "for_each_ingestion": + return {} + + lookup = next((activity for activity in original_activities if isinstance(activity, LookupActivity)), None) + copy = next((activity for activity in original_activities if isinstance(activity, CopyActivity)), None) + + config: dict[str, Any] = {} + if lookup is not None: + if lookup.source_query: + config["lookup_query"] = lookup.source_query + if lookup.source_type: + config["lookup_source_type"] = lookup.source_type + config["lookup_scope"] = lookup.task_key or motif_task_key + if copy is not None: + sink_properties = copy.sink_properties or {} + sink_table = sink_properties.get("table") or sink_properties.get("tableName") + if sink_table: + config["sink_table"] = sink_table + if copy.source_type: + config["copy_source_type"] = copy.source_type + config["copy_scope"] = copy.task_key or motif_task_key + return config + + +def _collect_external_dependencies( + activities: list[Activity], + matched_task_keys: set[str], +) -> list[Dependency]: + """Collects dependencies that point outside the matched activity group. + + *matched_task_keys* must be the sanitised task_keys of the matched + activities -- ``Dependency.task_key`` is sanitised by the translator, + so comparing against raw activity names produces silent false + positives whenever a name contains spaces or other characters that + are stripped during sanitisation. + """ + seen: set[str] = set() + external_deps: list[Dependency] = [] + + for activity in activities: + if not activity.depends_on: + continue + for dep in activity.depends_on: + if dep.task_key not in matched_task_keys and dep.task_key not in seen: + seen.add(dep.task_key) + external_deps.append(Dependency(task_key=dep.task_key, outcome=dep.outcome)) + + return external_deps + + +def _rewire_dependencies( + tasks: list[Activity], + motif_task_keys: dict[str, str], +) -> None: + """Rewire dependencies so activities that depended on collapsed activities""" + for task in tasks: + if not task.depends_on: + continue + new_deps: list[Dependency] = [] + seen_keys: set[str] = set() + for dep in task.depends_on: + replacement_key = motif_task_keys.get(dep.task_key) + effective_key = replacement_key if replacement_key else dep.task_key + if effective_key not in seen_keys: + seen_keys.add(effective_key) + new_deps.append(Dependency(task_key=effective_key, outcome=dep.outcome)) + task.depends_on = new_deps diff --git a/src/orchestra/motifs/detector.py b/src/orchestra/motifs/detector.py new file mode 100644 index 0000000..2579501 --- /dev/null +++ b/src/orchestra/motifs/detector.py @@ -0,0 +1,840 @@ +"""Heuristic-based motif detection for ADF pipelines.""" + +from __future__ import annotations + +import logging +from typing import Callable + +from flowx.models.adf_ast import ( + AdfActivity, + AdfDefinitions, + AdfPipeline, +) +from flowx.models.motifs import ( + MOTIF_CDC_CHANGE_TRACKING, + MOTIF_COPY_AND_NOTIFY, + MOTIF_FILE_EXISTENCE_VALIDATION, + MOTIF_FILE_LANDING_ZONE_PROCESSING, + MOTIF_INCREMENTAL_LOAD_WATERMARK, + MOTIF_METADATA_DRIVEN_BULK_COPY, + MOTIF_PARENT_CHILD_ORCHESTRATION, + MOTIF_REST_API_PAGINATION, + MOTIF_SCD_TYPE_2, + MOTIF_STAGED_LOAD_SYNAPSE, + DetectedMotif, + MotifDefinition, +) + +logger = logging.getLogger(__name__) + +_FILE_LS_TYPES: set[str] = { + "AzureBlobStorage", + "AzureBlobFS", + "AzureDataLakeStore", + "AzureDataLakeStoreGen2", + "AmazonS3", + "GoogleCloudStorage", + "FileServer", + "FtpServer", + "Sftp", + "HttpServer", +} + +_DATABASE_LS_TYPES: set[str] = { + "AzureSqlDatabase", + "AzureSqlDW", + "AzureSqlMI", + "SqlServer", + "AzureMySql", + "AzurePostgreSql", + "Oracle", + "Db2", + "Teradata", + "Snowflake", + "AmazonRedshift", + "GoogleBigQuery", + "AzureCosmosDb", + "AzureTableStorage", + "MongoDb", + "MongoDbAtlas", + "DynamoDB", +} + +_REST_LS_TYPES: set[str] = { + "RestService", + "HttpServer", + "OData", + "SharePointOnlineList", +} + +_Detector = Callable[ + [list[AdfActivity], dict[str, AdfActivity], AdfDefinitions, set[str]], + list[DetectedMotif], +] + + +def detect_motifs( + pipeline: AdfPipeline, + definitions: AdfDefinitions, +) -> list[DetectedMotif]: + """Scans *pipeline* for known multi-activity motifs. + + Args: + pipeline: Parsed ADF pipeline AST. + definitions: Full ADF definitions for cross-referencing datasets and + linked services. + + Returns: + List of :class:`DetectedMotif` instances, one per matched pattern. + """ + activities = pipeline.activities + if not activities: + return [] + + by_name: dict[str, AdfActivity] = {activity.name: activity for activity in activities} + claimed: set[str] = set() + results: list[DetectedMotif] = [] + + _detectors: list[tuple[MotifDefinition, _Detector]] = [ + (MOTIF_INCREMENTAL_LOAD_WATERMARK, _detect_incremental_watermark), + (MOTIF_CDC_CHANGE_TRACKING, _detect_cdc_change_tracking), + (MOTIF_METADATA_DRIVEN_BULK_COPY, _detect_metadata_driven_bulk_copy), + (MOTIF_FILE_LANDING_ZONE_PROCESSING, _detect_file_landing_zone), + (MOTIF_REST_API_PAGINATION, _detect_rest_api_pagination), + (MOTIF_PARENT_CHILD_ORCHESTRATION, _detect_parent_child_orchestration), + (MOTIF_FILE_EXISTENCE_VALIDATION, _detect_file_existence_validation), + (MOTIF_SCD_TYPE_2, _detect_scd_type_2), + (MOTIF_STAGED_LOAD_SYNAPSE, _detect_staged_load_synapse), + (MOTIF_COPY_AND_NOTIFY, _detect_copy_and_notify), + ] + + for motif_def, detector_fn in _detectors: + matches = detector_fn(activities, by_name, definitions, claimed) + for match in matches: + claimed.update(match.matched_activities) + results.append(match) + logger.info( + "Detected motif '%s' in pipeline '%s': activities=%s", + motif_def.motif_id, + pipeline.name, + match.matched_activities, + ) + + return results + + +def _get_upstream_names(activity: AdfActivity) -> list[str]: + """Return names of upstream dependencies for *activity*.""" + if not activity.depends_on: + return [] + return [dep.activity for dep in activity.depends_on] + + +def _depends_on( + downstream: AdfActivity, + upstream_name: str, +) -> bool: + """Return True if *downstream* directly depends on *upstream_name*.""" + return upstream_name in _get_upstream_names(downstream) + + +def _type_props_text(activity: AdfActivity) -> str: + """Flattens type_properties to a lowercase string for keyword searches.""" + if not activity.type_properties: + return "" + return str(activity.type_properties).lower() + + +def _infer_source_type( + activity: AdfActivity, + definitions: AdfDefinitions, +) -> str | None: + """Infer whether the source of a Copy/Lookup activity is files, database, or REST.""" + if activity.inputs: + for input_ref in activity.inputs: + dataset = definitions.datasets.get(input_ref.reference_name) + if dataset and dataset.linked_service_name: + linked_service = definitions.linked_services.get(dataset.linked_service_name) + if linked_service: + if linked_service.type in _FILE_LS_TYPES: + return "files" + if linked_service.type in _DATABASE_LS_TYPES: + return "database" + if linked_service.type in _REST_LS_TYPES: + return "rest_api" + + if activity.linked_service_name: + linked_service = definitions.linked_services.get(activity.linked_service_name.reference_name) + if linked_service: + if linked_service.type in _FILE_LS_TYPES: + return "files" + if linked_service.type in _DATABASE_LS_TYPES: + return "database" + if linked_service.type in _REST_LS_TYPES: + return "rest_api" + + type_properties = activity.type_properties or {} + source = type_properties.get("source", {}) + if isinstance(source, dict): + source_type = source.get("type", "") + if any( + keyword in source_type.lower() + for keyword in ("blob", "s3", "datalake", "file", "parquet", "csv", "json", "avro", "orc") + ): + return "files" + if any( + keyword in source_type.lower() + for keyword in ("sql", "oracle", "db2", "mysql", "postgre", "snowflake", "redshift", "cosmos") + ): + return "database" + if any(keyword in source_type.lower() for keyword in ("rest", "http", "odata")): + return "rest_api" + + return None + + +def _activities_of_type( + activities: list[AdfActivity], + adf_type: str, + claimed: set[str], +) -> list[AdfActivity]: + """Return unclaimed activities matching *adf_type*.""" + return [activity for activity in activities if activity.type == adf_type and activity.name not in claimed] + + +def _record_motif( + results: list[DetectedMotif], + *, + definition: MotifDefinition, + matched_activities: list[str], + source_type_hint: str | None, + confidence_notes: list[str], +) -> None: + """Appends a fully-populated :class:`DetectedMotif` to *results*.""" + results.append( + DetectedMotif( + definition=definition, + matched_activities=matched_activities, + source_type_hint=source_type_hint, + confidence_notes=confidence_notes, + ) + ) + + +def _has_keyword(text: str, *keywords: str) -> bool: + """Case-insensitive keyword check in *text*.""" + lower = text.lower() + return any(keyword.lower() in lower for keyword in keywords) + + +def _detect_incremental_watermark( + activities: list[AdfActivity], + by_name: dict[str, AdfActivity], + definitions: AdfDefinitions, + claimed: set[str], +) -> list[DetectedMotif]: + """Detects incremental-load-watermark pattern.""" + results: list[DetectedMotif] = [] + copies = _activities_of_type(activities, "Copy", claimed) + + for copy_act in copies: + upstream_lookups: list[AdfActivity] = [] + for name in _get_upstream_names(copy_act): + activity = by_name.get(name) + if activity and activity.type == "Lookup" and activity.name not in claimed: + upstream_lookups.append(activity) + + if len(upstream_lookups) < 2: + continue + + watermark_keywords_found = False + has_change_tracking = False + notes: list[str] = [] + + for lookup_activity in upstream_lookups: + type_properties_text = _type_props_text(lookup_activity) + if _has_keyword(type_properties_text, "watermark", "max(", "min(", "last_modified", "lastmodified"): + watermark_keywords_found = True + notes.append(f"Lookup '{lookup_activity.name}' contains watermark-style query") + if _has_keyword(type_properties_text, "change_tracking", "changetable", "sys_change_version"): + has_change_tracking = True + + if has_change_tracking: + continue + + if not watermark_keywords_found: + continue + + downstream_sp: AdfActivity | None = None + for activity in activities: + if activity.type == "SqlServerStoredProcedure" and activity.name not in claimed: + if _depends_on(activity, copy_act.name): + downstream_sp = activity + break + + if downstream_sp is None: + continue + + matched = [lookup_activity.name for lookup_activity in upstream_lookups] + [copy_act.name, downstream_sp.name] + source_hint = _infer_source_type(copy_act, definitions) + notes.append(f"StoredProcedure '{downstream_sp.name}' updates watermark after Copy") + + _record_motif( + results, + definition=MOTIF_INCREMENTAL_LOAD_WATERMARK, + matched_activities=matched, + source_type_hint=source_hint, + confidence_notes=notes, + ) + + return results + + +def _detect_cdc_change_tracking( + activities: list[AdfActivity], + by_name: dict[str, AdfActivity], + definitions: AdfDefinitions, + claimed: set[str], +) -> list[DetectedMotif]: + """Detects CDC change-tracking pattern.""" + results: list[DetectedMotif] = [] + copies = _activities_of_type(activities, "Copy", claimed) + + for copy_act in copies: + upstream_lookups: list[AdfActivity] = [] + for name in _get_upstream_names(copy_act): + activity = by_name.get(name) + if activity and activity.type == "Lookup" and activity.name not in claimed: + upstream_lookups.append(activity) + + if len(upstream_lookups) < 2: + continue + + # Must have change-tracking keywords + cdc_found = False + notes: list[str] = [] + for lookup_activity in upstream_lookups: + type_properties_text = _type_props_text(lookup_activity) + if _has_keyword(type_properties_text, "change_tracking", "changetable", "sys_change_version"): + cdc_found = True + notes.append(f"Lookup '{lookup_activity.name}' references SQL Server Change Tracking") + + if not cdc_found: + continue + + downstream_sp: AdfActivity | None = None + for activity in activities: + if activity.type == "SqlServerStoredProcedure" and activity.name not in claimed: + if _depends_on(activity, copy_act.name): + downstream_sp = activity + break + + if downstream_sp is None: + continue + + matched = [lookup_activity.name for lookup_activity in upstream_lookups] + [copy_act.name, downstream_sp.name] + source_hint = _infer_source_type(copy_act, definitions) + notes.append(f"StoredProcedure '{downstream_sp.name}' updates change-tracking version") + + _record_motif( + results, + definition=MOTIF_CDC_CHANGE_TRACKING, + matched_activities=matched, + source_type_hint=source_hint or "database", + confidence_notes=notes, + ) + + return results + + +def _detect_metadata_driven_bulk_copy( + activities: list[AdfActivity], + by_name: dict[str, AdfActivity], + definitions: AdfDefinitions, + claimed: set[str], +) -> list[DetectedMotif]: + """Detects metadata-driven bulk copy pattern.""" + results: list[DetectedMotif] = [] + for_each_activities = _activities_of_type(activities, "ForEach", claimed) + + for for_each_activity in for_each_activities: + # Bulk-copy motif requires the inner body to *be* the Copy: a single + # Copy child, with no other transform / orchestration activity in the + # loop body. Patterns like Notebook -> Copy or BuildReport -> Export + # are not bulk-copy motifs even when an upstream Lookup is present; + # they are generic "build then archive" pipelines and the user almost + # never wants the Copy collapsed into a metadata-driven ingestion + # template that ignores the upstream notebook work. + inner_activities = list(for_each_activity.activities or []) + if len(inner_activities) != 1 or inner_activities[0].type != "Copy": + continue + + upstream_lookups: list[AdfActivity] = [] + for name in _get_upstream_names(for_each_activity): + activity = by_name.get(name) + if activity and activity.type == "Lookup" and activity.name not in claimed: + upstream_lookups.append(activity) + + if not upstream_lookups: + continue + + notes: list[str] = [] + for lookup_activity in upstream_lookups: + type_properties_text = _type_props_text(lookup_activity) + if _has_keyword(type_properties_text, "table", "schema", "control", "metadata", "config"): + notes.append(f"Lookup '{lookup_activity.name}' appears to read a control/metadata table") + + # Even without keyword match we detect if the structure is right + if not notes: + notes.append("Lookup -> ForEach -> Copy structure matches bulk copy pattern") + + matched = [lookup_activity.name for lookup_activity in upstream_lookups] + [for_each_activity.name] + source_hint = None + if for_each_activity.activities: + for child in for_each_activity.activities: + if child.type == "Copy": + source_hint = _infer_source_type(child, definitions) + break + + _record_motif( + results, + definition=MOTIF_METADATA_DRIVEN_BULK_COPY, + matched_activities=matched, + source_type_hint=source_hint, + confidence_notes=notes, + ) + + return results + + +def _detect_file_landing_zone( + activities: list[AdfActivity], + by_name: dict[str, AdfActivity], + definitions: AdfDefinitions, + claimed: set[str], +) -> list[DetectedMotif]: + """Detects file landing zone processing pattern.""" + results: list[DetectedMotif] = [] + get_metadata_activities = _activities_of_type(activities, "GetMetadata", claimed) + + for get_metadata_activity in get_metadata_activities: + type_properties_text = _type_props_text(get_metadata_activity) + if not _has_keyword(type_properties_text, "childitems", "getchilditems", "childitem", "exists"): + if not _has_keyword(type_properties_text, "file", "folder", "blob", "path"): + continue + + matched: list[str] = [get_metadata_activity.name] + notes: list[str] = [f"GetMetadata '{get_metadata_activity.name}' lists files"] + + downstream_filter: AdfActivity | None = None + downstream_foreach: AdfActivity | None = None + downstream_delete: AdfActivity | None = None + + for activity in activities: + if activity.name in claimed: + continue + if activity.type == "Filter" and _depends_on(activity, get_metadata_activity.name): + downstream_filter = activity + if activity.type == "ForEach": + deps = _get_upstream_names(activity) + if get_metadata_activity.name in deps or (downstream_filter and downstream_filter.name in deps): + if activity.activities: + for child in activity.activities: + if child.type == "Copy": + downstream_foreach = activity + break + + if downstream_foreach is None: + continue + + if downstream_filter: + matched.append(downstream_filter.name) + notes.append(f"Filter '{downstream_filter.name}' narrows file list") + + matched.append(downstream_foreach.name) + notes.append(f"ForEach '{downstream_foreach.name}' processes files via Copy") + + for activity in activities: + if activity.name in claimed: + continue + if activity.type == "Delete" and _depends_on(activity, downstream_foreach.name): + downstream_delete = activity + break + + if downstream_delete: + matched.append(downstream_delete.name) + notes.append(f"Delete '{downstream_delete.name}' cleans up processed files") + + _record_motif( + results, + definition=MOTIF_FILE_LANDING_ZONE_PROCESSING, + matched_activities=matched, + source_type_hint="files", + confidence_notes=notes, + ) + + return results + + +def _detect_copy_and_notify( + activities: list[AdfActivity], + by_name: dict[str, AdfActivity], + definitions: AdfDefinitions, + claimed: set[str], +) -> list[DetectedMotif]: + """Detects copy-and-notify pattern.""" + results: list[DetectedMotif] = [] + copies = _activities_of_type(activities, "Copy", claimed) + + for copy_act in copies: + downstream_webs: list[AdfActivity] = [] + for activity in activities: + if activity.name in claimed: + continue + if activity.type == "WebActivity" and _depends_on(activity, copy_act.name): + downstream_webs.append(activity) + + if not downstream_webs: + continue + + notes: list[str] = [] + notification_found = False + for web in downstream_webs: + type_properties_text = _type_props_text(web) + if _has_keyword( + type_properties_text, "logic.azure.com", "email", "notify", "alert", "webhook", "slack", "teams" + ): + notification_found = True + notes.append(f"WebActivity '{web.name}' appears to be a notification call") + + if not notification_found: + # If there is no notification hint, we still accept if the Web + # activity depends on Copy with success/failure conditions + for web in downstream_webs: + if web.depends_on: + for dep in web.depends_on: + if dep.activity == copy_act.name and dep.dependency_conditions: + conds = [cond.lower() for cond in dep.dependency_conditions] + if "failed" in conds or "completed" in conds: + notification_found = True + notes.append( + f"WebActivity '{web.name}' triggers on {dep.dependency_conditions} of Copy" + ) + + if not notification_found: + continue + + matched = [copy_act.name] + [web.name for web in downstream_webs] + source_hint = _infer_source_type(copy_act, definitions) + + _record_motif( + results, + definition=MOTIF_COPY_AND_NOTIFY, + matched_activities=matched, + source_type_hint=source_hint, + confidence_notes=notes, + ) + + return results + + +def _detect_staged_load_synapse( + activities: list[AdfActivity], + by_name: dict[str, AdfActivity], + definitions: AdfDefinitions, + claimed: set[str], +) -> list[DetectedMotif]: + """Detects staged-load (Synapse) pattern.""" + results: list[DetectedMotif] = [] + copies = _activities_of_type(activities, "Copy", claimed) + + for copy_act in copies: + upstream_lookups = [ + by_name[name] + for name in _get_upstream_names(copy_act) + if name in by_name and by_name[name].type == "Lookup" + ] + if len(upstream_lookups) >= 2: + continue + + downstream_sp: AdfActivity | None = None + for activity in activities: + if activity.name in claimed: + continue + if activity.type == "SqlServerStoredProcedure" and _depends_on(activity, copy_act.name): + downstream_sp = activity + break + + if downstream_sp is None: + continue + + notes: list[str] = [] + type_properties_text = _type_props_text(copy_act) + + staging_hint = _has_keyword( + type_properties_text, + "polybase", + "staging", + "enablestaging", + "sqldw", + "synapse", + "copy_command", + "allowcopycommand", + ) + if staging_hint: + notes.append("Copy activity uses staging/PolyBase for Synapse loading") + else: + sink_hint = _has_keyword(type_properties_text, "sqldwsink", "azuresqldwsink", "synapsesink") + if sink_hint: + notes.append("Copy sink targets Azure Synapse / SQL DW") + else: + notes.append("Copy -> StoredProcedure pattern matches staged load") + + matched = [copy_act.name, downstream_sp.name] + source_hint = _infer_source_type(copy_act, definitions) + + _record_motif( + results, + definition=MOTIF_STAGED_LOAD_SYNAPSE, + matched_activities=matched, + source_type_hint=source_hint, + confidence_notes=notes, + ) + + return results + + +def _detect_rest_api_pagination( + activities: list[AdfActivity], + by_name: dict[str, AdfActivity], + definitions: AdfDefinitions, + claimed: set[str], +) -> list[DetectedMotif]: + """Detects REST API pagination pattern.""" + results: list[DetectedMotif] = [] + until_acts = _activities_of_type(activities, "Until", claimed) + + for until_act in until_acts: + # Until should contain Copy or WebActivity child + has_fetch_child = False + if until_act.activities: + for child in until_act.activities: + if child.type in ("Copy", "WebActivity"): + has_fetch_child = True + break + if not has_fetch_child: + continue + + upstream_names = _get_upstream_names(until_act) + upstream_webs: list[AdfActivity] = [] + upstream_setvars: list[AdfActivity] = [] + + for name in upstream_names: + activity = by_name.get(name) + if not activity or activity.name in claimed: + continue + if activity.type == "WebActivity": + upstream_webs.append(activity) + elif activity.type == "SetVariable": + upstream_setvars.append(activity) + + for set_variable_activity in list(upstream_setvars): + for name in _get_upstream_names(set_variable_activity): + activity = by_name.get(name) + if activity and activity.type == "WebActivity" and activity.name not in claimed: + if activity not in upstream_webs: + upstream_webs.append(activity) + + notes: list[str] = [] + + evidence = False + for web in upstream_webs: + type_properties_text = _type_props_text(web) + if _has_keyword(type_properties_text, "oauth", "token", "auth", "bearer", "client_id"): + evidence = True + notes.append(f"WebActivity '{web.name}' appears to fetch an auth token") + + # Check Until children for pagination keywords + if until_act.activities: + for child in until_act.activities: + type_properties_text = _type_props_text(child) + if _has_keyword(type_properties_text, "page", "cursor", "offset", "skip", "next", "continuation"): + evidence = True + notes.append(f"Until child '{child.name}' uses pagination") + if child.type == "SetVariable": + set_variable_text = _type_props_text(child) + if _has_keyword(set_variable_text, "cursor", "page", "offset", "next", "token"): + evidence = True + notes.append(f"SetVariable '{child.name}' updates pagination cursor") + + if not evidence: + continue + + matched: list[str] = [] + matched.extend(web.name for web in upstream_webs) + matched.extend(set_variable_activity.name for set_variable_activity in upstream_setvars) + matched.append(until_act.name) + + _record_motif( + results, + definition=MOTIF_REST_API_PAGINATION, + matched_activities=matched, + source_type_hint="rest_api", + confidence_notes=notes, + ) + + return results + + +def _detect_parent_child_orchestration( + activities: list[AdfActivity], + by_name: dict[str, AdfActivity], + definitions: AdfDefinitions, + claimed: set[str], +) -> list[DetectedMotif]: + """Detects parent-child orchestration pattern.""" + results: list[DetectedMotif] = [] + for_each_activities = _activities_of_type(activities, "ForEach", claimed) + + for for_each_activity in for_each_activities: + has_exec_child = False + if for_each_activity.activities: + for child in for_each_activity.activities: + if child.type == "ExecutePipeline": + has_exec_child = True + break + if not has_exec_child: + continue + + upstream_lookups: list[AdfActivity] = [] + for name in _get_upstream_names(for_each_activity): + activity = by_name.get(name) + if activity and activity.type == "Lookup" and activity.name not in claimed: + upstream_lookups.append(activity) + + if not upstream_lookups: + continue + + notes: list[str] = [ + f"Lookup '{upstream_lookups[0].name}' provides work items", + f"ForEach '{for_each_activity.name}' iterates and calls child pipelines via ExecutePipeline", + ] + + matched = [lookup_activity.name for lookup_activity in upstream_lookups] + [for_each_activity.name] + + _record_motif( + results, + definition=MOTIF_PARENT_CHILD_ORCHESTRATION, + matched_activities=matched, + source_type_hint=None, + confidence_notes=notes, + ) + + return results + + +def _detect_file_existence_validation( + activities: list[AdfActivity], + by_name: dict[str, AdfActivity], + definitions: AdfDefinitions, + claimed: set[str], +) -> list[DetectedMotif]: + """Detects file-existence validation pattern.""" + results: list[DetectedMotif] = [] + get_metadata_activities = _activities_of_type(activities, "GetMetadata", claimed) + + for get_metadata_activity in get_metadata_activities: + type_properties_text = _type_props_text(get_metadata_activity) + # Must check for existence + if not _has_keyword(type_properties_text, "exists"): + continue + + downstream_if: AdfActivity | None = None + for activity in activities: + if activity.name in claimed: + continue + if activity.type == "IfCondition" and _depends_on(activity, get_metadata_activity.name): + downstream_if = activity + break + + if downstream_if is None: + continue + + notes = [ + f"GetMetadata '{get_metadata_activity.name}' checks file existence", + f"IfCondition '{downstream_if.name}' gates on the result", + ] + + matched = [get_metadata_activity.name, downstream_if.name] + + _record_motif( + results, + definition=MOTIF_FILE_EXISTENCE_VALIDATION, + matched_activities=matched, + source_type_hint="files", + confidence_notes=notes, + ) + + return results + + +def _detect_scd_type_2( + activities: list[AdfActivity], + by_name: dict[str, AdfActivity], + definitions: AdfDefinitions, + claimed: set[str], +) -> list[DetectedMotif]: + """Detects SCD Type 2 pattern.""" + results: list[DetectedMotif] = [] + dataflow_activities = _activities_of_type(activities, "ExecuteDataFlow", claimed) + + for dataflow_activity in dataflow_activities: + upstream_copies: list[AdfActivity] = [] + for name in _get_upstream_names(dataflow_activity): + activity = by_name.get(name) + if activity and activity.type == "Copy" and activity.name not in claimed: + upstream_copies.append(activity) + + if not upstream_copies: + continue + + notes: list[str] = [] + type_properties_text = _type_props_text(dataflow_activity) + dataflow_name = dataflow_activity.name.lower() + + scd_evidence = _has_keyword( + type_properties_text + " " + dataflow_name, + "scd", + "slowly", + "dimension", + "alterrow", + "type2", + "type_2", + "surrogate", + "effective_date", + "end_date", + "is_current", + ) + + if not scd_evidence: + continue + + notes.append(f"Copy '{upstream_copies[0].name}' stages data") + notes.append(f"DataFlow '{dataflow_activity.name}' performs SCD Type 2 logic") + + matched = [copy.name for copy in upstream_copies] + [dataflow_activity.name] + + _record_motif( + results, + definition=MOTIF_SCD_TYPE_2, + matched_activities=matched, + source_type_hint=_infer_source_type(upstream_copies[0], definitions), + confidence_notes=notes, + ) + + return results + + +__all__ = [ + "detect_motifs", +] diff --git a/src/orchestra/parser/__init__.py b/src/orchestra/parser/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/orchestra/parser/adf_loader.py b/src/orchestra/parser/adf_loader.py new file mode 100644 index 0000000..03e6374 --- /dev/null +++ b/src/orchestra/parser/adf_loader.py @@ -0,0 +1,698 @@ +"""Loads ADF JSON files from a directory structure and produce typed AST objects.""" + +from __future__ import annotations + +import argparse +import json +import logging +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from flowx.models.adf_ast import ( + AdfActivity, + AdfDataset, + AdfDatasetReference, + AdfDefinitions, + AdfDependency, + AdfLinkedService, + AdfLinkedServiceReference, + AdfParameter, + AdfPipeline, + AdfPolicy, + AdfTrigger, + AdfVariable, + Inventory, + InventoryItem, + TranslationStrategy, +) + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Activity-type classification registries +# --------------------------------------------------------------------------- + +DETERMINISTIC_TYPES: set[str] = { + "Copy", + "DatabricksNotebook", + "DatabricksSparkJar", + "DatabricksSparkPython", + "ForEach", + "IfCondition", + "SetVariable", + "Switch", + "Lookup", + "WebActivity", + "Delete", + "ExecutePipeline", + "DatabricksJob", + "Wait", + "Filter", + "AppendVariable", +} + +AGENTIC_TYPES: dict[str, str] = { + "ExecuteDataFlow": "adf-to-databricks:adf-dataflow-converter", + "Until": "adf-to-databricks:adf-pipeline-converter", + "SqlServerStoredProcedure": "adf-to-databricks:adf-pipeline-converter", + "AzureFunction": "adf-to-databricks:adf-pipeline-converter", + "WebHook": "adf-to-databricks:adf-pipeline-converter", + "Custom": "adf-to-databricks:adf-pipeline-converter", + "ExecuteSSISPackage": "adf-to-databricks:adf-pipeline-converter", + "AzureMLExecutePipeline": "adf-to-databricks:adf-pipeline-converter", + "GetMetadata": "adf-to-databricks:adf-pipeline-converter", + "Validation": "adf-to-databricks:adf-pipeline-converter", + "Fail": "adf-to-databricks:adf-pipeline-converter", + "Script": "adf-to-databricks:adf-pipeline-converter", +} + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def load_adf_definitions(source_dir: Path) -> AdfDefinitions: + """Loads all ADF JSON files from *source_dir* and return an :class:`AdfDefinitions`. + + Args: + source_dir: Root directory containing ADF JSON exports. May be a + directory tree with ``pipelines/``, ``datasets/``, etc., or a + single ARM template file. + + Returns: + Fully populated :class:`AdfDefinitions` object. + """ + source_dir = Path(source_dir).resolve() + + if source_dir.is_file() and source_dir.suffix == ".json": + return _load_arm_template(source_dir) + + pipelines: list[AdfPipeline] = [] + datasets: dict[str, AdfDataset] = {} + linked_services: dict[str, AdfLinkedService] = {} + triggers: list[AdfTrigger] = [] + + pipeline_dir = _find_json_dir(source_dir, "pipelines", "pipeline") + if pipeline_dir is not None: + for json_file in sorted(pipeline_dir.glob("*.json")): + try: + data = json.loads(json_file.read_text(encoding="utf-8")) + pipelines.append(_parse_pipeline_json(data, fallback_name=json_file.stem)) + except Exception: + logger.exception("Failed to parse pipeline file %s", json_file) + + dataset_dir = _find_json_dir(source_dir, "datasets", "dataset") + if dataset_dir is not None: + for json_file in sorted(dataset_dir.glob("*.json")): + try: + data = json.loads(json_file.read_text(encoding="utf-8")) + dataset = _parse_dataset_json(data, fallback_name=json_file.stem) + datasets[dataset.name] = dataset + except Exception: + logger.exception("Failed to parse dataset file %s", json_file) + + linked_service_dir = _find_json_dir(source_dir, "linked_services", "linkedService", "linkedServices") + if linked_service_dir is not None: + for json_file in sorted(linked_service_dir.glob("*.json")): + try: + data = json.loads(json_file.read_text(encoding="utf-8")) + linked_service = _parse_linked_service_json(data, fallback_name=json_file.stem) + linked_services[linked_service.name] = linked_service + except Exception: + logger.exception("Failed to parse linked-service file %s", json_file) + + trigger_dir = _find_json_dir(source_dir, "triggers", "trigger") + if trigger_dir is not None: + for json_file in sorted(trigger_dir.glob("*.json")): + try: + data = json.loads(json_file.read_text(encoding="utf-8")) + triggers.append(_parse_trigger_json(data, fallback_name=json_file.stem)) + except Exception: + logger.exception("Failed to parse trigger file %s", json_file) + + return AdfDefinitions( + pipelines=pipelines, + datasets=datasets, + linked_services=linked_services, + triggers=triggers, + ) + + +def classify_activity(activity_type: str) -> tuple[TranslationStrategy, str | None]: + """Classify an ADF activity type into a translation strategy. + + Args: + activity_type: ADF activity type string (e.g. ``"Copy"``). + + Returns: + A ``(strategy, agentic_skill_name)`` tuple. *agentic_skill_name* is + ``None`` for deterministic and unsupported strategies. + """ + if activity_type in DETERMINISTIC_TYPES: + return TranslationStrategy.DETERMINISTIC, None + if activity_type in AGENTIC_TYPES: + return TranslationStrategy.AGENTIC, AGENTIC_TYPES[activity_type] + return TranslationStrategy.UNSUPPORTED, None + + +def build_inventory(definitions: AdfDefinitions) -> Inventory: + """Walks all pipelines in *definitions* and classify every activity. + + Args: + definitions: Parsed ADF definitions. + + Returns: + :class:`Inventory` with one :class:`InventoryItem` per activity. + """ + items: list[InventoryItem] = [] + deterministic = 0 + agentic = 0 + unsupported = 0 + + for pipeline in definitions.pipelines: + _classify_activities(pipeline.name, pipeline.activities, items) + + for item in items: + if item.strategy is TranslationStrategy.DETERMINISTIC: + deterministic += 1 + elif item.strategy is TranslationStrategy.AGENTIC: + agentic += 1 + else: + unsupported += 1 + + return Inventory( + items=items, + deterministic_count=deterministic, + agentic_count=agentic, + unsupported_count=unsupported, + pipeline_count=len(definitions.pipelines), + ) + + +# --------------------------------------------------------------------------- +# Internal helpers — parsing +# --------------------------------------------------------------------------- + + +def _find_json_dir(source_dir: Path, *candidate_names: str) -> Path | None: + """Return the first existing subdirectory matching one of *candidate_names*. + + Args: + source_dir: Parent directory to search within. + *candidate_names: Case-insensitive directory name candidates. + + Returns: + The resolved :class:`Path` of the matching directory, or ``None``. + """ + for name in candidate_names: + candidate = source_dir / name + if candidate.is_dir(): + return candidate + # Case-insensitive fallback + lower_candidates = {n.lower() for n in candidate_names} + for child in source_dir.iterdir(): + if child.is_dir() and child.name.lower() in lower_candidates: + return child + return None + + +def _parse_pipeline_json(data: dict[str, Any], *, fallback_name: str = "unknown") -> AdfPipeline: + """Parses a single pipeline JSON payload into an :class:`AdfPipeline`. + + Args: + data: Raw JSON dictionary (either a bare pipeline or wrapped in ``properties``). + fallback_name: Name to use if the JSON does not contain one. + + Returns: + Parsed :class:`AdfPipeline`. + """ + data = _normalize_arm(data) + props = data.get("properties", data) + name = data.get("name") or props.get("name") or fallback_name + + activities_raw: list[dict[str, Any]] = props.get("activities", []) + activities = [parse_activity(a) for a in activities_raw] + + parameters: dict[str, AdfParameter] | None = None + raw_params = props.get("parameters") + if raw_params: + parameters = {} + for pname, pval in raw_params.items(): + if isinstance(pval, dict): + parameters[pname] = AdfParameter( + type=pval.get("type", "String"), + default_value=pval.get("defaultValue"), + ) + else: + parameters[pname] = AdfParameter(default_value=pval) + + variables: dict[str, AdfVariable] | None = None + raw_vars = props.get("variables") + if raw_vars: + variables = {} + for vname, vval in raw_vars.items(): + if isinstance(vval, dict): + variables[vname] = AdfVariable( + type=vval.get("type", "String"), + default_value=vval.get("defaultValue"), + ) + else: + variables[vname] = AdfVariable(default_value=vval) + + annotations = props.get("annotations") + folder_raw = props.get("folder") + folder = folder_raw.get("name") if isinstance(folder_raw, dict) else folder_raw + + return AdfPipeline( + name=name, + activities=activities, + parameters=parameters, + variables=variables, + annotations=annotations, + folder=folder, + ) + + +def parse_activity(data: dict[str, Any]) -> AdfActivity: + """Parses an activity dict into a typed :class:`AdfActivity` AST node. + + Args: + data: Raw activity JSON dictionary. + + Returns: + Parsed :class:`AdfActivity`. + """ + name = data.get("name", "unnamed") + adf_type = data.get("type", "Unknown") + + depends_on: list[AdfDependency] | None = None + raw_deps = data.get("dependsOn") + if raw_deps: + depends_on = [ + AdfDependency( + activity=dep.get("activity", ""), + dependency_conditions=dep.get("dependencyConditions", ["Succeeded"]), + ) + for dep in raw_deps + ] + + policy: AdfPolicy | None = None + raw_policy = data.get("policy") + if raw_policy: + policy = AdfPolicy( + timeout=raw_policy.get("timeout"), + retry=raw_policy.get("retry"), + retry_interval_in_seconds=raw_policy.get("retryIntervalInSeconds"), + secure_input=raw_policy.get("secureInput", False), + secure_output=raw_policy.get("secureOutput", False), + ) + + # Type properties — prefer explicit typeProperties; fall back to + # collecting non-common top-level keys (flattened format). + type_properties = data.get("typeProperties") + if type_properties is None: + type_properties = _collect_type_properties(data) + + inputs = _parse_dataset_refs(data.get("inputs")) + outputs = _parse_dataset_refs(data.get("outputs")) + + linked_service_name: AdfLinkedServiceReference | None = None + raw_ls = data.get("linkedServiceName") + if raw_ls and isinstance(raw_ls, dict): + linked_service_name = AdfLinkedServiceReference( + reference_name=raw_ls.get("referenceName", ""), + type=raw_ls.get("type", "LinkedServiceReference"), + ) + + if_true_activities: list[AdfActivity] | None = None + if_false_activities: list[AdfActivity] | None = None + child_activities: list[AdfActivity] | None = None + + if type_properties: + raw_if_true = type_properties.get("ifTrueActivities") + if raw_if_true: + if_true_activities = [parse_activity(a) for a in raw_if_true] + raw_if_false = type_properties.get("ifFalseActivities") + if raw_if_false: + if_false_activities = [parse_activity(a) for a in raw_if_false] + raw_children = type_properties.get("activities") + if raw_children: + child_activities = [parse_activity(a) for a in raw_children] + + return AdfActivity( + name=name, + type=adf_type, + depends_on=depends_on, + policy=policy, + type_properties=type_properties, + inputs=inputs, + outputs=outputs, + linked_service_name=linked_service_name, + if_true_activities=if_true_activities, + if_false_activities=if_false_activities, + activities=child_activities, + ) + + +_COMMON_ACTIVITY_KEYS: frozenset[str] = frozenset( + { + "name", + "type", + "dependsOn", + "policy", + "userProperties", + "description", + "state", + "onInactiveMarkAs", + "additionalProperties", + "inputs", + "outputs", + "linkedServiceName", + "typeProperties", + } +) + + +def _collect_type_properties(data: dict[str, Any]) -> dict[str, Any] | None: + """Collects type-specific fields from a flattened activity dict. + + Args: + data: Raw activity JSON dictionary. + + Returns: + Synthesised type-properties dict, or ``None`` if no extra keys exist. + """ + type_properties: dict[str, Any] = {k: v for k, v in data.items() if k not in _COMMON_ACTIVITY_KEYS} + return type_properties if type_properties else None + + +def _parse_dataset_refs(raw: list[dict[str, Any]] | None) -> list[AdfDatasetReference] | None: + """Parses a list of dataset reference dicts into typed objects. + + Args: + raw: Raw list of dataset reference dictionaries, or ``None``. + + Returns: + List of :class:`AdfDatasetReference` objects, or ``None``. + """ + if not raw: + return None + refs: list[AdfDatasetReference] = [] + for item in raw: + ds_ref = item.get("dataset", item) + refs.append( + AdfDatasetReference( + reference_name=ds_ref.get("referenceName", ""), + type=ds_ref.get("type", "DatasetReference"), + parameters=ds_ref.get("parameters"), + ) + ) + return refs + + +def _parse_dataset_json(data: dict[str, Any], *, fallback_name: str = "unknown") -> AdfDataset: + """Parses a dataset JSON payload. + + Args: + data: Raw JSON dictionary. + fallback_name: Name to use if the JSON does not contain one. + + Returns: + Parsed :class:`AdfDataset`. + """ + data = _normalize_arm(data) + props = data.get("properties", data) + name = data.get("name") or fallback_name + ds_type = props.get("type", "Unknown") + ls_ref = props.get("linkedServiceName", {}) + ls_name = ls_ref.get("referenceName") if isinstance(ls_ref, dict) else ls_ref + + return AdfDataset( + name=name, + type=ds_type, + properties=props, + linked_service_name=ls_name, + ) + + +def _parse_linked_service_json(data: dict[str, Any], *, fallback_name: str = "unknown") -> AdfLinkedService: + """Parses a linked-service JSON payload. + + Args: + data: Raw JSON dictionary. + fallback_name: Name to use if the JSON does not contain one. + + Returns: + Parsed :class:`AdfLinkedService`. + """ + data = _normalize_arm(data) + props = data.get("properties", data) + name = data.get("name") or fallback_name + ls_type = props.get("type", "Unknown") + + return AdfLinkedService(name=name, type=ls_type, properties=props) + + +def _parse_trigger_json(data: dict[str, Any], *, fallback_name: str = "unknown") -> AdfTrigger: + """Parses a trigger JSON payload. + + Args: + data: Raw JSON dictionary. + fallback_name: Name to use if the JSON does not contain one. + + Returns: + Parsed :class:`AdfTrigger`. + """ + data = _normalize_arm(data) + props = data.get("properties", data) + name = data.get("name") or fallback_name + trigger_type = props.get("type", "Unknown") + trigger_pipelines = props.get("pipelines") + + return AdfTrigger( + name=name, + type=trigger_type, + properties=props, + pipelines=trigger_pipelines, + ) + + +def _normalize_arm(data: dict[str, Any]) -> dict[str, Any]: + """If *data* is an ARM template wrapper, unwrap to the inner resource definition. + + Args: + data: Possibly ARM-wrapped JSON dictionary. + + Returns: + The unwrapped resource dictionary, or *data* unchanged. + """ + if "$schema" not in data and "resources" not in data: + return data + + resources = data.get("resources", []) + if not resources: + return data + + adf_suffixes = ("/pipelines", "/datasets", "/linkedServices", "/triggers") + for resource in resources: + rtype = resource.get("type", "") + if any(rtype.endswith(suffix) for suffix in adf_suffixes): + result: dict[str, Any] = {} + arm_name = resource.get("name", "") + if "/" in arm_name: + result["name"] = arm_name.rsplit("/", 1)[-1].strip("'])") + else: + result["name"] = arm_name + result["properties"] = resource.get("properties", {}) + return result + + return data + + +def _load_arm_template(template_path: Path) -> AdfDefinitions: + """Loads all ADF resources from a single ARM template file. + + Args: + template_path: Path to the ARM template JSON file. + + Returns: + Parsed :class:`AdfDefinitions`. + """ + data = json.loads(template_path.read_text(encoding="utf-8")) + resources = data.get("resources", []) + + pipelines: list[AdfPipeline] = [] + datasets: dict[str, AdfDataset] = {} + linked_services: dict[str, AdfLinkedService] = {} + triggers: list[AdfTrigger] = [] + + for resource in resources: + rtype = resource.get("type", "") + props = resource.get("properties", {}) + raw_name = resource.get("name", "") + if "/" in raw_name: + name = raw_name.rsplit("/", 1)[-1].strip("'])") + else: + name = raw_name + + wrapped = {"name": name, "properties": props} + + if rtype.endswith("/pipelines"): + try: + pipelines.append(_parse_pipeline_json(wrapped, fallback_name=name)) + except Exception: + logger.exception("Failed to parse ARM pipeline resource %s", name) + elif rtype.endswith("/datasets"): + try: + dataset = _parse_dataset_json(wrapped, fallback_name=name) + datasets[dataset.name] = dataset + except Exception: + logger.exception("Failed to parse ARM dataset resource %s", name) + elif rtype.endswith("/linkedServices"): + try: + linked_service = _parse_linked_service_json(wrapped, fallback_name=name) + linked_services[linked_service.name] = linked_service + except Exception: + logger.exception("Failed to parse ARM linked-service resource %s", name) + elif rtype.endswith("/triggers"): + try: + triggers.append(_parse_trigger_json(wrapped, fallback_name=name)) + except Exception: + logger.exception("Failed to parse ARM trigger resource %s", name) + + return AdfDefinitions( + pipelines=pipelines, + datasets=datasets, + linked_services=linked_services, + triggers=triggers, + ) + + +# --------------------------------------------------------------------------- +# Internal helpers — classification +# --------------------------------------------------------------------------- + + +def _classify_activities( + pipeline_name: str, + activities: list[AdfActivity], + items: list[InventoryItem], +) -> None: + """Recursively classify activities and append to *items*. + + Args: + pipeline_name: Name of the owning pipeline (for the inventory row). + activities: Activities to classify. + items: Accumulator list to append results to. + """ + for activity in activities: + strategy, skill = classify_activity(activity.type) + dep_names = [d.activity for d in activity.depends_on] if activity.depends_on else None + + items.append( + InventoryItem( + pipeline_name=pipeline_name, + activity_name=activity.name, + activity_type=activity.type, + strategy=strategy, + agentic_skill=skill, + depends_on=dep_names, + ) + ) + + if activity.if_true_activities: + _classify_activities(pipeline_name, activity.if_true_activities, items) + if activity.if_false_activities: + _classify_activities(pipeline_name, activity.if_false_activities, items) + if activity.activities: + _classify_activities(pipeline_name, activity.activities, items) + + +# --------------------------------------------------------------------------- +# Serialisation helpers +# --------------------------------------------------------------------------- + + +def _inventory_to_dict(inventory: Inventory, source_dir: str) -> dict[str, Any]: + """Serialise an :class:`Inventory` to a JSON-friendly dictionary. + + Args: + inventory: The inventory to serialise. + source_dir: Original source directory path (for provenance). + + Returns: + Dictionary suitable for ``json.dumps``. + """ + pipeline_map: dict[str, list[dict[str, Any]]] = {} + for item in inventory.items: + entry: dict[str, Any] = { + "name": item.activity_name, + "type": item.activity_type, + "strategy": item.strategy.value, + } + if item.agentic_skill: + entry["skill"] = item.agentic_skill + if item.depends_on: + entry["depends_on"] = item.depends_on + pipeline_map.setdefault(item.pipeline_name, []).append(entry) + + total = inventory.deterministic_count + inventory.agentic_count + inventory.unsupported_count + coverage_pct = round((inventory.deterministic_count + inventory.agentic_count) / total * 100, 1) if total else 0.0 + + return { + "source_dir": source_dir, + "generated_at": datetime.now(timezone.utc).isoformat(), + "pipelines": [{"name": pname, "activities": acts} for pname, acts in pipeline_map.items()], + "summary": { + "pipeline_count": inventory.pipeline_count, + "activity_count": total, + "deterministic_count": inventory.deterministic_count, + "agentic_count": inventory.agentic_count, + "unsupported_count": inventory.unsupported_count, + "coverage_pct": coverage_pct, + }, + } + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Load ADF definitions and build a translation inventory.") + parser.add_argument("--source-dir", required=True, type=Path, help="Root directory containing ADF JSON exports.") + parser.add_argument( + "--output-dir", + type=Path, + default=Path("./orchestra_output/ingest"), + help="Directory to write inventory.json into.", + ) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + definitions = load_adf_definitions(args.source_dir) + logger.info("Loaded %d pipeline(s) from %s", len(definitions.pipelines), args.source_dir) + + inventory = build_inventory(definitions) + + output_dir: Path = args.output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + inventory_path = output_dir / "inventory.json" + inventory_dict = _inventory_to_dict(inventory, str(args.source_dir)) + inventory_path.write_text(json.dumps(inventory_dict, indent=2), encoding="utf-8") + logger.info("Wrote inventory to %s", inventory_path) + + summary = inventory_dict["summary"] + print("\nADF Ingestion Summary") + print("=====================") + print(f"Pipelines parsed: {summary['pipeline_count']}") + print(f"Total activities: {summary['activity_count']}") + print("\nStrategy Breakdown:") + print(f" Deterministic: {summary['deterministic_count']}") + print(f" Agentic: {summary['agentic_count']}") + print(f" Unsupported: {summary['unsupported_count']}") + print(f"\nCoverage: {summary['coverage_pct']}%") diff --git a/src/orchestra/parser/expression_parser.py b/src/orchestra/parser/expression_parser.py new file mode 100644 index 0000000..1fcb69c --- /dev/null +++ b/src/orchestra/parser/expression_parser.py @@ -0,0 +1,1397 @@ +"""Translates ADF expressions to a unified ExpressionResult.""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from typing import Any + +from flowx.models.ir import ExpressionResult, TranslationContext + +_ITEM_RE = re.compile(r"item\(\s*\)$", re.IGNORECASE) + +_ITEM_FIELD_RE = re.compile(r"item\(\s*\)\.(\w+)", re.IGNORECASE) + +_ACTIVITY_OUTPUT_RE = re.compile( + r"""activity\(\s*'([^']+)'\s*\)\.output(?:\.(.+))?""", + re.IGNORECASE, +) + +_PIPELINE_PARAM_RE = re.compile( + r"""pipeline\(\s*\)\.parameters\.(\w+)""", + re.IGNORECASE, +) + +_PIPELINE_PROPERTY_RE = re.compile( + r"""pipeline\(\s*\)\.(\w+)""", + re.IGNORECASE, +) + +_VARIABLE_RE = re.compile( + r"""variables\(\s*'([^']+)'\s*\)""", + re.IGNORECASE, +) + +_CONCAT_RE = re.compile( + r"""concat\((.+)\)""", + re.IGNORECASE | re.DOTALL, +) + +_UTCNOW_RE = re.compile( + r"""utcNow\(\s*(?:'([^']*)')?\s*\)""", + re.IGNORECASE, +) + +_DATE_FORMAT_MAP: dict[str, str] = { + "yyyy": "%Y", + "yy": "%y", + "MM": "%m", + "dd": "%d", + "HH": "%H", + "hh": "%I", + "mm": "%M", + "ss": "%S", + "fff": "%f", + "tt": "%p", +} + +_DAB_PIPELINE_PROPERTY_MAP: dict[str, str] = { + "RunId": "{{job.run_id}}", + "GroupId": "{{job.run_id}}", + "TriggerTime": "{{job.start_time.iso_datetime}}", + "Pipeline": "{{job.name}}", + "TriggerName": "{{job.trigger.type}}", + "DataFactory": "{{job.run_id}}", +} + +_INTERPOLATION_RE = re.compile(r"@\{(.+?)\}") + +_FUNCTION_CALL_RE = re.compile( + r"([a-zA-Z_]\w*)\((.*)?\)$", + re.IGNORECASE | re.DOTALL, +) + +_DATETIME_IMPORTS = ["from datetime import datetime, timezone, timedelta"] + +_TIME_UNIT_MAP: dict[str, str] = { + "Second": "seconds", + "Minute": "minutes", + "Hour": "hours", + "Day": "days", + "Week": "weeks", +} + + +def resolve_expression( + value: str | dict[str, Any] | int | float | bool, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> ExpressionResult | None: + """Translates an ADF expression to an :class:`ExpressionResult`. + + Args: + value: The ADF expression value. May be a plain scalar, an + ``@``-prefixed expression string, or an ``{"type": "Expression", + "value": "..."}`` dict. + context: Translation context carrying variable mappings. + variable_task_keys: Optional explicit mapping of variable names to + setter task keys. When provided these take precedence over + ``context.variable_cache``. + + Returns: + An :class:`ExpressionResult`, or ``None`` if the expression is too + complex for deterministic translation. + """ + if isinstance(value, dict): + if value.get("type") == "Expression" and "value" in value: + return resolve_expression(value["value"], context, variable_task_keys=variable_task_keys) + return None + + if isinstance(value, bool): + return ExpressionResult(kind="literal", value=str(value)) + if isinstance(value, (int, float)): + return ExpressionResult(kind="literal", value=str(value)) + + if not isinstance(value, str): + return None + + if not value.startswith("@"): + return ExpressionResult(kind="literal", value=value) + + expr = value[1:] # strip leading @ + + if _ITEM_RE.match(expr): + return ExpressionResult(kind="dab_ref", value="{{input}}") + + match = _ITEM_FIELD_RE.match(expr) + if match: + field_name = match.group(1) + return ExpressionResult(kind="dab_ref", value="{{input." + field_name + "}}") + + result = _resolve_pipeline_param(expr) + if result is not None: + return result + + result = _resolve_pipeline_property(expr) + if result is not None: + return result + + result = _resolve_activity_output(expr) + if result is not None: + return result + + result = _resolve_variable(expr, context, variable_task_keys=variable_task_keys) + if result is not None: + return result + + result = _resolve_utcnow(expr) + if result is not None: + return result + + result = _resolve_concat(expr, context, variable_task_keys=variable_task_keys) + if result is not None: + return result + + result = _resolve_function_call(expr, context, variable_task_keys=variable_task_keys) + if result is not None: + return result + + return None + + +def resolve_interpolated_string( + value: str, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> str: + """Resolves ``@{...}`` interpolation tokens within a string. + + Args: + value: A string potentially containing ``@{...}`` tokens. + context: Translation context for resolving variables. + variable_task_keys: Optional explicit variable-name-to-task-key map. + + Returns: + The string with all ``@{...}`` tokens replaced by resolved values. + Tokens that cannot be resolved are left unchanged. + """ + if not isinstance(value, str): + return value + + if "@{" not in value: + return value + + def _replace_match(match: re.Match[str]) -> str: + inner_expr = match.group(1) + result = resolve_expression("@" + inner_expr, context, variable_task_keys=variable_task_keys) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + return match.group(0) + + return _INTERPOLATION_RE.sub(_replace_match, value) + + +def resolve_interpolated_string_for_notebook( + value: str, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> str: + """Resolves ``@{...}`` tokens to Python f-string expressions for notebook code. + + Args: + value: A string containing ``@{...}`` tokens. + context: Translation context for resolving variables. + variable_task_keys: Optional explicit variable-name-to-task-key map. + + Returns: + A string with ``@{...}`` tokens replaced by Python f-string expressions. + """ + if not isinstance(value, str) or "@{" not in value: + return value + + def _replace_match(match: re.Match[str]) -> str: + inner_expr = match.group(1) + result = resolve_expression("@" + inner_expr, context, variable_task_keys=variable_task_keys) + if result is None: + return match.group(0) + if result.kind == "literal": + return result.value + if result.kind == "dab_ref": + ref = result.value + param_match = re.match(r"\{\{job\.parameters\.(\w+)\}\}", ref) + if param_match: + return "{dbutils.widgets.get('" + param_match.group(1) + "')}" + task_value_match = re.match(r"\{\{tasks\.([^.]+)\.values\.(\w+)\}\}", ref) + if task_value_match: + return ( + "{dbutils.jobs.taskValues.get(taskKey='" + + task_value_match.group(1) + + "', key='" + + task_value_match.group(2) + + "')}" + ) + if ref == "{{job.run_id}}": + return "{spark.conf.get('spark.databricks.job.runId', '')}" + if ref == "{{job.name}}": + return "{spark.conf.get('spark.databricks.job.parentName', '')}" + if ref == "{{job.start_time.iso_datetime}}": + return "{spark.conf.get('spark.databricks.job.triggerTime', '')}" + if ref == "{{input}}": + return "{dbutils.widgets.get('item')}" + return ref + return "{" + result.value + "}" + + return _INTERPOLATION_RE.sub(_replace_match, value) + + +def parse_expression(value: str | dict[str, Any] | int | float | bool, context: TranslationContext) -> str | None: + """Backward-compatible wrapper: return the resolved value for any kind, or None. + + Args: + value: The ADF expression value. + context: Translation context. + + Returns: + A string value, or ``None`` for unsupported expressions. + """ + result = resolve_expression(value, context) + if result is None: + return None + return result.value + + +def parse_expression_for_dab( + value: str | dict[str, Any] | int | float | bool, + *, + variable_task_keys: dict[str, str] | None = None, +) -> str | None: + """Backward-compatible wrapper: return DAB dynamic value ref or None. + + Args: + value: The ADF expression value. + variable_task_keys: Optional mapping of variable names to setter task keys. + + Returns: + A DAB dynamic value reference string, or ``None``. + """ + context = TranslationContext() + result = resolve_expression(value, context, variable_task_keys=variable_task_keys) + if result is None: + return None + if result.kind == "dab_ref": + return result.value + return None + + +def _resolve_pipeline_param(expr: str) -> ExpressionResult | None: + """Resolves ``pipeline().parameters.X`` -> DAB ref.""" + match = _PIPELINE_PARAM_RE.match(expr) + if match is None: + return None + param_name = match.group(1) + return ExpressionResult(kind="dab_ref", value="{{" + f"job.parameters.{param_name}" + "}}") + + +def _resolve_pipeline_property(expr: str) -> ExpressionResult | None: + """Resolves ``pipeline().PropertyName`` -> DAB ref.""" + match = _PIPELINE_PROPERTY_RE.match(expr) + if match is None: + return None + prop = match.group(1) + if prop == "parameters": + return None + dab_ref = _DAB_PIPELINE_PROPERTY_MAP.get(prop) + if dab_ref is not None: + return ExpressionResult(kind="dab_ref", value=dab_ref) + return None + + +def _resolve_activity_output(expr: str) -> ExpressionResult | None: + """Resolves ``activity('Name').output...`` -> DAB ref.""" + match = _ACTIVITY_OUTPUT_RE.match(expr) + if match is None: + return None + activity_name = match.group(1) + task_key = re.sub(r"[^a-zA-Z0-9_-]", "_", activity_name) + task_key = re.sub(r"_+", "_", task_key).strip("_") or "unnamed" + + property_path = match.group(2) or "" + if property_path: + parts = property_path.split(".") + field = parts[-1] if parts[-1] != "firstRow" else "result" + if field == "value": + field = "result" + else: + field = "result" + + return ExpressionResult(kind="dab_ref", value="{{" + f"tasks.{task_key}.values.{field}" + "}}") + + +def _resolve_variable( + expr: str, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> ExpressionResult | None: + """Resolves ``variables('name')`` -> task value DAB ref.""" + match = _VARIABLE_RE.match(expr) + if match is None: + return None + var_name = match.group(1) + + # Always resolve to the task value reference. This preserves the + # explicit task dependency chain — downstream tasks must depend on the + # setter task. Even when the variable was set to a DAB built-in like + # {{job.start_time.iso_datetime}}, the task value is the canonical + # source since the setter notebook may transform the value. + variable_task_keys_map = variable_task_keys or {} + setter_key = variable_task_keys_map.get(var_name) or context.get_variable_task_key(var_name) or var_name + return ExpressionResult(kind="dab_ref", value="{{" + f"tasks.{setter_key}.values.{var_name}" + "}}") + + +def _resolve_utcnow(expr: str) -> ExpressionResult | None: + """Resolves ``utcNow()`` or ``utcNow('format')`` -> notebook_code.""" + match = _UTCNOW_RE.match(expr) + if match is None: + return None + format_string = match.group(1) + if format_string: + python_format = _convert_date_format(format_string) + return ExpressionResult( + kind="notebook_code", + value=f"datetime.now(timezone.utc).strftime('{python_format}')", + imports=["from datetime import datetime, timezone"], + ) + return ExpressionResult( + kind="notebook_code", + value="datetime.now(timezone.utc).isoformat()", + imports=["from datetime import datetime, timezone"], + ) + + +def _convert_date_format(adf_format: str) -> str: + """Converts an ADF .NET date format string to Python strftime format. + + Args: + adf_format: ADF format string (e.g., ``"yyyy-MM-dd"``). + + Returns: + Python strftime format string (e.g., ``"%Y-%m-%d"``). + """ + result = adf_format + for adf_token, python_token in sorted(_DATE_FORMAT_MAP.items(), key=lambda x: -len(x[0])): + result = result.replace(adf_token, python_token) + return result + + +def _resolve_concat( + expr: str, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> ExpressionResult | None: + """Resolves ``concat(arg1, arg2, ...)`` -> notebook_code.""" + match = _CONCAT_RE.match(expr) + if match is None: + return None + + inner = match.group(1).strip() + parts: list[str] = _split_concat_args(inner) + if not parts: + return None + + all_imports: list[str] = [] + code_parts: list[str] = [] + all_required_parameters: dict[str, str] = {} + + for part in parts: + part = part.strip() + if not part: + continue + if part.startswith("'") and part.endswith("'"): + code_parts.append(repr(part[1:-1])) + else: + sub_result = resolve_expression("@" + part, context, variable_task_keys=variable_task_keys) + if sub_result is None: + return None + if sub_result.kind == "literal": + code_parts.append(repr(sub_result.value)) + elif sub_result.kind == "dab_ref": + code_parts.append(_dab_ref_to_widget_code(sub_result.value)) + widget_name, dab_ref = _required_parameter_for_ref(sub_result.value) + all_required_parameters.setdefault(widget_name, dab_ref) + elif sub_result.kind == "notebook_code": + code_parts.append(f"str({sub_result.value})") + all_imports.extend(sub_result.imports) + all_required_parameters.update(sub_result.required_parameters) + + if not code_parts: + return None + + value = " + ".join(code_parts) + return ExpressionResult( + kind="notebook_code", + value=value, + imports=list(dict.fromkeys(all_imports)), + required_parameters=all_required_parameters, + ) + + +def _dab_ref_to_widget_code(dab_ref: str) -> str: + """Converts a DAB ref like ``{{tasks.X.values.Y}}`` to widget get code.""" + widget_name, _ = _required_parameter_for_ref(dab_ref) + return f"dbutils.widgets.get('{widget_name}')" + + +def _required_parameter_for_ref(dab_ref: str) -> tuple[str, str]: + """Return ``(widget_name, dab_ref)`` for a DAB dynamic value reference.""" + inner = dab_ref.strip("{}") + widget_name = inner.split(".")[-1] + return widget_name, dab_ref + + +def _split_concat_args(inner: str) -> list[str]: + """Splits concat arguments respecting nested parentheses and quoted strings. + + Args: + inner: The inner content of ``concat(...)``. + + Returns: + List of argument strings. + """ + return _split_args(inner) + + +def _split_args(inner: str) -> list[str]: + """Splits function arguments respecting nested parentheses and quoted strings. + + Args: + inner: The inner content between the outermost parentheses. + + Returns: + List of argument strings. + """ + parts: list[str] = [] + depth = 0 + current: list[str] = [] + in_quote = False + + for char in inner: + if char == "'" and depth == 0: + in_quote = not in_quote + current.append(char) + elif in_quote: + current.append(char) + elif char == "(": + depth += 1 + current.append(char) + elif char == ")": + depth -= 1 + current.append(char) + elif char == "," and depth == 0: + parts.append("".join(current).strip()) + current = [] + else: + current.append(char) + + if current: + parts.append("".join(current).strip()) + + return parts + + +def _resolve_function_call( + expr: str, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> ExpressionResult | None: + """Resolves a generic ADF function call via the dispatch table.""" + match = _FUNCTION_CALL_RE.match(expr) + if match is None: + return None + + func_name = match.group(1) + inner = (match.group(2) or "").strip() + + handler = _FUNCTION_HANDLERS.get(func_name) + if handler is None: + handler = _FUNCTION_HANDLERS_CI.get(func_name.lower()) + if handler is None: + return None + + if not inner: + raw_args: list[str] = [] + else: + raw_args = _split_args(inner) + + resolved_args: list[ExpressionResult] = [] + for raw_arg in raw_args: + raw_arg = raw_arg.strip() + if not raw_arg: + continue + + if (raw_arg.startswith("'") and raw_arg.endswith("'")) or (raw_arg.startswith('"') and raw_arg.endswith('"')): + resolved_args.append(ExpressionResult(kind="literal", value=raw_arg[1:-1])) + elif _is_numeric(raw_arg): + resolved_args.append(ExpressionResult(kind="literal", value=raw_arg)) + elif raw_arg.lower() in ("true", "false"): + resolved_args.append( + ExpressionResult(kind="literal", value="True" if raw_arg.lower() == "true" else "False") + ) + elif raw_arg.lower() == "null": + resolved_args.append(ExpressionResult(kind="literal", value="None")) + else: + sub_expr = raw_arg if raw_arg.startswith("@") else "@" + raw_arg + sub_result = resolve_expression(sub_expr, context, variable_task_keys=variable_task_keys) + if sub_result is None: + return None + resolved_args.append(sub_result) + + handler_result = handler(resolved_args) + # Auto-propagate required_parameters from args onto notebook_code results + # so preparers can thread DAB refs into base_parameters even for handlers + # that pre-date the required_parameters contract. + if handler_result is not None and handler_result.kind == "notebook_code": + extra_parameters = _collect_required_parameters(*resolved_args) + if extra_parameters: + merged = dict(extra_parameters) + merged.update(handler_result.required_parameters) + handler_result = ExpressionResult( + kind=handler_result.kind, + value=handler_result.value, + imports=handler_result.imports, + required_parameters=merged, + ) + return handler_result + + +def _is_numeric(text: str) -> bool: + """Check if a string is a numeric literal.""" + try: + float(text) + return True + except ValueError: + return False + + +def _arg_to_code(arg: ExpressionResult) -> str: + """Converts a resolved argument to a Python code snippet.""" + if arg.kind == "literal": + if arg.value in ("True", "False", "None") or _is_numeric(arg.value): + return arg.value + return repr(arg.value) + elif arg.kind == "dab_ref": + return _dab_ref_to_widget_code(arg.value) + elif arg.kind == "notebook_code": + return arg.value + return repr(arg.value) + + +def _datetime_arg_code(arg: ExpressionResult) -> str: + """Return Python code that produces a ``datetime`` object from *arg*. + + Examples: + utcNow() -> ``datetime.now(timezone.utc)`` + '2024-01-01T00:00:00' -> ``datetime.fromisoformat('2024-01-01T00:00:00')`` + @pipeline().parameters.start_date -> ``datetime.fromisoformat(dbutils.widgets.get('start_date'))`` + """ + if arg.kind == "notebook_code": + # If the value already produces a datetime object that was just + # converted to ISO via .isoformat(), drop the conversion. + if arg.value.endswith(".isoformat()"): + return arg.value[: -len(".isoformat()")] + # If the value already ends in .strftime(...), the upstream caller + # was using it as a string; we still need a datetime, so parse it. + return f"datetime.fromisoformat({_arg_to_code(arg)})" + + +def _collect_imports(*args: ExpressionResult) -> list[str]: + """Collects unique imports from resolved arguments.""" + imports: list[str] = [] + for arg in args: + imports.extend(arg.imports) + return list(dict.fromkeys(imports)) + + +def _collect_required_parameters(*args: ExpressionResult) -> dict[str, str]: + """Collects widget → DAB ref mappings across resolved arguments.""" + merged: dict[str, str] = {} + for arg in args: + merged.update(arg.required_parameters) + if arg.kind == "dab_ref": + widget_name, dab_ref = _required_parameter_for_ref(arg.value) + merged.setdefault(widget_name, dab_ref) + return merged + + +def _result_from_args( + value: str, + args: list[ExpressionResult], + *, + extra_imports: list[str] | None = None, +) -> ExpressionResult: + """Builds a ``notebook_code`` result that propagates imports and widget refs. + + Args: + value: The Python expression string for the result. + args: Resolved arguments of the surrounding ADF function call. + extra_imports: Imports the handler itself introduces (e.g. + ``_DATETIME_IMPORTS`` or ``_ZONEINFO_IMPORTS``) in addition to + those already declared by *args*. + """ + imports = list(extra_imports or ()) + _collect_imports(*args) + return ExpressionResult( + kind="notebook_code", + value=value, + imports=imports, + required_parameters=_collect_required_parameters(*args), + ) + + +def _handle_concat(args: list[ExpressionResult]) -> ExpressionResult | None: + """concat(a, b, ...) -> str(a) + str(b) + ...""" + if not args: + return None + parts = [f"str({_arg_to_code(a)})" for a in args] + return _result_from_args(" + ".join(parts), args) + + +def _handle_ends_with(args: list[ExpressionResult]) -> ExpressionResult | None: + """endsWith(text, search) -> str(text).endswith(str(search))""" + if len(args) != 2: + return None + return _result_from_args(f"str({_arg_to_code(args[0])}).endswith(str({_arg_to_code(args[1])}))", args) + + +def _handle_guid(args: list[ExpressionResult]) -> ExpressionResult | None: + """guid() -> str(uuid4()) or guid('N') -> no-dash variant.""" + if len(args) == 0: + return ExpressionResult( + kind="notebook_code", + value="str(__import__('uuid').uuid4())", + ) + if len(args) == 1 and args[0].kind == "literal" and args[0].value == "N": + return ExpressionResult( + kind="notebook_code", + value="str(__import__('uuid').uuid4()).replace('-', '')", + ) + return ExpressionResult( + kind="notebook_code", + value="str(__import__('uuid').uuid4())", + ) + + +def _handle_index_of(args: list[ExpressionResult]) -> ExpressionResult | None: + """indexOf(text, search) -> str(text).lower().find(str(search).lower())""" + if len(args) != 2: + return None + return _result_from_args(f"str({_arg_to_code(args[0])}).lower().find(str({_arg_to_code(args[1])}).lower())", args) + + +def _handle_last_index_of(args: list[ExpressionResult]) -> ExpressionResult | None: + """lastIndexOf(text, search) -> str(text).lower().rfind(str(search).lower())""" + if len(args) != 2: + return None + return _result_from_args(f"str({_arg_to_code(args[0])}).lower().rfind(str({_arg_to_code(args[1])}).lower())", args) + + +def _handle_replace(args: list[ExpressionResult]) -> ExpressionResult | None: + """replace(text, old, new) -> str(text).replace(str(old), str(new))""" + if len(args) != 3: + return None + return _result_from_args( + f"str({_arg_to_code(args[0])}).replace(str({_arg_to_code(args[1])}), str({_arg_to_code(args[2])}))", args + ) + + +def _handle_split(args: list[ExpressionResult]) -> ExpressionResult | None: + """split(text, delim) -> str(text).split(str(delim))""" + if len(args) != 2: + return None + return _result_from_args(f"str({_arg_to_code(args[0])}).split(str({_arg_to_code(args[1])}))", args) + + +def _handle_starts_with(args: list[ExpressionResult]) -> ExpressionResult | None: + """startsWith(text, search) -> str(text).lower().startswith(str(search).lower())""" + if len(args) != 2: + return None + return _result_from_args( + f"str({_arg_to_code(args[0])}).lower().startswith(str({_arg_to_code(args[1])}).lower())", args + ) + + +def _handle_substring(args: list[ExpressionResult]) -> ExpressionResult | None: + """substring(text, start, length) -> str(text)[int(start):int(start)+int(length)]""" + if len(args) != 3: + return None + text = _arg_to_code(args[0]) + start = _arg_to_code(args[1]) + length = _arg_to_code(args[2]) + return _result_from_args(f"str({text})[int({start}):int({start})+int({length})]", args) + + +def _handle_to_lower(args: list[ExpressionResult]) -> ExpressionResult | None: + """toLower(text) -> str(text).lower()""" + if len(args) != 1: + return None + return _result_from_args(f"str({_arg_to_code(args[0])}).lower()", args) + + +def _handle_to_upper(args: list[ExpressionResult]) -> ExpressionResult | None: + """toUpper(text) -> str(text).upper()""" + if len(args) != 1: + return None + return _result_from_args(f"str({_arg_to_code(args[0])}).upper()", args) + + +def _handle_trim(args: list[ExpressionResult]) -> ExpressionResult | None: + """trim(text) -> str(text).strip()""" + if len(args) != 1: + return None + return _result_from_args(f"str({_arg_to_code(args[0])}).strip()", args) + + +def _handle_contains(args: list[ExpressionResult]) -> ExpressionResult | None: + """contains(collection, value) -> (value in collection)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[1])} in {_arg_to_code(args[0])})", args) + + +def _handle_empty(args: list[ExpressionResult]) -> ExpressionResult | None: + """empty(collection) -> (len(collection) == 0)""" + if len(args) != 1: + return None + return _result_from_args(f"(len({_arg_to_code(args[0])}) == 0)", args) + + +def _handle_first(args: list[ExpressionResult]) -> ExpressionResult | None: + """first(collection) -> collection[0]""" + if len(args) != 1: + return None + return _result_from_args(f"{_arg_to_code(args[0])}[0]", args) + + +def _handle_intersection(args: list[ExpressionResult]) -> ExpressionResult | None: + """intersection(c1, c2, ...) -> list(set(c1) & set(c2) & ...)""" + if len(args) < 2: + return None + parts = " & ".join(f"set({_arg_to_code(a)})" for a in args) + return _result_from_args(f"list({parts})", args) + + +def _handle_join(args: list[ExpressionResult]) -> ExpressionResult | None: + """join(array, delim) -> str(delim).join(str(x) for x in array)""" + if len(args) != 2: + return None + return _result_from_args(f"str({_arg_to_code(args[1])}).join(str(x) for x in {_arg_to_code(args[0])})", args) + + +def _handle_last(args: list[ExpressionResult]) -> ExpressionResult | None: + """last(collection) -> collection[-1]""" + if len(args) != 1: + return None + return _result_from_args(f"{_arg_to_code(args[0])}[-1]", args) + + +def _handle_length(args: list[ExpressionResult]) -> ExpressionResult | None: + """length(collection) -> len(collection)""" + if len(args) != 1: + return None + return _result_from_args(f"len({_arg_to_code(args[0])})", args) + + +def _handle_skip(args: list[ExpressionResult]) -> ExpressionResult | None: + """skip(collection, count) -> collection[int(count):]""" + if len(args) != 2: + return None + return _result_from_args(f"{_arg_to_code(args[0])}[int({_arg_to_code(args[1])}):]", args) + + +def _handle_take(args: list[ExpressionResult]) -> ExpressionResult | None: + """take(collection, count) -> collection[:int(count)]""" + if len(args) != 2: + return None + return _result_from_args(f"{_arg_to_code(args[0])}[:int({_arg_to_code(args[1])})]", args) + + +def _handle_union(args: list[ExpressionResult]) -> ExpressionResult | None: + """union(c1, c2, ...) -> list(set(c1) | set(c2) | ...)""" + if len(args) < 2: + return None + parts = " | ".join(f"set({_arg_to_code(a)})" for a in args) + return _result_from_args(f"list({parts})", args) + + +def _handle_and(args: list[ExpressionResult]) -> ExpressionResult | None: + """and(a, b) -> (a and b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} and {_arg_to_code(args[1])})", args) + + +def _handle_equals(args: list[ExpressionResult]) -> ExpressionResult | None: + """equals(a, b) -> (a == b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} == {_arg_to_code(args[1])})", args) + + +def _handle_greater(args: list[ExpressionResult]) -> ExpressionResult | None: + """greater(a, b) -> (a > b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} > {_arg_to_code(args[1])})", args) + + +def _handle_greater_or_equals(args: list[ExpressionResult]) -> ExpressionResult | None: + """greaterOrEquals(a, b) -> (a >= b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} >= {_arg_to_code(args[1])})", args) + + +def _handle_if(args: list[ExpressionResult]) -> ExpressionResult | None: + """if(expr, trueVal, falseVal) -> (trueVal if expr else falseVal)""" + if len(args) != 3: + return None + return _result_from_args(f"({_arg_to_code(args[1])} if {_arg_to_code(args[0])} else {_arg_to_code(args[2])})", args) + + +def _handle_less(args: list[ExpressionResult]) -> ExpressionResult | None: + """less(a, b) -> (a < b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} < {_arg_to_code(args[1])})", args) + + +def _handle_less_or_equals(args: list[ExpressionResult]) -> ExpressionResult | None: + """lessOrEquals(a, b) -> (a <= b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} <= {_arg_to_code(args[1])})", args) + + +def _handle_not(args: list[ExpressionResult]) -> ExpressionResult | None: + """not(expr) -> (not expr)""" + if len(args) != 1: + return None + return _result_from_args(f"(not {_arg_to_code(args[0])})", args) + + +def _handle_or(args: list[ExpressionResult]) -> ExpressionResult | None: + """or(a, b) -> (a or b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} or {_arg_to_code(args[1])})", args) + + +def _handle_array(args: list[ExpressionResult]) -> ExpressionResult | None: + """array(value) -> [value]""" + if len(args) != 1: + return None + return _result_from_args(f"[{_arg_to_code(args[0])}]", args) + + +def _handle_base64(args: list[ExpressionResult]) -> ExpressionResult | None: + """base64(value) -> base64.b64encode(str(value).encode()).decode()""" + if len(args) != 1: + return None + return _result_from_args(f"__import__('base64').b64encode(str({_arg_to_code(args[0])}).encode()).decode()", args) + + +def _handle_base64_to_binary(args: list[ExpressionResult]) -> ExpressionResult | None: + """base64ToBinary(value) -> base64.b64decode(value)""" + if len(args) != 1: + return None + return _result_from_args(f"__import__('base64').b64decode({_arg_to_code(args[0])})", args) + + +def _handle_base64_to_string(args: list[ExpressionResult]) -> ExpressionResult | None: + """base64ToString(value) -> base64.b64decode(value).decode()""" + if len(args) != 1: + return None + return _result_from_args(f"__import__('base64').b64decode({_arg_to_code(args[0])}).decode()", args) + + +def _handle_binary(args: list[ExpressionResult]) -> ExpressionResult | None: + """binary(value) -> str(value).encode()""" + if len(args) != 1: + return None + return _result_from_args(f"str({_arg_to_code(args[0])}).encode()", args) + + +def _handle_bool(args: list[ExpressionResult]) -> ExpressionResult | None: + """bool(value) -> bool(value)""" + if len(args) != 1: + return None + return _result_from_args(f"bool({_arg_to_code(args[0])})", args) + + +def _handle_coalesce(args: list[ExpressionResult]) -> ExpressionResult | None: + """coalesce(a, b, ...) -> next((x for x in [a, b, ...] if x is not None), None)""" + if not args: + return None + items = ", ".join(_arg_to_code(a) for a in args) + return _result_from_args(f"next((x for x in [{items}] if x is not None), None)", args) + + +def _handle_create_array(args: list[ExpressionResult]) -> ExpressionResult | None: + """createArray(a, b, ...) -> [a, b, ...]""" + items = ", ".join(_arg_to_code(a) for a in args) + return _result_from_args(f"[{items}]", args) + + +def _handle_agentic(_args: list[ExpressionResult]) -> ExpressionResult | None: + """Return None for agentic functions that are too complex for deterministic translation.""" + return None + + +def _handle_decode_uri_component(args: list[ExpressionResult]) -> ExpressionResult | None: + """decodeUriComponent(value) -> urllib.parse.unquote(value)""" + if len(args) != 1: + return None + return _result_from_args(f"__import__('urllib.parse', fromlist=['unquote']).unquote({_arg_to_code(args[0])})", args) + + +def _handle_encode_uri_component(args: list[ExpressionResult]) -> ExpressionResult | None: + """encodeUriComponent(value) -> urllib.parse.quote(str(value), safe='')""" + if len(args) != 1: + return None + return _result_from_args( + f"__import__('urllib.parse', fromlist=['quote']).quote(str({_arg_to_code(args[0])}), safe='')", args + ) + + +def _handle_float(args: list[ExpressionResult]) -> ExpressionResult | None: + """float(value) -> float(value)""" + if len(args) != 1: + return None + return _result_from_args(f"float({_arg_to_code(args[0])})", args) + + +def _handle_int(args: list[ExpressionResult]) -> ExpressionResult | None: + """int(value) -> int(value)""" + if len(args) != 1: + return None + return _result_from_args(f"int({_arg_to_code(args[0])})", args) + + +def _handle_json(args: list[ExpressionResult]) -> ExpressionResult | None: + """json(value) -> json.loads(value)""" + if len(args) != 1: + return None + return _result_from_args(f"__import__('json').loads({_arg_to_code(args[0])})", args) + + +def _handle_string(args: list[ExpressionResult]) -> ExpressionResult | None: + """string(value) -> str(value).""" + if len(args) != 1: + return None + sole_arg = args[0] + if sole_arg.kind == "dab_ref": + return sole_arg + return _result_from_args(f"str({_arg_to_code(sole_arg)})", args) + + +def _handle_add(args: list[ExpressionResult]) -> ExpressionResult | None: + """add(a, b) -> (a + b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} + {_arg_to_code(args[1])})", args) + + +def _handle_div(args: list[ExpressionResult]) -> ExpressionResult | None: + """div(a, b) -> (a // b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} // {_arg_to_code(args[1])})", args) + + +def _handle_max(args: list[ExpressionResult]) -> ExpressionResult | None: + """max(a, b, ...) -> max(a, b, ...)""" + if not args: + return None + items = ", ".join(_arg_to_code(a) for a in args) + return _result_from_args(f"max({items})", args) + + +def _handle_min(args: list[ExpressionResult]) -> ExpressionResult | None: + """min(a, b, ...) -> min(a, b, ...)""" + if not args: + return None + items = ", ".join(_arg_to_code(a) for a in args) + return _result_from_args(f"min({items})", args) + + +def _handle_mod(args: list[ExpressionResult]) -> ExpressionResult | None: + """mod(a, b) -> (a % b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} % {_arg_to_code(args[1])})", args) + + +def _handle_mul(args: list[ExpressionResult]) -> ExpressionResult | None: + """mul(a, b) -> (a * b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} * {_arg_to_code(args[1])})", args) + + +def _handle_rand(args: list[ExpressionResult]) -> ExpressionResult | None: + """rand(min, max) -> random.randint(min, max-1)""" + if len(args) != 2: + return None + return _result_from_args( + f"__import__('random').randint({_arg_to_code(args[0])}, {_arg_to_code(args[1])} - 1)", args + ) + + +def _handle_range(args: list[ExpressionResult]) -> ExpressionResult | None: + """range(start, count) -> list(range(start, start + count))""" + if len(args) != 2: + return None + start = _arg_to_code(args[0]) + count = _arg_to_code(args[1]) + return _result_from_args(f"list(range({start}, {start} + {count}))", args) + + +def _handle_sub(args: list[ExpressionResult]) -> ExpressionResult | None: + """sub(a, b) -> (a - b)""" + if len(args) != 2: + return None + return _result_from_args(f"({_arg_to_code(args[0])} - {_arg_to_code(args[1])})", args) + + +def _make_add_unit_handler( + timedelta_keyword: str, +) -> Callable[[list[ExpressionResult]], ExpressionResult | None]: + """Builds a handler for ``addDays`` / ``addHours`` / ``addMinutes`` / ``addSeconds``.""" + + def handler(args: list[ExpressionResult]) -> ExpressionResult | None: + if len(args) < 2 or len(args) > 3: + return None + timestamp_dt = _datetime_arg_code(args[0]) + amount = _arg_to_code(args[1]) + format_string = _get_format_arg(args, 2) + return _result_from_args( + f"({timestamp_dt} + timedelta({timedelta_keyword}={amount})).strftime({format_string})", + args, + extra_imports=_DATETIME_IMPORTS, + ) + + return handler + + +_handle_add_days = _make_add_unit_handler("days") +_handle_add_hours = _make_add_unit_handler("hours") +_handle_add_minutes = _make_add_unit_handler("minutes") +_handle_add_seconds = _make_add_unit_handler("seconds") + + +def _handle_add_to_time(args: list[ExpressionResult]) -> ExpressionResult | None: + """addToTime(ts, interval, unit, fmt?) -> datetime + timedelta.""" + if len(args) < 3 or len(args) > 4: + return None + timestamp_dt = _datetime_arg_code(args[0]) + interval = _arg_to_code(args[1]) + unit_str = args[2].value if args[2].kind == "literal" else None + if unit_str is None: + return None + timedelta_keyword = _TIME_UNIT_MAP.get(unit_str) + if timedelta_keyword is None: + return None + format_string = _get_format_arg(args, 3) + return _result_from_args( + (f"({timestamp_dt} + timedelta({timedelta_keyword}={interval})).strftime({format_string})"), + args, + extra_imports=_DATETIME_IMPORTS, + ) + + +def _handle_day_of_month(args: list[ExpressionResult]) -> ExpressionResult | None: + """dayOfMonth(ts) -> .day""" + if len(args) != 1: + return None + return _result_from_args(f"{_datetime_arg_code(args[0])}.day", args, extra_imports=_DATETIME_IMPORTS) + + +def _handle_day_of_week(args: list[ExpressionResult]) -> ExpressionResult | None: + """dayOfWeek(ts) -> .isoweekday() % 7 (ADF: 0=Sunday)""" + if len(args) != 1: + return None + return _result_from_args(f"{_datetime_arg_code(args[0])}.isoweekday() % 7", args, extra_imports=_DATETIME_IMPORTS) + + +def _handle_day_of_year(args: list[ExpressionResult]) -> ExpressionResult | None: + """dayOfYear(ts) -> .timetuple().tm_yday""" + if len(args) != 1: + return None + return _result_from_args( + f"{_datetime_arg_code(args[0])}.timetuple().tm_yday", args, extra_imports=_DATETIME_IMPORTS + ) + + +def _handle_format_date_time(args: list[ExpressionResult]) -> ExpressionResult | None: + """formatDateTime(ts, fmt?) -> datetime.fromisoformat(ts).strftime(converted_fmt)""" + if len(args) < 1 or len(args) > 2: + return None + timestamp_dt = _datetime_arg_code(args[0]) + if len(args) == 2 and args[1].kind == "literal": + python_format = _convert_date_format(args[1].value) + return _result_from_args(f"{timestamp_dt}.strftime('{python_format}')", args, extra_imports=_DATETIME_IMPORTS) + return _result_from_args(f"{timestamp_dt}.isoformat()", args, extra_imports=_DATETIME_IMPORTS) + + +def _make_now_offset_handler( + operator: str, +) -> Callable[[list[ExpressionResult]], ExpressionResult | None]: + """Builds a ``getFutureTime`` / ``getPastTime`` handler.""" + + def handler(args: list[ExpressionResult]) -> ExpressionResult | None: + if len(args) < 2 or len(args) > 3: + return None + interval = _arg_to_code(args[0]) + unit_str = args[1].value if args[1].kind == "literal" else None + if unit_str is None: + return None + timedelta_keyword = _TIME_UNIT_MAP.get(unit_str) + if timedelta_keyword is None: + return None + format_string = _get_format_arg(args, 2) + return ExpressionResult( + kind="notebook_code", + value=( + f"(datetime.now(timezone.utc) {operator} " + f"timedelta({timedelta_keyword}={interval})).strftime({format_string})" + ), + imports=_DATETIME_IMPORTS + _collect_imports(*args), + ) + + return handler + + +_handle_get_future_time = _make_now_offset_handler("+") +_handle_get_past_time = _make_now_offset_handler("-") + + +def _handle_start_of_day(args: list[ExpressionResult]) -> ExpressionResult | None: + """startOfDay(ts, fmt?) -> datetime.fromisoformat(ts).replace(hour=0,...).strftime(fmt)""" + if len(args) < 1 or len(args) > 2: + return None + timestamp_dt = _datetime_arg_code(args[0]) + format_string = _get_format_arg(args, 1) + return _result_from_args( + (f"{timestamp_dt}.replace(hour=0, minute=0, second=0, microsecond=0).strftime({format_string})"), + args, + extra_imports=_DATETIME_IMPORTS, + ) + + +def _handle_start_of_hour(args: list[ExpressionResult]) -> ExpressionResult | None: + """startOfHour(ts, fmt?) -> datetime.fromisoformat(ts).replace(minute=0,...).strftime(fmt)""" + if len(args) < 1 or len(args) > 2: + return None + timestamp_dt = _datetime_arg_code(args[0]) + format_string = _get_format_arg(args, 1) + return _result_from_args( + (f"{timestamp_dt}.replace(minute=0, second=0, microsecond=0).strftime({format_string})"), + args, + extra_imports=_DATETIME_IMPORTS, + ) + + +def _handle_start_of_month(args: list[ExpressionResult]) -> ExpressionResult | None: + """startOfMonth(ts, fmt?) -> datetime.fromisoformat(ts).replace(day=1,...).strftime(fmt)""" + if len(args) < 1 or len(args) > 2: + return None + timestamp_dt = _datetime_arg_code(args[0]) + format_string = _get_format_arg(args, 1) + return _result_from_args( + (f"{timestamp_dt}.replace(day=1, hour=0, minute=0, second=0, microsecond=0).strftime({format_string})"), + args, + extra_imports=_DATETIME_IMPORTS, + ) + + +def _handle_subtract_from_time(args: list[ExpressionResult]) -> ExpressionResult | None: + """subtractFromTime(ts, interval, unit, fmt?) -> datetime - timedelta.""" + if len(args) < 3 or len(args) > 4: + return None + timestamp_dt = _datetime_arg_code(args[0]) + interval = _arg_to_code(args[1]) + unit_str = args[2].value if args[2].kind == "literal" else None + if unit_str is None: + return None + timedelta_keyword = _TIME_UNIT_MAP.get(unit_str) + if timedelta_keyword is None: + return None + format_string = _get_format_arg(args, 3) + return _result_from_args( + (f"({timestamp_dt} - timedelta({timedelta_keyword}={interval})).strftime({format_string})"), + args, + extra_imports=_DATETIME_IMPORTS, + ) + + +def _get_format_arg(args: list[ExpressionResult], idx: int) -> str: + """Extracts a format argument from the args list, converting ADF .NET format if needed.""" + if idx < len(args) and args[idx].kind == "literal" and args[idx].value: + python_format = _convert_date_format(args[idx].value) + return repr(python_format) + return "'%Y-%m-%dT%H:%M:%SZ'" + + +_ZONEINFO_IMPORTS = ["from datetime import datetime, timezone", "from zoneinfo import ZoneInfo"] + + +def _handle_convert_from_utc(args: list[ExpressionResult]) -> ExpressionResult | None: + """convertFromUtc(timestamp, destinationTimeZone, fmt?)""" + if len(args) < 2 or len(args) > 3: + return None + src = _datetime_arg_code(args[0]) + dest_tz = _arg_to_code(args[1]) + fmt = _get_format_arg(args, 2) + return _result_from_args( + f"{src}.replace(tzinfo=timezone.utc).astimezone(ZoneInfo({dest_tz})).strftime({fmt})", + args, + extra_imports=_ZONEINFO_IMPORTS, + ) + + +def _handle_convert_to_utc(args: list[ExpressionResult]) -> ExpressionResult | None: + """convertToUtc(timestamp, sourceTimeZone, fmt?) -> UTC datetime.""" + if len(args) < 2 or len(args) > 3: + return None + src = _datetime_arg_code(args[0]) + src_tz = _arg_to_code(args[1]) + fmt = _get_format_arg(args, 2) + return _result_from_args( + f"{src}.replace(tzinfo=ZoneInfo({src_tz})).astimezone(timezone.utc).strftime({fmt})", + args, + extra_imports=_ZONEINFO_IMPORTS, + ) + + +def _handle_convert_time_zone(args: list[ExpressionResult]) -> ExpressionResult | None: + """convertTimeZone(timestamp, sourceTimeZone, destinationTimeZone, fmt?).""" + if len(args) < 3 or len(args) > 4: + return None + src = _datetime_arg_code(args[0]) + src_tz = _arg_to_code(args[1]) + dst_tz = _arg_to_code(args[2]) + fmt = _get_format_arg(args, 3) + return _result_from_args( + (f"{src}.replace(tzinfo=ZoneInfo({src_tz})).astimezone(ZoneInfo({dst_tz})).strftime({fmt})"), + args, + extra_imports=_ZONEINFO_IMPORTS, + ) + + +def _handle_ticks(args: list[ExpressionResult]) -> ExpressionResult | None: + """ticks(timestamp) -> .NET FILETIME ticks (100-ns intervals since 0001-01-01).""" + if len(args) != 1: + return None + src = _datetime_arg_code(args[0]) + return _result_from_args( + (f"int(({src} - datetime(1, 1, 1, tzinfo=timezone.utc)).total_seconds() * 10_000_000)"), + args, + extra_imports=_DATETIME_IMPORTS, + ) + + +_FUNCTION_HANDLERS: dict[str, Callable[[list[ExpressionResult]], ExpressionResult | None]] = { + "concat": _handle_concat, + "endsWith": _handle_ends_with, + "guid": _handle_guid, + "indexOf": _handle_index_of, + "lastIndexOf": _handle_last_index_of, + "replace": _handle_replace, + "split": _handle_split, + "startsWith": _handle_starts_with, + "substring": _handle_substring, + "toLower": _handle_to_lower, + "toUpper": _handle_to_upper, + "trim": _handle_trim, + "contains": _handle_contains, + "empty": _handle_empty, + "first": _handle_first, + "intersection": _handle_intersection, + "join": _handle_join, + "last": _handle_last, + "length": _handle_length, + "skip": _handle_skip, + "take": _handle_take, + "union": _handle_union, + "and": _handle_and, + "equals": _handle_equals, + "greater": _handle_greater, + "greaterOrEquals": _handle_greater_or_equals, + "if": _handle_if, + "less": _handle_less, + "lessOrEquals": _handle_less_or_equals, + "not": _handle_not, + "or": _handle_or, + "array": _handle_array, + "base64": _handle_base64, + "base64ToBinary": _handle_base64_to_binary, + "base64ToString": _handle_base64_to_string, + "binary": _handle_binary, + "bool": _handle_bool, + "coalesce": _handle_coalesce, + "createArray": _handle_create_array, + "dataUri": _handle_agentic, + "dataUriToBinary": _handle_agentic, + "dataUriToString": _handle_agentic, + "decodeBase64": _handle_base64_to_string, + "decodeDataUri": _handle_agentic, + "decodeUriComponent": _handle_decode_uri_component, + "encodeUriComponent": _handle_encode_uri_component, + "float": _handle_float, + "int": _handle_int, + "json": _handle_json, + "string": _handle_string, + "uriComponent": _handle_encode_uri_component, + "uriComponentToBinary": _handle_agentic, + "uriComponentToString": _handle_decode_uri_component, + "xml": _handle_agentic, + "xpath": _handle_agentic, + "add": _handle_add, + "div": _handle_div, + "max": _handle_max, + "min": _handle_min, + "mod": _handle_mod, + "mul": _handle_mul, + "rand": _handle_rand, + "range": _handle_range, + "sub": _handle_sub, + "addDays": _handle_add_days, + "addHours": _handle_add_hours, + "addMinutes": _handle_add_minutes, + "addSeconds": _handle_add_seconds, + "addToTime": _handle_add_to_time, + "convertFromUtc": _handle_convert_from_utc, + "convertTimeZone": _handle_convert_time_zone, + "convertToUtc": _handle_convert_to_utc, + "dayOfMonth": _handle_day_of_month, + "dayOfWeek": _handle_day_of_week, + "dayOfYear": _handle_day_of_year, + "formatDateTime": _handle_format_date_time, + "getFutureTime": _handle_get_future_time, + "getPastTime": _handle_get_past_time, + "startOfDay": _handle_start_of_day, + "startOfHour": _handle_start_of_hour, + "startOfMonth": _handle_start_of_month, + "subtractFromTime": _handle_subtract_from_time, + "ticks": _handle_ticks, + # utcNow is intentionally absent -- handled by _resolve_utcnow upstream + # in resolve_expression() before the generic dispatch is reached. +} + +_FUNCTION_HANDLERS_CI: dict[str, Callable[[list[ExpressionResult]], ExpressionResult | None]] = { + k.lower(): v for k, v in _FUNCTION_HANDLERS.items() if v is not None +} diff --git a/src/orchestra/preparer/__init__.py b/src/orchestra/preparer/__init__.py new file mode 100644 index 0000000..992bcd6 --- /dev/null +++ b/src/orchestra/preparer/__init__.py @@ -0,0 +1,15 @@ +"""Preparer layer: convert translated IR into deployable DAB artifacts.""" + +from flowx.preparer.workflow_preparer import ( + PreparedActivity, + PreparedWorkflow, + prepare_activity, + prepare_workflow, +) + +__all__ = [ + "PreparedActivity", + "PreparedWorkflow", + "prepare_activity", + "prepare_workflow", +] diff --git a/src/orchestra/preparer/activity_preparers/__init__.py b/src/orchestra/preparer/activity_preparers/__init__.py new file mode 100644 index 0000000..6ad9920 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/__init__.py @@ -0,0 +1 @@ +"""Activity preparers: one module per IR activity type.""" diff --git a/src/orchestra/preparer/activity_preparers/append_variable.py b/src/orchestra/preparer/activity_preparers/append_variable.py new file mode 100644 index 0000000..f69742c --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/append_variable.py @@ -0,0 +1,38 @@ +"""Preparer for AppendVariableActivity -> notebook_task with generated notebook.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.preparer.activity_preparers.helpers import build_notebook_activity_task +from flowx.preparer.activity_preparers.naming import notebook_filename +from flowx.preparer.code_generator import generate_append_variable_notebook +from flowx.preparer.workflow_preparer import PreparedActivity + +if TYPE_CHECKING: + from flowx.models.ir import AppendVariableActivity + + +def prepare( + activity: AppendVariableActivity, + *, + scope: str = "", + variable_task_keys: dict[str, str] | None = None, +) -> PreparedActivity: + """Converts an AppendVariableActivity into a notebook_task that appends to an array.""" + base_parameters: dict[str, str] = { + "variable_name": activity.variable_name, + "source_task_key": (variable_task_keys or {}).get(activity.variable_name, ""), + } + if activity.value_kind in ("literal", "dab_ref"): + base_parameters["value"] = activity.append_value + for widget_name, dab_ref in activity.required_parameters.items(): + base_parameters.setdefault(widget_name, dab_ref) + + task, notebooks = build_notebook_activity_task( + activity, + notebook_relative_path=f"notebooks/{notebook_filename(activity.task_key, activity.name)}", + notebook_content=generate_append_variable_notebook(activity), + base_parameters=base_parameters, + ) + return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/activity_preparers/copy.py b/src/orchestra/preparer/activity_preparers/copy.py new file mode 100644 index 0000000..fb648ca --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/copy.py @@ -0,0 +1,175 @@ +"""Preparer for CopyActivity -> notebook_task with generated copy notebook.""" + +from __future__ import annotations + +import dataclasses +import re +from dataclasses import dataclass + +from flowx.models.dab import SecretInstruction, SetupTask +from flowx.models.ir import CopyActivity +from flowx.models.source_types import FILE_SOURCE_TYPES, JDBC_SOURCE_TYPES +from flowx.preparer.activity_preparers.helpers import ( + build_notebook_activity_task, + make_jdbc_secrets, +) +from flowx.preparer.activity_preparers.naming import notebook_filename +from flowx.preparer.code_generator import generate_copy_notebook +from flowx.preparer.workflow_preparer import PreparedActivity + +_ABFSS_URL_RE = re.compile(r"abfss://([^@]+)@([^/]+)/?(.*)") +_VOLUME_NAME_SANITIZE_RE = re.compile(r"[^a-zA-Z0-9_]") + + +@dataclass(frozen=True, slots=True) +class _VolumeBinding: + """Resolved UC volume info derived from a file source's resolved ABFSS URL.""" + + volume_name: str + container_location: str + volume_base: str + source_path: str + + +def _resolve_volume_binding(source_properties: dict) -> _VolumeBinding | None: + """Derives UC volume info from a file source's resolved ABFSS URL.""" + resolved_path = source_properties.get("resolved_path", "") + if not resolved_path: + return None + + match = _ABFSS_URL_RE.match(resolved_path) + if not match: + return None + + container, storage_account, folder_path = match.group(1), match.group(2), match.group(3).rstrip("/") + volume_name = _VOLUME_NAME_SANITIZE_RE.sub("_", container) + volume_base = f"/Volumes/${{var.catalog}}/${{var.schema}}/{volume_name}" + return _VolumeBinding( + volume_name=volume_name, + container_location=f"abfss://{container}@{storage_account}", + volume_base=volume_base, + source_path=f"{volume_base}/{folder_path}" if folder_path else volume_base, + ) + + +def _augment_with_volume_paths(activity: CopyActivity, binding: _VolumeBinding) -> CopyActivity: + """Returns a copy of *activity* with volume paths threaded into source_properties.""" + augmented_properties = { + **(activity.source_properties or {}), + "volume_path": binding.source_path, + "volume_base": binding.volume_base, + } + return dataclasses.replace(activity, source_properties=augmented_properties) + + +def prepare(activity: CopyActivity, *, scope: str = "") -> PreparedActivity: + """Converts a CopyActivity into a notebook_task with a generated copy notebook.""" + source_type = activity.source_type or "" + volume_binding: _VolumeBinding | None = None + if source_type in FILE_SOURCE_TYPES: + volume_binding = _resolve_volume_binding(activity.source_properties or {}) + if volume_binding is not None: + activity = _augment_with_volume_paths(activity, volume_binding) + + base_parameters: dict[str, str] = {} + if activity.source_type: + base_parameters["source_type"] = activity.source_type + if activity.sink_type: + base_parameters["sink_type"] = activity.sink_type + source_path = ( + volume_binding.source_path + if volume_binding is not None + else (activity.source_properties or {}).get("resolved_path") + ) + if source_path: + base_parameters["source_path"] = source_path + + task, notebooks = build_notebook_activity_task( + activity, + notebook_relative_path=f"notebooks/{notebook_filename(activity.task_key, activity.name)}", + notebook_content=generate_copy_notebook(activity, scope=scope), + base_parameters=base_parameters, + ) + + scope_name = scope or activity.task_key + secrets = _build_secrets(activity, source_type, scope_name) + setup_tasks = _build_setup_tasks(activity, source_type, volume_binding) + + return PreparedActivity(task=task, notebooks=notebooks, secrets=secrets, setup_tasks=setup_tasks) + + +def _build_secrets(activity: CopyActivity, source_type: str, scope_name: str) -> list[SecretInstruction]: + """Returns the SecretInstructions a Copy activity needs to deploy.""" + if source_type in JDBC_SOURCE_TYPES: + return make_jdbc_secrets( + scope_name=scope_name, + source_type=source_type, + activity_name=activity.name, + role="source", + ) + if source_type not in FILE_SOURCE_TYPES: + return [] + source_properties = activity.source_properties or {} + if not (source_properties.get("connection_string") or source_properties.get("sasUri")): + return [] + return [ + SecretInstruction( + scope=scope_name, + key="connection-string", + value_source=f"Connection string for {source_type} source in activity '{activity.name}'", + ) + ] + + +def _build_setup_tasks( + activity: CopyActivity, + source_type: str, + volume_binding: _VolumeBinding | None, +) -> list[SetupTask]: + """Returns the SetupTasks (UC volumes etc.) a Copy activity needs.""" + setup_tasks: list[SetupTask] = [] + if source_type in FILE_SOURCE_TYPES and volume_binding is not None: + setup_tasks.append( + SetupTask( + type="volume", + config={ + "volume_name": volume_binding.volume_name, + "volume_type": "EXTERNAL", + "location": volume_binding.container_location, + }, + ) + ) + + sink_volume = _build_sink_volume_setup_task(activity) + if sink_volume is not None: + setup_tasks.append(sink_volume) + + return setup_tasks + + +def _build_sink_volume_setup_task(activity: CopyActivity) -> SetupTask | None: + """Returns a UC volume SetupTask for the sink side of *activity*, or None. + + Sink-side volume info is populated by the Copy translator when the + output dataset resolves to a cloud-storage location. The setup task + creates the External Location and External Volume so the generated + notebook can write into ``/Volumes////...``. + """ + sink_properties = activity.sink_properties or {} + volume_name = sink_properties.get("volume_name") + external_location = sink_properties.get("volume_external_location") + if not volume_name or not external_location: + return None + return SetupTask( + type="volume", + config={ + "volume_name": volume_name, + "volume_type": "EXTERNAL", + "location": external_location, + # ``location_type`` drives the storage-credential DDL the setup + # notebook emits (Azure managed identity vs S3 IAM vs GCS service + # account); omitting it leaves the user a manual TODO. + "location_type": sink_properties.get("volume_location_type", ""), + "storage_account": sink_properties.get("volume_storage_account", ""), + }, + ) diff --git a/src/orchestra/preparer/activity_preparers/databricks_job.py b/src/orchestra/preparer/activity_preparers/databricks_job.py new file mode 100644 index 0000000..eddde4e --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/databricks_job.py @@ -0,0 +1,58 @@ +"""Preparer for RunJobActivity -> run_job_task dict.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.models.ir import TranslationContext +from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string +from flowx.preparer.workflow_preparer import PreparedActivity, build_common_task_fields + +if TYPE_CHECKING: + from flowx.models.ir import RunJobActivity + + +def _resolve_param_value(value: str) -> str: + """Resolves an ADF expression parameter value to a DAB ref. + + Args: + value: A parameter value string that may contain ADF expressions. + + Returns: + Resolved value string. + """ + context = TranslationContext() + + if "@{" in value: + return resolve_interpolated_string(value, context) + + if value.startswith("@"): + result = resolve_expression(value, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + + return value + + +def prepare(activity: RunJobActivity, *, scope: str = "") -> PreparedActivity: + """Converts a RunJobActivity into a DAB run_job_task definition. + + Args: + activity: The translated run-job activity from the IR. + + Returns: + A PreparedActivity containing the run_job_task dict. + """ + task = build_common_task_fields(activity) + run_job: dict = {} + + if activity.existing_job_id: + run_job["job_id"] = activity.existing_job_id + elif activity.job_name: + run_job["job_id"] = f"${{resources.jobs.{activity.job_name}.id}}" + + if activity.job_parameters: + run_job["job_parameters"] = {k: _resolve_param_value(str(v)) for k, v in activity.job_parameters.items()} + + task["run_job_task"] = run_job + return PreparedActivity(task=task) diff --git a/src/orchestra/preparer/activity_preparers/delete.py b/src/orchestra/preparer/activity_preparers/delete.py new file mode 100644 index 0000000..ca90b31 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/delete.py @@ -0,0 +1,34 @@ +"""Preparer for DeleteActivity -> notebook_task with generated delete notebook.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.preparer.activity_preparers.helpers import ( + build_notebook_activity_task, + resolve_param_value, +) +from flowx.preparer.activity_preparers.naming import notebook_filename +from flowx.preparer.code_generator import generate_delete_notebook +from flowx.preparer.workflow_preparer import PreparedActivity + +if TYPE_CHECKING: + from flowx.models.ir import DeleteActivity + + +def prepare(activity: DeleteActivity, *, scope: str = "") -> PreparedActivity: + """Converts a DeleteActivity into a notebook_task with a generated delete notebook.""" + base_parameters = { + "dataset_name": resolve_param_value(activity.dataset_name), + "recursive": str(activity.recursive).lower(), + } + if activity.folder_path: + base_parameters["folder_path"] = resolve_param_value(activity.folder_path) + + task, notebooks = build_notebook_activity_task( + activity, + notebook_relative_path=f"notebooks/{notebook_filename(activity.task_key, activity.name)}", + notebook_content=generate_delete_notebook(activity), + base_parameters=base_parameters, + ) + return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/activity_preparers/execute_pipeline.py b/src/orchestra/preparer/activity_preparers/execute_pipeline.py new file mode 100644 index 0000000..d43eadc --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/execute_pipeline.py @@ -0,0 +1,55 @@ +"""Preparer for ExecutePipelineActivity -> run_job_task dict.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.models.ir import TranslationContext +from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string +from flowx.preparer.workflow_preparer import PreparedActivity, build_common_task_fields +from flowx.utils import normalize_task_key + +if TYPE_CHECKING: + from flowx.models.ir import ExecutePipelineActivity + + +def _resolve_param_value(value: str) -> str: + """Resolves an ADF expression parameter value to a DAB ref. + + Args: + value: A parameter value string that may contain ADF expressions. + + Returns: + Resolved value string. + """ + s = str(value) + context = TranslationContext() + + if "@{" in s: + return resolve_interpolated_string(s, context) + + if s.startswith("@"): + result = resolve_expression(s, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + + return s + + +def prepare(activity: ExecutePipelineActivity, *, scope: str = "") -> PreparedActivity: + """Converts an ExecutePipelineActivity into a DAB run_job_task definition. + + Args: + activity: The translated execute-pipeline activity from the IR. + + Returns: + A PreparedActivity containing the run_job_task dict. + """ + task = build_common_task_fields(activity) + resource_key = normalize_task_key(activity.pipeline_name) + task["run_job_task"] = { + "job_id": f"${{resources.jobs.{resource_key}.id}}", + } + if activity.parameters: + task["run_job_task"]["job_parameters"] = {k: _resolve_param_value(v) for k, v in activity.parameters.items()} + return PreparedActivity(task=task) diff --git a/src/orchestra/preparer/activity_preparers/filter.py b/src/orchestra/preparer/activity_preparers/filter.py new file mode 100644 index 0000000..3988e10 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/filter.py @@ -0,0 +1,32 @@ +"""Preparer for FilterActivity -> notebook_task with generated filter notebook.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.models.ir import TranslationContext +from flowx.parser.expression_parser import resolve_expression +from flowx.preparer.activity_preparers.helpers import build_notebook_activity_task +from flowx.preparer.activity_preparers.naming import notebook_filename +from flowx.preparer.code_generator import generate_filter_notebook +from flowx.preparer.workflow_preparer import PreparedActivity + +if TYPE_CHECKING: + from flowx.models.ir import FilterActivity + + +def prepare(activity: FilterActivity, *, scope: str = "") -> PreparedActivity: + """Converts a FilterActivity into a notebook_task that filters an array.""" + items_result = resolve_expression(activity.items_expression, TranslationContext()) + if items_result is not None and items_result.kind in ("dab_ref", "literal"): + items_value = items_result.value + else: + items_value = activity.items_expression + + task, notebooks = build_notebook_activity_task( + activity, + notebook_relative_path=f"notebooks/{notebook_filename(activity.task_key, activity.name)}", + notebook_content=generate_filter_notebook(activity), + base_parameters={"items_expression": items_value}, + ) + return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/activity_preparers/for_each.py b/src/orchestra/preparer/activity_preparers/for_each.py new file mode 100644 index 0000000..a59c117 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/for_each.py @@ -0,0 +1,165 @@ +"""Preparer for ForEachActivity -> for_each_task dict. + +References: +- https://docs.databricks.com/aws/en/jobs/for-each +- https://docs.databricks.com/aws/en/jobs/task-values +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from flowx.bundler.inner_job_params import ( + collect_inner_job_params, + normalize_inner_task_params, +) +from flowx.models.dab import DabNotebook, SecretInstruction, SetupTask +from flowx.models.ir import TranslationContext +from flowx.parser.expression_parser import resolve_expression +from flowx.preparer.workflow_preparer import ( + PreparedActivity, + PreparedWorkflow, + build_common_task_fields, + prepare_activity, +) +from flowx.utils import normalize_task_key + +if TYPE_CHECKING: + from flowx.models.ir import ForEachActivity + + +def _resolve_for_each_inputs(items_expression: str) -> str: + """Converts an ADF items expression to a DAB dynamic value reference. + + Args: + items_expression: The raw ADF expression for ForEach items. + + Returns: + A DAB dynamic value reference string, or the original expression if + it cannot be resolved. + """ + if items_expression.startswith("{{"): + return items_expression + + context = TranslationContext() + result = resolve_expression(items_expression, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + + # Also try with @ prefix if not present + if not items_expression.startswith("@"): + result = resolve_expression("@" + items_expression, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + + return items_expression + + +def _inject_input_parameter(inner_task: dict) -> dict: + """Adds ``{{input}}`` as a base_parameter on the inner task. + + Args: + inner_task: The prepared inner task dict. + + Returns: + The task dict with ``item`` parameter injected. + """ + if "notebook_task" in inner_task: + params = inner_task["notebook_task"].setdefault("base_parameters", {}) + params["item"] = "{{input}}" + elif "run_job_task" in inner_task: + params = inner_task["run_job_task"].setdefault("job_parameters", {}) + params["item"] = "{{input}}" + return inner_task + + +def prepare(activity: ForEachActivity, *, scope: str = "") -> PreparedActivity: + """Converts a ForEachActivity into a DAB for_each_task definition. + + Args: + activity: The translated for-each activity from the IR. + scope: Secret scope name (typically the pipeline/job name). + + Returns: + A PreparedActivity with the for_each_task, plus any notebooks, secrets, + and inner_workflows from the child activities. + """ + task = build_common_task_fields(activity) + concurrency = activity.concurrency if activity.concurrency is not None else 20 + inputs = _resolve_for_each_inputs(activity.items_expression) + + inner_activities = activity.inner_activities + all_notebooks: list[DabNotebook] = [] + all_secrets: list[SecretInstruction] = [] + all_setup_tasks: list[SetupTask] = [] + inner_workflows: list[PreparedWorkflow] = [] + + if len(inner_activities) == 1: + inner_prepared = prepare_activity(inner_activities[0], scope=scope) + inner_task = _inject_input_parameter(inner_prepared.task) + all_notebooks.extend(inner_prepared.notebooks) + all_secrets.extend(inner_prepared.secrets) + all_setup_tasks.extend(inner_prepared.setup_tasks) + inner_workflows.extend(inner_prepared.inner_workflows) + + task["for_each_task"] = { + "inputs": inputs, + "task": inner_task, + "concurrency": concurrency, + } + + elif len(inner_activities) > 1: + inner_job_name = f"{activity.task_key}_inner_tasks" + inner_tasks: list[dict[str, Any]] = [] + + for child in inner_activities: + child_prepared = prepare_activity(child, scope=scope) + inner_tasks.append(child_prepared.task) + all_notebooks.extend(child_prepared.notebooks) + all_secrets.extend(child_prepared.secrets) + all_setup_tasks.extend(child_prepared.setup_tasks) + inner_workflows.extend(child_prepared.inner_workflows) + + normalize_inner_task_params(inner_tasks) + + parameters, job_parameters = collect_inner_job_params(inner_tasks) + + inner_workflow = PreparedWorkflow( + name=inner_job_name, + tasks=inner_tasks, + notebooks=[], # notebooks already collected in all_notebooks + secrets=[], + setup_tasks=[], + parameters=parameters, + ) + inner_workflows.append(inner_workflow) + + inner_job_key = normalize_task_key(inner_job_name) + body_task: dict[str, Any] = { + "task_key": f"{activity.task_key}_iteration", + "run_job_task": { + "job_id": f"${{resources.jobs.{inner_job_key}.id}}", + "job_parameters": job_parameters, + }, + } + + task["for_each_task"] = { + "inputs": inputs, + "task": body_task, + "concurrency": concurrency, + } + + else: + task["for_each_task"] = { + "inputs": inputs, + "task": {"task_key": f"{activity.task_key}_noop"}, + "concurrency": concurrency, + } + + return PreparedActivity( + task=task, + notebooks=all_notebooks, + secrets=all_secrets, + setup_tasks=all_setup_tasks, + inner_workflows=inner_workflows, + ) diff --git a/src/orchestra/preparer/activity_preparers/helpers.py b/src/orchestra/preparer/activity_preparers/helpers.py new file mode 100644 index 0000000..393ff34 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/helpers.py @@ -0,0 +1,86 @@ +"""Shared helpers used by every activity preparer.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from flowx.models.dab import DabNotebook, SecretInstruction +from flowx.models.ir import TranslationContext +from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string +from flowx.preparer.workflow_preparer import build_common_task_fields + +if TYPE_CHECKING: + from flowx.models.ir import Activity + + +def resolve_param_value(value: str) -> str: + """Resolves an ADF expression in a parameter value to its DAB form.""" + if "@{" in value: + return resolve_interpolated_string(value, TranslationContext()) + if value.startswith("@"): + result = resolve_expression(value, TranslationContext()) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + return value + + +def build_notebook_task_artifacts( + *, + notebook_relative_path: str, + notebook_content: str, + base_parameters: dict[str, Any] | None = None, +) -> tuple[dict[str, Any], list[DabNotebook]]: + """Builds the ``notebook_task`` dict and the matching DabNotebook artifact.""" + notebook_task: dict[str, Any] = { + "notebook_path": f"../src/{notebook_relative_path}", + } + if base_parameters is not None: + notebook_task["base_parameters"] = base_parameters + + notebooks = [ + DabNotebook( + relative_path=notebook_relative_path, + content=notebook_content, + ) + ] + return notebook_task, notebooks + + +def build_notebook_activity_task( + activity: Activity, + *, + notebook_relative_path: str, + notebook_content: str, + base_parameters: dict[str, Any] | None = None, +) -> tuple[dict[str, Any], list[DabNotebook]]: + """Builds the common task fields and notebook_task scaffolding for *activity*.""" + task = build_common_task_fields(activity) + notebook_task, notebooks = build_notebook_task_artifacts( + notebook_relative_path=notebook_relative_path, + notebook_content=notebook_content, + base_parameters=base_parameters, + ) + task["notebook_task"] = notebook_task + return task, notebooks + + +def make_jdbc_secrets( + *, + scope_name: str, + source_type: str, + activity_name: str, + role: str = "source", +) -> list[SecretInstruction]: + """Returns the ``jdbc-url`` / ``jdbc-password`` secret pair for a JDBC connector.""" + return [ + SecretInstruction( + scope=scope_name, + key="jdbc-url", + value_source=f"JDBC URL for {source_type} {role} in activity '{activity_name}'", + ), + SecretInstruction( + scope=scope_name, + key="jdbc-password", + value_source=f"JDBC password for {source_type} {role} in activity '{activity_name}'", + ), + ] diff --git a/src/orchestra/preparer/activity_preparers/if_condition.py b/src/orchestra/preparer/activity_preparers/if_condition.py new file mode 100644 index 0000000..85473d3 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/if_condition.py @@ -0,0 +1,93 @@ +"""Preparer for IfConditionActivity -> condition_task + branch sibling tasks. + +References: +- https://docs.databricks.com/aws/en/jobs/if-else +- https://docs.databricks.com/aws/en/dev-tools/bundles/job-task-types +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from flowx.preparer.workflow_preparer import ( + PreparedActivity, + PreparedArtifacts, + build_common_task_fields, + merge_prepared_artifacts, + prepare_activity, +) + +if TYPE_CHECKING: + from flowx.models.ir import IfConditionActivity + + +def inject_outcome_dependency(tasks: list[dict[str, Any]], condition_key: str, outcome: str) -> None: + """Gates branch-root tasks on the condition's outcome. + + A "branch root" is any task in the branch that does not depend on a + sibling within the same branch. External dependencies (on a task + outside the branch) are preserved so a branch task can still wait + on a global setup task or another upstream activity; the outcome + edge is appended to ``depends_on`` rather than replacing it. + + Args: + tasks: Tasks in one branch (mutated in place). + condition_key: Task key of the enclosing condition task. + outcome: ``"true"`` or ``"false"``. + """ + branch_keys = {task.get("task_key") for task in tasks} + outcome_dep = {"task_key": condition_key, "outcome": outcome} + for task in tasks: + deps = list(task.get("depends_on") or []) + refers_to_branch_sibling = any(dep.get("task_key") in branch_keys for dep in deps) + if refers_to_branch_sibling: + continue + if any(dep.get("task_key") == condition_key and dep.get("outcome") == outcome for dep in deps): + continue + task["depends_on"] = [outcome_dep, *deps] + + +def prepare(activity: IfConditionActivity, *, scope: str = "") -> PreparedActivity: + """Converts an IfConditionActivity into a condition_task + flattened branches. + + Args: + activity: The translated if-condition activity from the IR. + scope: Secret scope name passed through to child preparers. + + Returns: + A PreparedActivity with the condition_task, sibling branch tasks, + and aggregated artifacts from both branches. + """ + task = build_common_task_fields(activity) + task["condition_task"] = { + "op": activity.op, + "left": activity.left, + "right": activity.right, + } + + artifacts = PreparedArtifacts() + + if_true_tasks: list[dict[str, Any]] = [] + for child in activity.if_true_activities: + prepared = prepare_activity(child, scope=scope) + if_true_tasks.append(prepared.task) + if_true_tasks.extend(prepared.extra_tasks) + artifacts = merge_prepared_artifacts(artifacts, prepared) + inject_outcome_dependency(if_true_tasks, activity.task_key, "true") + + if_false_tasks: list[dict[str, Any]] = [] + for child in activity.if_false_activities: + prepared = prepare_activity(child, scope=scope) + if_false_tasks.append(prepared.task) + if_false_tasks.extend(prepared.extra_tasks) + artifacts = merge_prepared_artifacts(artifacts, prepared) + inject_outcome_dependency(if_false_tasks, activity.task_key, "false") + + return PreparedActivity( + task=task, + extra_tasks=if_true_tasks + if_false_tasks, + notebooks=list(artifacts.notebooks), + secrets=list(artifacts.secrets), + setup_tasks=list(artifacts.setup_tasks), + inner_workflows=list(artifacts.inner_workflows), + ) diff --git a/src/orchestra/preparer/activity_preparers/lookup.py b/src/orchestra/preparer/activity_preparers/lookup.py new file mode 100644 index 0000000..d68cbae --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/lookup.py @@ -0,0 +1,42 @@ +"""Preparer for LookupActivity -> notebook_task with generated lookup notebook.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.models.dab import SecretInstruction +from flowx.models.source_types import JDBC_SOURCE_TYPES +from flowx.preparer.activity_preparers.helpers import ( + build_notebook_activity_task, + make_jdbc_secrets, +) +from flowx.preparer.activity_preparers.naming import notebook_filename +from flowx.preparer.code_generator import generate_lookup_notebook +from flowx.preparer.workflow_preparer import PreparedActivity + +if TYPE_CHECKING: + from flowx.models.ir import LookupActivity + + +def prepare(activity: LookupActivity, *, scope: str = "") -> PreparedActivity: + """Converts a LookupActivity into a notebook_task with a generated lookup notebook.""" + task, notebooks = build_notebook_activity_task( + activity, + notebook_relative_path=f"notebooks/{notebook_filename(activity.task_key, activity.name)}", + notebook_content=generate_lookup_notebook(activity, scope=scope), + base_parameters={"first_row_only": str(activity.first_row_only).lower()}, + ) + + secrets: list[SecretInstruction] = [] + source_type = activity.source_type or "" + if source_type in JDBC_SOURCE_TYPES: + secrets.extend( + make_jdbc_secrets( + scope_name=scope or activity.task_key, + source_type=source_type, + activity_name=activity.name, + role="lookup", + ) + ) + + return PreparedActivity(task=task, notebooks=notebooks, secrets=secrets) diff --git a/src/orchestra/preparer/activity_preparers/motif.py b/src/orchestra/preparer/activity_preparers/motif.py new file mode 100644 index 0000000..a6ade86 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/motif.py @@ -0,0 +1,22 @@ +"""Preparer for MotifActivity -> notebook_task with motif scaffold notebook.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.preparer.activity_preparers.helpers import build_notebook_activity_task +from flowx.preparer.code_generator import generate_motif_notebook +from flowx.preparer.workflow_preparer import PreparedActivity + +if TYPE_CHECKING: + from flowx.models.ir import MotifActivity + + +def prepare(activity: MotifActivity, *, scope: str = "") -> PreparedActivity: + """Converts a MotifActivity into a notebook_task with the motif scaffold.""" + task, notebooks = build_notebook_activity_task( + activity, + notebook_relative_path=f"notebooks/{activity.task_key}.py", + notebook_content=generate_motif_notebook(activity), + ) + return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/activity_preparers/naming.py b/src/orchestra/preparer/activity_preparers/naming.py new file mode 100644 index 0000000..30214fb --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/naming.py @@ -0,0 +1,30 @@ +"""Shared naming helpers used by every activity preparer.""" + +from __future__ import annotations + +import re + + +def to_snake_case(name: str) -> str: + """Converts PascalCase / camelCase / mixed identifiers to snake_case. + + Examples: + ``BronzeIngest`` -> ``bronze_ingest`` + ``copySQLToBlob`` -> ``copy_sql_to_blob`` + ``ETL_Main`` -> ``etl_main`` + ``spaces and dashes-here`` -> ``spaces_and_dashes_here`` + """ + s1 = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", name) + s2 = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s1) + s2 = re.sub(r"[^a-zA-Z0-9_]+", "_", s2) + s2 = re.sub(r"_+", "_", s2).strip("_") + return s2.lower() + + +def notebook_filename(task_key: str, activity_name: str | None = None) -> str: + """Derive a snake_case ``.py`` filename for a generated notebook.""" + source = activity_name or task_key + snake = to_snake_case(source) + if not snake: + snake = task_key.lower() or "notebook" + return f"{snake}.py" diff --git a/src/orchestra/preparer/activity_preparers/notebook.py b/src/orchestra/preparer/activity_preparers/notebook.py new file mode 100644 index 0000000..3f8e460 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/notebook.py @@ -0,0 +1,133 @@ +"""Preparer for NotebookActivity -> notebook_task dict.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.models.dab import DabNotebook +from flowx.models.ir import TranslationContext +from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string +from flowx.preparer.activity_preparers.naming import notebook_filename +from flowx.preparer.workflow_preparer import PreparedActivity, build_common_task_fields +from flowx.preparer.workspace_downloader import download_notebook + +if TYPE_CHECKING: + from flowx.models.ir import NotebookActivity + + +def _notebook_placeholder(original_path: str, activity_name: str, filename: str) -> str: + """Return placeholder notebook content with manual-export instructions.""" + return ( + "# Databricks notebook source\n" + "# MAGIC %md\n" + f"# MAGIC # {activity_name}\n" + "# MAGIC\n" + f"# MAGIC **Source workspace path**: `{original_path}`\n" + "# MAGIC\n" + f"# MAGIC This notebook was referenced by ADF pipeline activity `{activity_name}`.\n" + "# MAGIC Replace this placeholder with the actual notebook content.\n" + "# MAGIC\n" + "# MAGIC Export from workspace:\n" + "# MAGIC ```\n" + f'# MAGIC databricks workspace export "{original_path}" --format SOURCE -o src/notebooks/{filename}\n' + "# MAGIC ```\n" + "\n# COMMAND ----------\n\n" + "# TODO: Replace this placeholder with the exported notebook content\n" + "raise NotImplementedError(\n" + f' f"Export notebook from workspace: {original_path}"\n' + ")\n" + ) + + +def _resolve_base_parameters( + params: dict[str, str], + *, + variable_task_keys: dict[str, str] | None = None, + existing_notebook: bool = False, +) -> dict[str, str]: + """Resolves ADF expressions in ``base_parameters`` to DAB-compatible values. + + For *existing* notebooks (absolute workspace paths) we keep ``notebook_code`` + parameters as their raw original expression so the downstream + ``_extract_manual_parameters_from_existing_notebook_tasks`` scanner can + pick them up and surface them in SETUP.md -- flowx cannot patch the + notebook body, so the user has to compute the value in-line themselves. + + For bundle-generated notebooks the embedded notebook body owns the + runtime computation (via ``required_parameters``), so ``notebook_code`` + parameters are dropped here. + """ + context = TranslationContext() + resolved: dict[str, str] = {} + for key, value in params.items(): + result = resolve_expression(value, context, variable_task_keys=variable_task_keys) + if result is not None and result.kind in ("literal", "dab_ref"): + resolved[key] = result.value + continue + if result is not None and result.kind == "notebook_code": + if existing_notebook: + resolved[key] = _raw_expression(value) + continue + if isinstance(value, dict) and value.get("type") == "Expression" and "value" in value: + resolved[key] = str(value["value"]) + else: + resolved[key] = str(value) + return resolved + + +def _raw_expression(value: object) -> str: + """Returns the original ADF expression text for an unresolved parameter.""" + if isinstance(value, dict) and value.get("type") == "Expression" and "value" in value: + return str(value["value"]) + return str(value) + + +def _resolve_notebook_path(path: str) -> str: + """Resolves any ADF expression embedded in a notebook workspace path.""" + context = TranslationContext() + if "@{" in path: + return resolve_interpolated_string(path, context) + if path.startswith("@"): + result = resolve_expression(path, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + return path + + +def prepare( + activity: NotebookActivity, + *, + scope: str = "", + variable_task_keys: dict[str, str] | None = None, +) -> PreparedActivity: + """Converts a NotebookActivity into a DAB notebook_task definition.""" + resolved_path = _resolve_notebook_path(activity.notebook_path) + task = build_common_task_fields(activity) + is_existing_notebook = resolved_path.startswith("/") + + base_parameters: dict[str, str] | None = None + if activity.base_parameters: + base_parameters = _resolve_base_parameters( + dict(activity.base_parameters), + variable_task_keys=variable_task_keys, + existing_notebook=is_existing_notebook, + ) + + if is_existing_notebook: + task["notebook_task"] = {"notebook_path": resolved_path} + if base_parameters is not None: + task["notebook_task"]["base_parameters"] = base_parameters + return PreparedActivity(task=task) + + placeholder_filename = notebook_filename(activity.task_key, activity.name) + notebook_relative_path = f"notebooks/{placeholder_filename}" + content = download_notebook(resolved_path) or _notebook_placeholder( + resolved_path, activity.name, placeholder_filename + ) + + task["notebook_task"] = {"notebook_path": f"../src/{notebook_relative_path}"} + if base_parameters is not None: + task["notebook_task"]["base_parameters"] = base_parameters + + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=content)] + return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/activity_preparers/set_variable.py b/src/orchestra/preparer/activity_preparers/set_variable.py new file mode 100644 index 0000000..5820929 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/set_variable.py @@ -0,0 +1,30 @@ +"""Preparer for SetVariableActivity -> notebook_task with generated notebook.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.preparer.activity_preparers.helpers import build_notebook_activity_task +from flowx.preparer.activity_preparers.naming import notebook_filename +from flowx.preparer.code_generator import generate_set_variable_notebook +from flowx.preparer.workflow_preparer import PreparedActivity + +if TYPE_CHECKING: + from flowx.models.ir import SetVariableActivity + + +def prepare(activity: SetVariableActivity, *, scope: str = "") -> PreparedActivity: + """Converts a SetVariableActivity into a notebook_task that sets a task value.""" + base_parameters: dict[str, str] = {"variable_name": activity.variable_name} + if activity.value_kind in ("literal", "dab_ref"): + base_parameters["value"] = activity.variable_value + for widget_name, dab_ref in activity.required_parameters.items(): + base_parameters.setdefault(widget_name, dab_ref) + + task, notebooks = build_notebook_activity_task( + activity, + notebook_relative_path=f"notebooks/{notebook_filename(activity.task_key, activity.name)}", + notebook_content=generate_set_variable_notebook(activity), + base_parameters=base_parameters, + ) + return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/activity_preparers/spark_jar.py b/src/orchestra/preparer/activity_preparers/spark_jar.py new file mode 100644 index 0000000..ab3742f --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/spark_jar.py @@ -0,0 +1,98 @@ +"""Preparer for SparkJarActivity -> spark_jar_task dict.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.models.dab import DabNotebook +from flowx.preparer.activity_preparers.helpers import resolve_param_value +from flowx.preparer.workflow_preparer import PreparedActivity, build_common_task_fields +from flowx.preparer.workspace_downloader import download_dbfs_file + +if TYPE_CHECKING: + from flowx.models.ir import SparkJarActivity + + +def _jar_placeholder(libraries: list[dict] | None, activity_name: str) -> str: + """Generates a placeholder file with download instructions for JAR libraries. + + Args: + libraries: Library descriptors from the SparkJarActivity. + activity_name: The ADF activity name. + + Returns: + Placeholder content as a string. + """ + lib_lines = "" + if libraries: + for lib in libraries: + for key, path in lib.items(): + lib_lines += f"# {key}: {path}\n" + return ( + f"# Placeholder for JARs referenced by activity: {activity_name}\n" + "#\n" + "# Download the following libraries and place them in this directory:\n" + f"{lib_lines}" + "#\n" + "# Use the Databricks CLI to upload JARs:\n" + "# databricks fs cp dbfs:/FileStore/jars/\n" + ) + + +def prepare(activity: SparkJarActivity, *, scope: str = "") -> PreparedActivity: + """Converts a SparkJarActivity into a DAB spark_jar_task definition. + + Args: + activity: The translated Spark JAR activity from the IR. + + Returns: + A PreparedActivity containing the spark_jar_task dict and placeholder files. + """ + task = build_common_task_fields(activity) + + rewritten_libraries: list[dict] = [] + notebooks: list[DabNotebook] = [] + downloaded_any = False + if activity.libraries: + for lib in activity.libraries: + rewritten_lib = {} + for key, path in lib.items(): + if isinstance(path, str) and ("dbfs:" in path or "/" in path): + filename = path.rsplit("/", 1)[-1] if "/" in path else path + rewritten_lib[key] = f"../lib/{filename}" + if key == "jar": + jar_content = download_dbfs_file(path) + if jar_content is not None: + downloaded_any = True + notebooks.append( + DabNotebook( + relative_path=f"lib/{filename}", + binary_content=jar_content, + ) + ) + else: + rewritten_lib[key] = path + rewritten_libraries.append(rewritten_lib) + + if not downloaded_any: + placeholder_content = _jar_placeholder(activity.libraries, activity.name) + notebooks.append( + DabNotebook( + relative_path=f"lib/{activity.task_key}_README.txt", + content=placeholder_content, + language="python", + ) + ) + + resolved_main_class = resolve_param_value(activity.main_class_name) if activity.main_class_name else "" + task["spark_jar_task"] = { + "main_class_name": resolved_main_class, + } + if activity.parameters: + task["spark_jar_task"]["parameters"] = list(activity.parameters) + if rewritten_libraries: + task["libraries"] = rewritten_libraries + elif activity.libraries: + task["libraries"] = list(activity.libraries) + + return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/activity_preparers/spark_python.py b/src/orchestra/preparer/activity_preparers/spark_python.py new file mode 100644 index 0000000..9a9c5e7 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/spark_python.py @@ -0,0 +1,72 @@ +"""Preparer for SparkPythonActivity -> spark_python_task dict.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.models.dab import DabNotebook +from flowx.preparer.activity_preparers.helpers import resolve_param_value +from flowx.preparer.workflow_preparer import PreparedActivity, build_common_task_fields +from flowx.preparer.workspace_downloader import download_dbfs_file + +if TYPE_CHECKING: + from flowx.models.ir import SparkPythonActivity + + +def _python_placeholder(original_path: str, activity_name: str) -> str: + """Generates placeholder Python script with download instructions. + + Args: + original_path: The original DBFS/workspace path from ADF. + activity_name: The ADF activity name. + + Returns: + Placeholder Python content as a string. + """ + return ( + f"# Placeholder for Python script referenced by activity: {activity_name}\n" + f"# Original path: {original_path}\n" + "#\n" + "# Download and replace this file with the actual script:\n" + f'# databricks fs cp "{original_path}" src/scripts/{original_path.rsplit("/", 1)[-1]}\n' + "#\n" + f'raise NotImplementedError("Download script from: {original_path}")\n' + ) + + +def prepare(activity: SparkPythonActivity, *, scope: str = "") -> PreparedActivity: + """Converts a SparkPythonActivity into a DAB spark_python_task definition. + + Args: + activity: The translated Spark Python activity from the IR. + + Returns: + A PreparedActivity containing the spark_python_task dict and placeholder file. + """ + task = build_common_task_fields(activity) + + original_path = resolve_param_value(activity.python_file) if activity.python_file else "" + if original_path and ("dbfs:" in original_path or "/" in original_path): + filename = original_path.rsplit("/", 1)[-1] if "/" in original_path else original_path + else: + filename = f"{activity.task_key}.py" + script_rel_path = f"scripts/{filename}" + + downloaded = download_dbfs_file(original_path) + content = ( + downloaded.decode("utf-8") if downloaded is not None else _python_placeholder(original_path, activity.name) + ) + notebooks = [ + DabNotebook( + relative_path=script_rel_path, + content=content, + language="python", + ) + ] + + task["spark_python_task"] = { + "python_file": f"../src/{script_rel_path}", + } + if activity.parameters: + task["spark_python_task"]["parameters"] = list(activity.parameters) + return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/activity_preparers/switch.py b/src/orchestra/preparer/activity_preparers/switch.py new file mode 100644 index 0000000..a442f0c --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/switch.py @@ -0,0 +1,156 @@ +"""Preparer for SwitchActivity -> chained condition_tasks + flattened branches. + +References: +- https://docs.databricks.com/aws/en/jobs/if-else +- https://docs.databricks.com/aws/en/dev-tools/bundles/job-task-types +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any + +from flowx.models.ir import TranslationContext +from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string +from flowx.preparer.activity_preparers.if_condition import inject_outcome_dependency +from flowx.preparer.workflow_preparer import ( + PreparedActivity, + PreparedArtifacts, + build_common_task_fields, + merge_prepared_artifacts, + prepare_activity, +) + +if TYPE_CHECKING: + from flowx.models.ir import SwitchActivity + + +def sanitize_case_key(value: str) -> str: + """Returns a task-key-safe form of a switch case value. + + Used by both the in-process Switch preparer and the JSON-reload path in + ``dab_writer`` so the rendered task graph is identical regardless of + which path produced it. + """ + sanitized = re.sub(r"[^a-zA-Z0-9_]", "_", value) + sanitized = re.sub(r"_+", "_", sanitized).strip("_") + return sanitized or "empty" + + +def resolve_switch_on_expression(on_expression: str) -> str: + """Resolves the switch ``on`` expression to a DAB dynamic value ref. + + Idempotent: an already-resolved DAB ref (``{{job.parameters.X}}``) or + plain literal passes through unchanged. Both the in-process preparer + and the JSON-reload path call this so a hand-edited IR with a raw + ``@variables(...)`` is still resolved before being written to YAML. + """ + context = TranslationContext() + if "@{" in on_expression: + return resolve_interpolated_string(on_expression, context) + if on_expression.startswith("@"): + result = resolve_expression(on_expression, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + return on_expression + + +def prepare(activity: SwitchActivity, *, scope: str = "") -> PreparedActivity: + """Converts a SwitchActivity into a chain of flattened condition tasks. + + Args: + activity: The translated switch activity from the IR. + scope: Secret scope name passed through to child preparers. + + Returns: + A PreparedActivity with the first condition as ``task`` and all + subsequent conditions + case bodies as ``extra_tasks``. + """ + artifacts = PreparedArtifacts() + extra_tasks: list[dict[str, Any]] = [] + + resolved_expr = resolve_switch_on_expression(activity.on_expression) + + if not activity.cases: + task = build_common_task_fields(activity) + task["condition_task"] = {"op": "EQUAL_TO", "left": "true", "right": "true"} + + default_tasks: list[dict[str, Any]] = [] + for child in activity.default_activities: + prepared = prepare_activity(child, scope=scope) + default_tasks.append(prepared.task) + default_tasks.extend(prepared.extra_tasks) + artifacts = merge_prepared_artifacts(artifacts, prepared) + inject_outcome_dependency(default_tasks, activity.task_key, "true") + + return PreparedActivity( + task=task, + extra_tasks=default_tasks, + notebooks=list(artifacts.notebooks), + secrets=list(artifacts.secrets), + setup_tasks=list(artifacts.setup_tasks), + inner_workflows=list(artifacts.inner_workflows), + ) + + # Build one condition task per case, chained via outcome="false" deps. + # Every case (including the first) is named ``_case_`` + # for clarity in the rendered job graph. The first case carries the + # original Switch's depends_on edges; ``prepare_workflow`` rewrites any + # downstream task that referenced the bare ```` key to point + # at the renamed first case. + case_keys: list[str] = [] + for index, case in enumerate(activity.cases): + is_first = index == 0 + case_key = f"{activity.task_key}_case_{sanitize_case_key(case.value)}" + case_keys.append(case_key) + + condition_task: dict[str, Any] = { + "task_key": case_key, + "condition_task": {"op": "EQUAL_TO", "left": resolved_expr, "right": case.value}, + } + if is_first: + # First condition takes the original activity's depends_on, timeouts, + # retries, etc. — same baseline as before, just under the new key. + base = build_common_task_fields(activity) + base.pop("task_key", None) + condition_task.update(base) + else: + condition_task["depends_on"] = [{"task_key": case_keys[index - 1], "outcome": "false"}] + + case_branch_tasks: list[dict[str, Any]] = [] + for child in case.activities: + prepared = prepare_activity(child, scope=scope) + case_branch_tasks.append(prepared.task) + case_branch_tasks.extend(prepared.extra_tasks) + artifacts = merge_prepared_artifacts(artifacts, prepared) + inject_outcome_dependency(case_branch_tasks, case_key, "true") + + if is_first: + first_condition_task = condition_task + else: + extra_tasks.append(condition_task) + extra_tasks.extend(case_branch_tasks) + + branch_default_tasks: list[dict[str, Any]] = [] + for child in activity.default_activities: + prepared = prepare_activity(child, scope=scope) + branch_default_tasks.append(prepared.task) + branch_default_tasks.extend(prepared.extra_tasks) + artifacts = merge_prepared_artifacts(artifacts, prepared) + if branch_default_tasks: + inject_outcome_dependency(branch_default_tasks, case_keys[-1], "false") + extra_tasks.extend(branch_default_tasks) + + # Tell prepare_workflow to remap any depends_on that referenced the + # original Switch task_key onto the renamed first case. + remap = {activity.task_key: case_keys[0]} + + return PreparedActivity( + task=first_condition_task, + extra_tasks=extra_tasks, + notebooks=list(artifacts.notebooks), + secrets=list(artifacts.secrets), + setup_tasks=list(artifacts.setup_tasks), + inner_workflows=list(artifacts.inner_workflows), + task_key_remap=remap, + ) diff --git a/src/orchestra/preparer/activity_preparers/wait.py b/src/orchestra/preparer/activity_preparers/wait.py new file mode 100644 index 0000000..d716dd2 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/wait.py @@ -0,0 +1,24 @@ +"""Preparer for WaitActivity -> notebook_task with generated sleep notebook.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.preparer.activity_preparers.helpers import build_notebook_activity_task +from flowx.preparer.activity_preparers.naming import notebook_filename +from flowx.preparer.code_generator import generate_wait_notebook +from flowx.preparer.workflow_preparer import PreparedActivity + +if TYPE_CHECKING: + from flowx.models.ir import WaitActivity + + +def prepare(activity: WaitActivity, *, scope: str = "") -> PreparedActivity: + """Converts a WaitActivity into a notebook_task that sleeps for N seconds.""" + task, notebooks = build_notebook_activity_task( + activity, + notebook_relative_path=f"notebooks/{notebook_filename(activity.task_key, activity.name)}", + notebook_content=generate_wait_notebook(activity), + base_parameters={"wait_seconds": str(activity.wait_time_seconds)}, + ) + return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/activity_preparers/web_activity.py b/src/orchestra/preparer/activity_preparers/web_activity.py new file mode 100644 index 0000000..b1d72f4 --- /dev/null +++ b/src/orchestra/preparer/activity_preparers/web_activity.py @@ -0,0 +1,43 @@ +"""Preparer for WebActivity -> notebook_task with generated HTTP notebook.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.models.dab import SecretInstruction +from flowx.preparer.activity_preparers.helpers import ( + build_notebook_activity_task, + resolve_param_value, +) +from flowx.preparer.activity_preparers.naming import notebook_filename +from flowx.preparer.code_generator import generate_web_activity_notebook +from flowx.preparer.workflow_preparer import PreparedActivity + +if TYPE_CHECKING: + from flowx.models.ir import WebActivity + + +def prepare(activity: WebActivity, *, scope: str = "") -> PreparedActivity: + """Converts a WebActivity into a notebook_task with a generated HTTP notebook.""" + task, notebooks = build_notebook_activity_task( + activity, + notebook_relative_path=f"notebooks/{notebook_filename(activity.task_key, activity.name)}", + notebook_content=generate_web_activity_notebook(activity, scope=scope), + base_parameters={ + "url": resolve_param_value(activity.url), + "method": resolve_param_value(activity.method), + }, + ) + + secrets: list[SecretInstruction] = [] + if activity.authentication: + auth_type = activity.authentication.get("type", "unknown") + secrets.append( + SecretInstruction( + scope=scope or activity.task_key, + key="auth-credential", + value_source=f"Authentication credential ({auth_type}) for web activity '{activity.name}'", + ) + ) + + return PreparedActivity(task=task, notebooks=notebooks, secrets=secrets) diff --git a/src/orchestra/preparer/code_generator.py b/src/orchestra/preparer/code_generator.py new file mode 100644 index 0000000..89e04f9 --- /dev/null +++ b/src/orchestra/preparer/code_generator.py @@ -0,0 +1,1410 @@ +"""Generates Python notebook content for activities that need custom notebooks.""" + +from __future__ import annotations + +import ast +import json +import re +import textwrap +from typing import TYPE_CHECKING, Any + +from flowx.models.ir import TranslationContext +from flowx.models.source_types import FILE_SOURCE_TYPES, JDBC_SOURCE_TYPES, REST_SOURCE_TYPES +from flowx.parser.expression_parser import ( + resolve_expression, + resolve_interpolated_string_for_notebook, +) + +if TYPE_CHECKING: + from flowx.models.ir import ( + AppendVariableActivity, + CopyActivity, + DeleteActivity, + FilterActivity, + LookupActivity, + MotifActivity, + SetVariableActivity, + WaitActivity, + WebActivity, + ) + + +def generate_lookup_notebook(activity: LookupActivity, *, scope: str = "") -> str: + """Generates a Python notebook that executes a lookup query. + + Args: + activity: The LookupActivity IR node. + scope: Secret scope name (defaults to task_key if empty). + + Returns: + Complete notebook source code as a string. + """ + header = _notebook_header(f"Lookup: {activity.name}") + source_type = activity.source_type or "" + query = activity.source_query or "" + + query_assignment = _render_query_assignment(query) + + if source_type in JDBC_SOURCE_TYPES: + scope = scope or activity.task_key + + body = textwrap.dedent(f"""\ + import json + + # Parameters + first_row_only = dbutils.widgets.get("first_row_only") == "true" + + # Credentials + jdbc_url = dbutils.secrets.get(scope="{scope}", key="jdbc-url") + jdbc_password = dbutils.secrets.get(scope="{scope}", key="jdbc-password") + jdbc_user = dbutils.secrets.get(scope="{scope}", key="jdbc-user") + + # Execute lookup query + {query_assignment} + + df = ( + spark.read.format("jdbc") + .option("url", jdbc_url) + .option("user", jdbc_user) + .option("password", jdbc_password) + .option("query", query) + .load() + ) + + if first_row_only: + result = df.first() + output = result.asDict() if result else {{}} + else: + output = [row.asDict() for row in df.collect()] + + # Set task values so downstream tasks can reference them via + # {{{{tasks.{activity.task_key}.values.}}}}. + # The full result is stored under "result" as a JSON string so + # for_each_task can consume it as `inputs`. For firstRow lookups, + # each column is also stored as an individual task value so + # condition_task can reference e.g. {{{{tasks.{activity.task_key}.values.cnt}}}}. + dbutils.jobs.taskValues.set(key="result", value=json.dumps(output)) + if first_row_only and isinstance(output, dict): + for col_name, col_value in output.items(): + dbutils.jobs.taskValues.set(key=col_name, value=col_value) + """) + else: + body = textwrap.dedent(f"""\ + import json + + # Parameters + first_row_only = dbutils.widgets.get("first_row_only") == "true" + + # Execute lookup query via Spark SQL + {query_assignment} + df = spark.sql(query) + + if first_row_only: + result = df.first() + output = result.asDict() if result else {{}} + else: + output = [row.asDict() for row in df.collect()] + + # Set task values so downstream tasks can reference them via + # {{{{tasks.{activity.task_key}.values.}}}}. + dbutils.jobs.taskValues.set(key="result", value=json.dumps(output)) + if first_row_only and isinstance(output, dict): + for col_name, col_value in output.items(): + dbutils.jobs.taskValues.set(key=col_name, value=col_value) + """) + + return header + _command_separator() + body + + +def generate_web_activity_notebook(activity: WebActivity, *, scope: str = "") -> str: + """Generates a Python notebook that makes an HTTP request. + + Args: + activity: The WebActivity IR node. + scope: Secret scope name (defaults to task_key if empty). + + Returns: + Complete notebook source code as a string. + """ + header = _notebook_header(f"Web Activity: {activity.name}") + + # Resolve header values — some may contain ADF expressions like + # {"Authorization": {"type": "Expression", "value": "@concat('Bearer ', ...)"}} + headers_literal, headers_preamble = _resolve_headers(activity.headers) + + auth_block = "" + auth = activity.authentication + if auth: + scope = scope or activity.task_key + auth_type = auth.get("type", "") + if auth_type in ("ServicePrincipal", "MSI", "ManagedServiceIdentity"): + auth_block = textwrap.dedent(f"""\ + # Authentication ({auth_type}) + auth_token = dbutils.secrets.get(scope="{scope}", key="auth-credential") + headers["Authorization"] = f"Bearer {{auth_token}}" + """) + elif auth_type == "Basic": + auth_block = textwrap.dedent(f"""\ + # Authentication (Basic) + import base64 + username = dbutils.secrets.get(scope="{scope}", key="auth-username") + password = dbutils.secrets.get(scope="{scope}", key="auth-credential") + token = base64.b64encode(f"{{username}}:{{password}}".encode()).decode() + headers["Authorization"] = f"Basic {{token}}" + """) + else: + auth_block = textwrap.dedent(f"""\ + # Authentication + auth_credential = dbutils.secrets.get(scope="{scope}", key="auth-credential") + headers["Authorization"] = f"Bearer {{auth_credential}}" + """) + + body_block = "" + request_call = "" + if activity.method in ("POST", "PUT", "PATCH"): + raw_body = activity.body + # If the body was pre-resolved to Python code by the translator + # (contains function calls like __import__ or json.loads), embed directly. + if isinstance(raw_body, str) and ("__import__" in raw_body or "json.loads" in raw_body): + body_block = f"body = {raw_body}\n" + else: + body_str = _resolve_body(raw_body) + # ``_resolve_body`` may return either a JSON literal, a Python + # dict literal, or a ``repr()``'d string containing Python-like + # concat syntax. Parse strings as JSON when possible so the + # downstream ``requests.request(json=...)`` gets a real object. + body_block = textwrap.dedent(f"""\ + body_raw = dbutils.widgets.get("body") or {body_str} + if isinstance(body_raw, str): + try: + body = json.loads(body_raw) + except (ValueError, TypeError): + body = body_raw + else: + body = body_raw + """) + # ``json=`` encodes dicts; string bodies go over ``data=`` so we don't + # double-encode them as JSON strings. + request_call = textwrap.dedent( + """\ + if isinstance(body, (dict, list)): + response = requests.request(method, url, headers=headers, json=body, timeout=300) + else: + response = requests.request(method, url, headers=headers, data=body, timeout=300) + """ + ) + else: + request_call = "response = requests.request(method, url, headers=headers, timeout=300)\n" + + # When the URL is a DAB dynamic ref ({{...}}), it will be resolved and + # passed via base_parameters at runtime — just read from the widget. + if "{{" in activity.url: + url_line = 'url = dbutils.widgets.get("url")' + else: + url_line = f'url = dbutils.widgets.get("url") or "{activity.url}"' + + if "{{" in activity.method: + method_line = 'method = dbutils.widgets.get("method")' + else: + method_line = f'method = dbutils.widgets.get("method") or "{activity.method}"' + + body = textwrap.dedent(f"""\ + import json + import requests + + # Parameters + {url_line} + {method_line} + headers = {headers_literal} + + """) + body += headers_preamble + body += auth_block + body += body_block + body += request_call + body += textwrap.dedent("""\ + + response.raise_for_status() + + # Return response + try: + result = response.json() + except ValueError: + result = {"status_code": response.status_code, "text": response.text} + + """) + + return header + _command_separator() + body + + +def generate_delete_notebook(activity: DeleteActivity) -> str: + """Generates a notebook using dbutils.fs.rm(). + + Args: + activity: The DeleteActivity IR node. + + Returns: + Complete notebook source code as a string. + """ + header = _notebook_header(f"Delete: {activity.name}") + folder_path = activity.folder_path or "" + + body = textwrap.dedent(f"""\ + # Parameters + dataset_name = dbutils.widgets.get("dataset_name") or "{activity.dataset_name}" + folder_path = dbutils.widgets.get("folder_path") or "{folder_path}" + recursive = dbutils.widgets.get("recursive") == "true" + + # Build the full path to delete + target_path = folder_path if folder_path else dataset_name + print(f"Deleting: {{target_path}} (recursive={{recursive}})") + result = dbutils.fs.rm(target_path, recurse=recursive) + + """) + + return header + _command_separator() + body + + +def generate_set_variable_notebook(activity: SetVariableActivity) -> str: + """Generates a notebook that sets a task value. + + Args: + activity: The SetVariableActivity IR node. + + Returns: + Complete notebook source code as a string. + """ + header = _notebook_header(f"Set Variable: {activity.name}") + + if activity.value_kind == "notebook_code" and activity.notebook_code: + # Embed imports and Python code directly in the notebook + import_lines = "\n".join(activity.notebook_imports) if activity.notebook_imports else "" + if import_lines: + import_block = import_lines + "\n" + else: + import_block = "" + + # Build body lines list to avoid textwrap.dedent issues when + # import_block starts at column 0 (which would prevent dedent + # from stripping the common leading whitespace). + lines = ["import json"] + if import_block: + lines.append(import_block.rstrip("\n")) + lines.append("") + lines.append(f'variable_name = "{activity.variable_name}"') + lines.append("") + lines.append("# Compute value at runtime.") + lines.append("# Original ADF expression resolved to notebook code.") + lines.append(f"value = {activity.notebook_code}") + lines.append("") + lines.append("# Set task value so downstream tasks can reference it via") + lines.append(f"# {{{{tasks.{activity.task_key}.values.{activity.variable_name}}}}}.") + lines.append("dbutils.jobs.taskValues.set(key=variable_name, value=value)") + lines.append("print(f\"Set task value '{variable_name}' = '{value}'\")") + lines.append("") + body = "\n".join(lines) + "\n" + else: + body = textwrap.dedent(f"""\ + import json + + variable_name = "{activity.variable_name}" + + # Read value from widget parameter (set via base_parameters). + # DAB resolves dynamic references (e.g. {{{{job.run_id}}}}) before passing. + value = dbutils.widgets.get("value") + + # Set task value so downstream tasks can reference it via + # {{{{tasks.{activity.task_key}.values.{activity.variable_name}}}}}. + dbutils.jobs.taskValues.set(key=variable_name, value=value) + print(f"Set task value '{{variable_name}}' = '{{value}}'") + + """) + + return header + _command_separator() + body + + +def generate_wait_notebook(activity: WaitActivity) -> str: + """Generates a notebook that sleeps for a specified duration. + + Args: + activity: The WaitActivity IR node. + + Returns: + Complete notebook source code as a string. + """ + header = _notebook_header(f"Wait: {activity.name}") + + body = textwrap.dedent(f"""\ + import time + + # Parameters + default_wait = {activity.wait_time_seconds} + param = dbutils.widgets.get("wait_seconds") + wait_seconds = int(param) if param else default_wait + + print(f"Waiting for {{wait_seconds}} seconds...") + time.sleep(wait_seconds) + print("Wait complete.") + + """) + + return header + _command_separator() + body + + +def generate_copy_notebook(activity: CopyActivity, *, scope: str = "") -> str: + """Generates a notebook for copy operations (Auto Loader, COPY INTO, or JDBC). + + Args: + activity: The CopyActivity IR node. + scope: Secret scope name (defaults to task_key if empty). + + Returns: + Complete notebook source code as a string. + """ + header = _notebook_header(f"Copy: {activity.name}") + source_type = activity.source_type or "" + + if source_type in FILE_SOURCE_TYPES: + body = _generate_autoloader_body(activity) + elif source_type in JDBC_SOURCE_TYPES: + body = _generate_jdbc_body(activity, scope=scope) + elif source_type in REST_SOURCE_TYPES: + body = _generate_rest_copy_body(activity) + else: + body = _generate_generic_copy_body(activity) + + # Hoist any imports the body needs into a single cell at the top of + # the notebook. ``_render_sink_write`` and a few other helpers used + # to inline ``from datetime import ...`` next to the call site, which + # produced an awkward block sandwiched between two comment groups. + imports = _detect_imports(body) + if imports: + body = _strip_inline_imports(body, imports) + return header + _command_separator() + "\n".join(imports) + "\n" + _command_separator() + body + return header + _command_separator() + body + + +def _detect_imports(body: str) -> list[str]: + """Return import lines required by code references found in *body*.""" + needed: list[str] = [] + if "datetime." in body: + needed.append("from datetime import datetime, timezone, timedelta") + if "ZoneInfo(" in body: + needed.append("from zoneinfo import ZoneInfo") + return needed + + +def _strip_inline_imports(body: str, hoisted: list[str]) -> str: + """Remove inline import lines that match the ones we hoisted to the top.""" + lines = body.splitlines() + keep: list[str] = [] + skip_prefixes = ( + "from datetime import", + "from zoneinfo import", + "import datetime", + ) + for line in lines: + stripped = line.lstrip() + if any(stripped.startswith(prefix) for prefix in skip_prefixes): + continue + keep.append(line) + return "\n".join(keep) + ("\n" if body.endswith("\n") else "") + + +def generate_filter_notebook(activity: FilterActivity) -> str: + """Generates a notebook that filters an array and stores the result as a task value. + + The condition is pre-translated at translate-time when possible; the + notebook never calls ``eval()`` on widget input. Conditions the + translator could not safely lower fall back to a TODO placeholder + notebook. + """ + header = _notebook_header(f"Filter: {activity.name}") + if activity.condition_code is None: + return header + _command_separator() + _filter_placeholder_body(activity) + return header + _command_separator() + _filter_resolved_body(activity) + + +def _filter_resolved_body(activity: FilterActivity) -> str: + """Returns the notebook body for a Filter whose condition was resolved to Python.""" + import_block = "\n".join(activity.condition_imports or []) + items_default = repr(activity.items_expression) + original_expression_comment = _safe_inline_comment(activity.condition_expression) + return textwrap.dedent(f"""\ + import json + {import_block} + + # ``items_expression`` carries either a JSON-encoded array (when DAB + # substitutes a {{{{tasks.X.values.Y}}}} reference) or a raw JSON literal + # -- both parse via ``json.loads``. + items_expression = dbutils.widgets.get("items_expression") or {items_default} + items = json.loads(items_expression) if items_expression else [] + if not isinstance(items, list): + items = [items] + + # Pre-translated condition (no eval on widget input). + # Original ADF expression: {original_expression_comment} + filtered = [item for item in items if {activity.condition_code}] + + dbutils.jobs.taskValues.set(key="output", value=json.dumps(filtered)) + print(f"Filtered {{len(items)}} items to {{len(filtered)}} items") + """) + + +def _filter_placeholder_body(activity: FilterActivity) -> str: + """Returns a TODO placeholder body for a Filter whose condition didn't translate.""" + items_default = repr(activity.items_expression) + original_expression_comment = _safe_inline_comment(activity.condition_expression) + activity_name_literal = repr(activity.name) + return textwrap.dedent(f"""\ + import json + + items_expression = dbutils.widgets.get("items_expression") or {items_default} + items = json.loads(items_expression) if items_expression else [] + if not isinstance(items, list): + items = [items] + + # The translator could not safely lower the original ADF condition + # to Python without invoking ``eval()`` on widget input. Implement + # the per-item check below by hand, then drop this NotImplementedError. + # Original ADF expression: {original_expression_comment} + def _matches(item): + raise NotImplementedError( + f"Implement filter condition for activity {activity_name_literal}" + ) + + filtered = [item for item in items if _matches(item)] + + dbutils.jobs.taskValues.set(key="output", value=json.dumps(filtered)) + print(f"Filtered {{len(items)}} items to {{len(filtered)}} items") + """) + + +def _safe_inline_comment(text: str) -> str: + """Returns *text* on a single line, suitable for an inline ``#`` comment.""" + return text.replace("\n", " ").replace("\r", " ").strip() + + +def generate_append_variable_notebook(activity: AppendVariableActivity) -> str: + """Generates a notebook that appends a value to an array task value. + + Args: + activity: The AppendVariableActivity IR node. + + Returns: + Complete notebook source code as a string. + """ + header = _notebook_header(f"Append Variable: {activity.name}") + + if activity.value_kind == "notebook_code" and activity.notebook_code: + import_lines = "\n".join(activity.notebook_imports) if activity.notebook_imports else "" + if import_lines: + import_block = import_lines + "\n" + else: + import_block = "" + + # Build body lines list to avoid textwrap.dedent issues when + # import_block starts at column 0. + lines = ["import json"] + if import_block: + lines.append(import_block.rstrip("\n")) + lines.append("") + lines.append("# Parameters") + lines.append(f'variable_name = dbutils.widgets.get("variable_name") or "{activity.variable_name}"') + lines.append("") + lines.append("# Compute value at runtime.") + lines.append(f"value = {activity.notebook_code}") + lines.append("") + lines.append("# Read the current array from task values (or start with empty list)") + lines.append("# `source_task_key` is populated at deploy time with the task that most") + lines.append("# recently set this variable. An empty value falls back to [].") + lines.append("source_task_key = dbutils.widgets.get('source_task_key')") + lines.append("current: list = []") + lines.append("if source_task_key:") + lines.append(" try:") + lines.append(" current_raw = dbutils.jobs.taskValues.get(taskKey=source_task_key, key=variable_name)") + lines.append(" if isinstance(current_raw, str):") + lines.append(" current = json.loads(current_raw)") + lines.append(" elif isinstance(current_raw, list):") + lines.append(" current = current_raw") + lines.append(" elif current_raw is not None:") + lines.append(" current = [current_raw]") + lines.append(" except Exception:") + lines.append(" current = []") + lines.append("") + lines.append("if not isinstance(current, list):") + lines.append(" current = [current] if current else []") + lines.append("") + lines.append("# Append and write back") + lines.append("current.append(value)") + lines.append("result = json.dumps(current)") + lines.append("dbutils.jobs.taskValues.set(key=variable_name, value=result)") + lines.append("print(f\"Appended to '{variable_name}': array now has {len(current)} item(s)\")") + lines.append("") + body = "\n".join(lines) + "\n" + else: + body = textwrap.dedent(f"""\ + import json + + # Parameters + variable_name = dbutils.widgets.get("variable_name") or "{activity.variable_name}" + value = dbutils.widgets.get("value") or {json.dumps(activity.append_value)} + + # Read the current array from a prior task's values. + # `source_task_key` is passed via base_parameters and points at the + # task that most recently set this variable; empty → start with []. + source_task_key = dbutils.widgets.get("source_task_key") + current: list = [] + if source_task_key: + try: + current_raw = dbutils.jobs.taskValues.get(taskKey=source_task_key, key=variable_name) + if isinstance(current_raw, str): + current = json.loads(current_raw) + elif isinstance(current_raw, list): + current = current_raw + elif current_raw is not None: + current = [current_raw] + except Exception: + current = [] + + # Evaluate the value to append + try: + starts = ('{{', '[', '"') + append_val = json.loads(value) if isinstance(value, str) and value.startswith(starts) else value + except (json.JSONDecodeError, ValueError): + append_val = value + + # Append and write back + current.append(append_val) + result = json.dumps(current) + dbutils.jobs.taskValues.set(key=variable_name, value=result) + print(f"Appended to '{{variable_name}}': array now has {{len(current)}} item(s)") + + """) + + return header + _command_separator() + body + + +def _notebook_header(title: str) -> str: + """Return the standard Databricks notebook header block.""" + return textwrap.dedent(f"""\ + # Databricks notebook source + # MAGIC %md + # MAGIC # {title} + # MAGIC + # MAGIC *Auto-generated by Flowx. Do not edit manually unless necessary.* + """) + + +def _command_separator() -> str: + """Return the Databricks cell separator comment.""" + return "\n# COMMAND ----------\n\n" + + +def _render_query_assignment(query: str) -> str: + """Returns ``query = `` source for embedding a lookup query in a notebook. + + The flowx expression resolver may have pre-translated the original + ADF query into Python source that builds the SQL string at runtime + (e.g. ``"SELECT ... FROM " + dbutils.widgets.get('table')``). When + that's the case the embedded text is *code*, not data, so we + defensively parse it as a Python expression before splicing it in -- + if it doesn't parse, fall back to a safe string literal. + + All other queries are embedded as ``repr()`` string literals so any + embedded quotes (including ``\"\"\"``) round-trip correctly. + """ + if "dbutils.widgets.get" in query or "dbutils.jobs.taskValues" in query: + try: + ast.parse(query, mode="eval") + except SyntaxError: + return f"query = {query!r}" + return f"query = {query}" + return f"query = {query!r}" + + +_DAB_REF_PARAMETER_RE = re.compile(r"\{\{job\.parameters\.(\w+)\}\}") +_DAB_REF_RUN_ID_RE = re.compile(r"\{\{job\.run_id\}\}") +_DAB_REF_JOB_NAME_RE = re.compile(r"\{\{job\.name\}\}") +_DAB_REF_START_TIME_RE = re.compile(r"\{\{job\.start_time\.iso_datetime\}\}") +_DAB_REF_TASK_VALUE_RE = re.compile(r"\{\{tasks\.([^.]+)\.values\.(\w+)\}\}") + + +def _dab_ref_to_fstring_expr(ref: str) -> str: + """Converts a DAB ref (``{{job.name}}``) to a Python f-string expression.""" + parameter_match = _DAB_REF_PARAMETER_RE.match(ref) + if parameter_match: + return "{dbutils.widgets.get('" + parameter_match.group(1) + "')}" + if _DAB_REF_RUN_ID_RE.match(ref): + return "{spark.conf.get('spark.databricks.job.runId', 'unknown')}" + if _DAB_REF_JOB_NAME_RE.match(ref): + return "{spark.conf.get('spark.databricks.job.parentName', 'unknown')}" + if _DAB_REF_START_TIME_RE.match(ref): + return "{spark.conf.get('spark.databricks.job.triggerTime', 'unknown')}" + task_value_match = _DAB_REF_TASK_VALUE_RE.match(ref) + if task_value_match: + task_key, value_key = task_value_match.group(1), task_value_match.group(2) + return "{dbutils.jobs.taskValues.get(taskKey='" + task_key + "', key='" + value_key + "')}" + return ref + + +def _resolve_body(body: Any) -> str: + """Resolves ADF expressions in a request body and return a Python expression. + + Args: + body: The raw body from the WebActivity IR. + + Returns: + A Python expression string suitable for embedding in generated code. + """ + if body is None: + return "None" + if isinstance(body, str): + return _resolve_string_body(body) + if isinstance(body, dict): + return _resolve_dict_body(body) + return json.dumps(body) if body else "''" + + +def _resolve_string_body(body: str) -> str: + """Renders a string-shaped WebActivity body as Python source.""" + context = TranslationContext() + if "@{" in body: + resolved_str = resolve_interpolated_string_for_notebook(body, context) + return f"f{json.dumps(resolved_str)}" + if not body.startswith("@"): + return json.dumps(body) + + result = resolve_expression(body, context) + if result is None: + return json.dumps(body) + if result.kind == "notebook_code": + return result.value + return json.dumps(result.value) + + +def _resolve_dict_body(body: dict[str, Any]) -> str: + """Renders a dict-shaped WebActivity body as Python source.""" + if body.get("type") == "Expression" and "value" in body: + return _resolve_body(body["value"]) + + context = TranslationContext() + needs_fstring = False + resolved: dict[str, Any] = {} + for key, value in body.items(): + new_value, value_needs_fstring = _resolve_dict_value(value, context) + resolved[key] = new_value + needs_fstring = needs_fstring or value_needs_fstring + + if not needs_fstring: + return json.dumps(resolved) + + parts: list[str] = [] + for key, value in resolved.items(): + if isinstance(value, str) and "{" in value and "dbutils" in value: + parts.append(f'"{key}": f"{value}"') + else: + parts.append(f'"{key}": {json.dumps(value)}') + return "{" + ", ".join(parts) + "}" + + +def _resolve_dict_value(value: Any, context: TranslationContext) -> tuple[Any, bool]: + """Resolves a single dict value; return ``(new_value, needs_fstring)``.""" + if isinstance(value, str) and "@{" in value: + return resolve_interpolated_string_for_notebook(value, context), True + if isinstance(value, str) and value.startswith("@"): + return _resolve_expression_value(value, context, fallback=value) + if isinstance(value, dict) and value.get("type") == "Expression": + fallback = value.get("value", str(value)) + return _resolve_expression_value(value, context, fallback=fallback) + return value, False + + +def _resolve_expression_value(raw: Any, context: TranslationContext, *, fallback: Any) -> tuple[Any, bool]: + """Resolves a string/dict expression to either an f-string or a literal.""" + result = resolve_expression(raw, context) + if result is None: + return fallback, False + if result.kind == "dab_ref": + return _dab_ref_to_fstring_expr(result.value), True + if result.kind == "literal": + return result.value, False + return fallback, False + + +def _resolve_headers(headers: dict[str, str] | None) -> tuple[str, str]: + """Resolves ADF expressions in HTTP header values. + + Returns: + A tuple of ``(headers_literal, preamble_code)``: + - ``headers_literal`` is a Python dict literal for the initial headers + - ``preamble_code`` is Python code to execute after the headers dict + is created, adding dynamically computed header values + """ + if not headers: + return "{}", "" + + context = TranslationContext() + static_headers: dict[str, str] = {} + preamble_lines: list[str] = [] + + for key, value in headers.items(): + result = resolve_expression(value, context) + if result is None: + static_headers[key] = value if isinstance(value, str) else str(value) + elif result.kind == "literal": + static_headers[key] = result.value + elif result.kind == "dab_ref": + preamble_lines.append(f'headers["{key}"] = dbutils.widgets.get("{key}")') + elif result.kind == "notebook_code": + for import_line in result.imports: + preamble_lines.insert(0, import_line) + preamble_lines.append(f'headers["{key}"] = {result.value}') + + headers_literal = json.dumps(static_headers) if static_headers else "{}" + preamble = "" + if preamble_lines: + seen_lines: set[str] = set() + unique_lines: list[str] = [] + for line in preamble_lines: + if line not in seen_lines: + seen_lines.add(line) + unique_lines.append(line) + preamble = "\n".join(unique_lines) + "\n" + + return headers_literal, preamble + + +def _render_sink_write( + activity: CopyActivity, + df_var: str = "df", + *, + mode: str = "overwrite", + indent: str = "", +) -> str: + """Return the Python ``df.write.*`` expression for the activity's sink. + + Args: + activity: The CopyActivity. Reads ``sink_format``, + ``sink_resolved_path``, and ``sink_dataset_type``. + df_var: The Python identifier of the DataFrame to write. + mode: Spark write mode. ``overwrite`` for full reads, ``append`` + for incremental ones (e.g. inside ForEach loops). + indent: Prefix prepended to every emitted line, so the snippet drops + cleanly into already-indented bodies. + + Returns: + A multi-line Python snippet. Trailing newline included. + """ + fmt = activity.sink_format + sink_props = activity.sink_properties or {} + + # File-format sink — write the actual format declared by the ADF + # output dataset. Delta files written with ``.save(path)`` skip the + # metastore, which matches the ADF semantic of a path-based dataset. + if fmt and fmt != "delta": + opts: list[str] = [] + format_settings = sink_props.get("formatSettings") or {} + if fmt == "csv": + if format_settings.get("firstRowAsHeader") is not False: + opts.append('.option("header", "true")') + if format_settings.get("columnDelimiter"): + delim = format_settings["columnDelimiter"] + opts.append(f'.option("delimiter", "{delim}")') + opts_str = "".join(opts) + + volume_relative = sink_props.get("volume_relative_path") + if volume_relative is not None: + # Volume-rooted sink: ``output_path_root`` is set by the bundler + # as a base_parameter (with DAB-substituted ``${var.catalog}`` + # / ``${var.schema}``). Any ``@{...}`` expressions in the + # ADF dataset's folderPath / fileName have already been + # rewritten to Python f-string fragments, so we wrap the + # relative path in an f-string and join. + rel_literal = volume_relative.replace('"', '\\"') + preamble = "" + # Pull in any modules the rewritten f-string fragments reference + # so the notebook is runnable as-is. Today the only one is + # ``datetime`` (from ``@{formatDateTime(...)}`` rewrites). + if "datetime." in rel_literal: + preamble = f"{indent}from datetime import datetime\n" + return ( + f"{preamble}" + f"{indent}# Volume root is bound by the task's ``output_path_root`` parameter\n" + f"{indent}# (resolved by DAB to /Volumes///).\n" + f'{indent}output_path_root = dbutils.widgets.get("output_path_root")\n' + f'{indent}output_path = f"{{output_path_root}}/{rel_literal}"\n' + f'{indent}{df_var}.write.format("{fmt}"){opts_str}.mode("{mode}").save(output_path)\n' + ) + + # No structured sink volume — fall back to a single ``output_path`` + # widget the user fills in. Common when the linked service uses + # a masked connection string and we can't reconstruct any path. + return ( + f"{indent}# The ADF output dataset path could not be resolved at translation time.\n" + f"{indent}# Set ``output_path`` on this task to the destination URI.\n" + f'{indent}output_path = dbutils.widgets.get("output_path")\n' + f'{indent}{df_var}.write.format("{fmt}"){opts_str}.mode("{mode}").save(output_path)\n' + ) + + # Delta sink (or unclassified fallback) — keep existing behaviour. + if fmt == "delta" or fmt is None: + if mode == "append": + return ( + f'{indent}{df_var}.write.format("delta").mode("append")' + f'.option("mergeSchema", "true").saveAsTable(target_table)\n' + ) + return ( + f'{indent}{df_var}.write.format("delta").mode("{mode}")' + f'.option("overwriteSchema", "true").saveAsTable(target_table)\n' + ) + + raise ValueError("Invalid fmt string") + + +def _infer_file_format(source_type: str | None, source_properties: dict | None) -> str: + """Infer the file format from the source type string and source properties. + + Args: + source_type: The ADF source type string (e.g. ``"DelimitedTextSource"``). + source_properties: The source properties dict from the CopyActivity. + + Returns: + File format string (e.g. ``"csv"``, ``"json"``, ``"parquet"``). + """ + if source_type: + type_lower = source_type.lower() + if "delimitedtext" in type_lower or "csv" in type_lower: + return "csv" + if "json" in type_lower: + return "json" + if "parquet" in type_lower: + return "parquet" + if "avro" in type_lower: + return "avro" + if "orc" in type_lower: + return "orc" + + if source_properties: + fmt_settings = source_properties.get("formatSettings", {}) + fmt_type = fmt_settings.get("type", "") + if "Csv" in fmt_type or "Delimited" in fmt_type: + return "csv" + if "Json" in fmt_type: + return "json" + if "Parquet" in fmt_type: + return "parquet" + if "Avro" in fmt_type: + return "avro" + if "Orc" in fmt_type: + return "orc" + store = source_properties.get("storeSettings", {}) + store_type = store.get("type", "") + if "BinaryRead" in store_type: + return "binaryFile" + + return "parquet" + + +def _generate_autoloader_body(activity: CopyActivity) -> str: + """Generates Auto Loader ingestion body for file-based sources.""" + source_properties = activity.source_properties or {} + sink_properties = activity.sink_properties or {} + + # Prefer the UC volume path when available (set by the copy preparer + # when an external volume setup task is created); otherwise fall back + # to the resolved abfss:// path or raw dataset path. + source_path = source_properties.get( + "volume_path", + source_properties.get( + "resolved_path", + source_properties.get("path", source_properties.get("filePath", "/mnt/source")), + ), + ) + sink_table = sink_properties.get("table", sink_properties.get("tableName", f"{activity.task_key}_raw")) + file_format = _infer_file_format(activity.source_type, source_properties) + + # Use the volume for checkpoints and schema evolution storage instead of + # /tmp. This ensures state persists across cluster restarts and is + # visible in Unity Catalog. + volume_base = source_properties.get("volume_base", "") + if volume_base: + checkpoint = f"{volume_base}/_checkpoints/{activity.task_key}" + else: + checkpoint = f"/tmp/checkpoints/{activity.task_key}" + + return textwrap.dedent(f"""\ + # Parameters + source_path = dbutils.widgets.get("source_path") if dbutils.widgets.get("source_path") else "{source_path}" + target_table = dbutils.widgets.get("target_table") if dbutils.widgets.get("target_table") else "{sink_table}" + checkpoint_path = "{checkpoint}" + + # Auto Loader: stream file-based source into Delta table + df = ( + spark.readStream.format("cloudFiles") + .option("cloudFiles.format", "{file_format}") + .option("cloudFiles.schemaLocation", checkpoint_path + "/_schema") + .option("cloudFiles.inferColumnTypes", "true") + .load(source_path) + ) + + # Write to Delta table + ( + df.writeStream.format("delta") + .option("checkpointLocation", checkpoint_path) + .option("mergeSchema", "true") + .outputMode("append") + .trigger(availableNow=True) + .toTable(target_table) + ) + + """) + + +def _adf_timeout_to_seconds(value: Any) -> int | None: + """Parses ADF duration strings (e.g. ``"02:00:00"`` / ``"0.01:30:00"``) to seconds.""" + if not value: + return None + text = str(value).strip() + if not text: + return None + if "." in text and text.split(".", 1)[0].isdigit(): + days_part, hms_part = text.split(".", 1) + days = int(days_part) + else: + days = 0 + hms_part = text + parts = hms_part.split(":") + if len(parts) != 3: + return None + try: + hours, minutes, seconds = (int(part) for part in parts) + except ValueError: + return None + return days * 86400 + hours * 3600 + minutes * 60 + seconds + + +def _generate_jdbc_body(activity: CopyActivity, *, scope: str = "") -> str: + """Generates JDBC ingestion body for database sources.""" + scope = scope or activity.task_key + source_properties = activity.source_properties or {} + sink_properties = activity.sink_properties or {} + + sink_table = sink_properties.get("table", sink_properties.get("tableName", f"{activity.task_key}_raw")) + table_name = source_properties.get("tableName", source_properties.get("table", "")) + query_raw = source_properties.get("sqlReaderQuery", source_properties.get("query", "")) + query_timeout_seconds = _adf_timeout_to_seconds(source_properties.get("queryTimeout")) + query_timeout_option = ( + f'\n .option("queryTimeout", "{query_timeout_seconds}")' if query_timeout_seconds else "" + ) + + query = "" + is_expression = False + if isinstance(query_raw, dict) and query_raw.get("type") == "Expression": + is_expression = True + query = query_raw.get("value", "") + elif isinstance(query_raw, str): + query = query_raw + if query.startswith("@"): + is_expression = True + + if is_expression: + # The query is an ADF expression (e.g. @concat('SELECT * FROM ', item().schema_name, ...)). + # Generate a notebook that reads the current ForEach item from the + # "item" widget (set to {{input}} by the for_each_task) and builds + # the SQL query dynamically. + return ( + textwrap.dedent(f"""\ + import json + + # Parameters + default_table = "{sink_table}" + target_table = dbutils.widgets.get("target_table") or default_table + + # The ForEach task passes the current item as the "item" widget + # parameter via {{{{input}}}}. Parse it as JSON to access fields. + item_raw = dbutils.widgets.get("item") + item = json.loads(item_raw) if item_raw else {{}} + + # Credentials + jdbc_url = dbutils.secrets.get(scope="{scope}", key="jdbc-url") + jdbc_password = dbutils.secrets.get(scope="{scope}", key="jdbc-password") + jdbc_user = dbutils.secrets.get(scope="{scope}", key="jdbc-user") + + # Build the SQL query from the ForEach item fields. + # Original ADF expression: {query} + schema_name = item.get("schema_name", "dbo") + table_name = item.get("table_name", "UNKNOWN_TABLE") + query = f"SELECT * FROM {{schema_name}}.{{table_name}}" + print(f"Executing query: {{query}}") + + # Read from source database via JDBC + df = ( + spark.read.format("jdbc") + .option("url", jdbc_url) + .option("user", jdbc_user) + .option("password", jdbc_password) + .option("query", query){query_timeout_option} + .load() + ) + + # Write to the sink defined by the ADF output dataset. No + # count/print: those trigger an extra Spark action and can + # double the read cost. + """) + + _render_sink_write(activity, mode="append", indent="") + + textwrap.dedent("""\ + + """) + ) + + if table_name: + read_option = f' .option("dbtable", "{table_name}")' + elif query and "@{" in query: + # ``@{...}`` interpolation in a SQL query becomes an f-string so + # ``dbutils.widgets.get(...)`` resolves at runtime. + resolved_query = resolve_interpolated_string_for_notebook(query, TranslationContext()) + read_option = f' .option("query", f"""{resolved_query}""")' + elif query: + read_option = f' .option("query", """{query}""")' + else: + read_option = ' .option("dbtable", "REPLACE_WITH_TABLE_NAME")' + + return ( + textwrap.dedent(f"""\ + # Parameters + target_table = dbutils.widgets.get("target_table") if dbutils.widgets.get("target_table") else "{sink_table}" + + # Credentials + jdbc_url = dbutils.secrets.get(scope="{scope}", key="jdbc-url") + jdbc_password = dbutils.secrets.get(scope="{scope}", key="jdbc-password") + jdbc_user = dbutils.secrets.get(scope="{scope}", key="jdbc-user") + + # Read from source database via JDBC + df = ( + spark.read.format("jdbc") + .option("url", jdbc_url) + .option("user", jdbc_user) + .option("password", jdbc_password) + {read_option}{query_timeout_option} + .load() + ) + + # Write to the sink defined by the ADF output dataset. No count/ + # print: those trigger an extra Spark action and can double the + # read cost. + """) + + _render_sink_write(activity, mode="overwrite", indent="") + + textwrap.dedent("""\ + + """) + ) + + +def _generate_rest_copy_body(activity: CopyActivity) -> str: + """Generates REST API ingestion body.""" + source_properties = activity.source_properties or {} + sink_properties = activity.sink_properties or {} + + url = source_properties.get("url", source_properties.get("relativeUrl", "")) + sink_table = sink_properties.get("table", sink_properties.get("tableName", f"{activity.task_key}_raw")) + + return ( + textwrap.dedent(f"""\ + import json + import requests + + # Parameters + url = dbutils.widgets.get("url") if dbutils.widgets.get("url") else "{url}" + target_table = dbutils.widgets.get("target_table") if dbutils.widgets.get("target_table") else "{sink_table}" + headers = {{"Content-Type": "application/json"}} + + # Fetch data from REST API + response = requests.get(url, headers=headers, timeout=300) + response.raise_for_status() + data = response.json() + + # Normalize to list of records + if isinstance(data, dict): + for key in ("value", "data", "results", "items", "records"): + if key in data and isinstance(data[key], list): + data = data[key] + break + else: + data = [data] + + # Write to the sink defined by the ADF output dataset. + df = spark.createDataFrame(data) + """) + + _render_sink_write(activity, mode="overwrite", indent="") + + textwrap.dedent("""\ + + """) + ) + + +def _generate_generic_copy_body(activity: CopyActivity) -> str: + """Generates a generic Spark read/write copy body as fallback.""" + source_properties = activity.source_properties or {} + sink_properties = activity.sink_properties or {} + + # Use the resolved path from dataset if available, otherwise fall back + source_path = source_properties.get( + "resolved_path", + source_properties.get("path", source_properties.get("filePath", "/mnt/source")), + ) + sink_table = sink_properties.get("table", sink_properties.get("tableName", f"{activity.task_key}_raw")) + file_format = _infer_file_format(activity.source_type, source_properties) + + return ( + textwrap.dedent(f"""\ + # Parameters + source_path = dbutils.widgets.get("source_path") if dbutils.widgets.get("source_path") else "{source_path}" + target_table = dbutils.widgets.get("target_table") if dbutils.widgets.get("target_table") else "{sink_table}" + + # Read source data + df = spark.read.format("{file_format}").load(source_path) + + # Write to the sink defined by the ADF output dataset. + """) + + _render_sink_write(activity, mode="overwrite", indent="") + + textwrap.dedent("""\ + + """) + ) + + +def generate_motif_notebook(activity: MotifActivity) -> str: + """Generates a notebook scaffold for a collapsed motif activity.""" + return _build_motif_notebook( + task_key=activity.task_key, + activity_name=activity.name, + motif_id=activity.motif_id, + databricks_replacement=activity.databricks_replacement, + matched_activity_names=list(activity.matched_activity_names), + source_type_hint=activity.source_type_hint or "", + confidence_notes=list(activity.confidence_notes), + motif_config=dict(activity.motif_config) if activity.motif_config else None, + ) + + +def _build_motif_notebook( + *, + task_key: str, + activity_name: str, + motif_id: str, + databricks_replacement: str, + matched_activity_names: list[str], + source_type_hint: str, + confidence_notes: list[str], + motif_config: dict[str, Any] | None = None, +) -> str: + """Builds the motif notebook scaffold body shared by both preparation paths.""" + matched_list = "\n".join(f"# MAGIC - `{name}`" for name in matched_activity_names) + notes_list = "\n".join(f"# MAGIC - {note}" for note in confidence_notes) if confidence_notes else "# MAGIC (none)" + + source_line = f"# MAGIC **Source type**: `{source_type_hint}`" if source_type_hint else "" + + lines = [ + "# Databricks notebook source", + "# MAGIC %md", + f"# MAGIC # Motif: {activity_name}", + "# MAGIC", + f"# MAGIC **Pattern**: `{motif_id}`", + f"# MAGIC **Databricks replacement**: `{databricks_replacement}`", + ] + if source_line: + lines.append(source_line) + lines.extend( + [ + "# MAGIC", + "# MAGIC ## Collapsed ADF Activities", + "# MAGIC", + matched_list, + "# MAGIC", + "# MAGIC ## Detection Notes", + "# MAGIC", + notes_list, + "# MAGIC", + "# MAGIC *Auto-generated by Flowx motif collapser.*", + "", + "# COMMAND ----------", + "", + ] + ) + + if databricks_replacement == "auto_loader": + lines.extend(_auto_loader_motif_body(task_key)) + elif databricks_replacement == "dlt_apply_changes": + lines.extend(_dlt_apply_changes_motif_body(motif_id)) + elif databricks_replacement == "for_each_ingestion": + lines.extend(_for_each_ingestion_motif_body(task_key, motif_config or {})) + elif databricks_replacement == "python_rest_ingestion": + lines.extend(_python_rest_ingestion_motif_body()) + elif databricks_replacement == "auto_loader_file_notification": + lines.extend(_auto_loader_file_notification_motif_body(task_key)) + else: + lines.extend( + [ + f"# TODO: Implement Databricks-native replacement for motif '{motif_id}'", + f"# Strategy: {databricks_replacement}", + f"raise NotImplementedError('Motif {motif_id}: implement {databricks_replacement}')", + ] + ) + + lines.append("") + return "\n".join(lines) + + +def _auto_loader_motif_body(task_key: str) -> list[str]: + return [ + "# Auto Loader ingestion -- replaces Lookup/Copy/StoredProcedure watermark chain", + "source_path = dbutils.widgets.get('source_path')", + "target_table = dbutils.widgets.get('target_table')", + f"checkpoint_path = '/tmp/checkpoints/{task_key}'", + "", + "df = (", + ' spark.readStream.format("cloudFiles")', + ' .option("cloudFiles.format", "parquet")', + ' .option("cloudFiles.schemaLocation", checkpoint_path + "/_schema")', + " .load(source_path)", + ")", + "", + "(", + ' df.writeStream.format("delta")', + ' .option("checkpointLocation", checkpoint_path)', + ' .option("mergeSchema", "true")', + " .outputMode('append')", + " .trigger(availableNow=True)", + " .toTable(target_table)", + ")", + ] + + +def _dlt_apply_changes_motif_body(motif_id: str) -> list[str]: + return [ + "# DLT APPLY CHANGES -- replaces Copy/DataFlow SCD or CDC chain", + "# This motif is best implemented as a DLT pipeline definition.", + "# See: https://docs.databricks.com/en/delta-live-tables/cdc.html", + "", + "# import dlt", + "# @dlt.table", + "# def target_table():", + "# return spark.readStream.table('staging_table')", + "#", + "# dlt.apply_changes(", + "# target='target_table',", + "# source='staging_table',", + "# keys=['id'],", + "# sequence_by='updated_at',", + "# )", + "", + f"raise NotImplementedError('Motif {motif_id}: implement as DLT pipeline')", + ] + + +def _for_each_ingestion_motif_body(task_key: str, motif_config: dict[str, Any]) -> list[str]: + lookup_query = motif_config.get("lookup_query", "") + lookup_scope = motif_config.get("lookup_scope") or task_key + copy_scope = motif_config.get("copy_scope") or task_key + sink_table_pattern = motif_config.get("sink_table") or "raw.{schema_name}_{table_name}" + return [ + "# Parameterised bulk ingestion -- replaces the collapsed Lookup/ForEach/Copy chain.", + "import json", + "", + f"lookup_jdbc_url = dbutils.secrets.get(scope='{lookup_scope}', key='jdbc-url')", + f"lookup_jdbc_user = dbutils.secrets.get(scope='{lookup_scope}', key='jdbc-user')", + f"lookup_jdbc_password = dbutils.secrets.get(scope='{lookup_scope}', key='jdbc-password')", + "", + "items_override = dbutils.widgets.get('items')", + "if items_override:", + " items = json.loads(items_override)", + "else:", + f" control_query = {lookup_query!r}", + " control_df = (", + " spark.read.format('jdbc')", + " .option('url', lookup_jdbc_url)", + " .option('user', lookup_jdbc_user)", + " .option('password', lookup_jdbc_password)", + " .option('query', control_query)", + " .load()", + " )", + " items = [row.asDict() for row in control_df.collect()]", + "", + f"copy_jdbc_url = dbutils.secrets.get(scope='{copy_scope}', key='jdbc-url')", + f"copy_jdbc_user = dbutils.secrets.get(scope='{copy_scope}', key='jdbc-user')", + f"copy_jdbc_password = dbutils.secrets.get(scope='{copy_scope}', key='jdbc-password')", + "", + "for item in items:", + " table_name = item.get('table_name') or item.get('name') or 'UNKNOWN_TABLE'", + " schema_name = item.get('schema_name', 'dbo')", + f" target = {sink_table_pattern!r}.format(schema_name=schema_name, table_name=table_name)", + " query = f'SELECT * FROM {schema_name}.{table_name}'", + " (", + " spark.read.format('jdbc')", + " .option('url', copy_jdbc_url)", + " .option('user', copy_jdbc_user)", + " .option('password', copy_jdbc_password)", + " .option('query', query)", + " .load()", + " .write.format('delta')", + " .mode('overwrite')", + " .option('overwriteSchema', 'true')", + " .saveAsTable(target)", + " )", + "", + "dbutils.notebook.exit(json.dumps({'ingested_tables': len(items)}))", + ] + + +def _python_rest_ingestion_motif_body() -> list[str]: + return [ + "# REST API pagination -- replaces WebActivity/Until/SetVariable chain", + "import json", + "import requests", + "", + "base_url = dbutils.widgets.get('api_url')", + "auth_token = dbutils.secrets.get(scope='rest_api', key='token')", + "headers = {'Authorization': f'Bearer {auth_token}'}", + "", + "all_records = []", + "next_url = base_url", + "", + "while next_url:", + " response = requests.get(next_url, headers=headers, timeout=60)", + " response.raise_for_status()", + " data = response.json()", + " records = data.get('value', data.get('data', []))", + " all_records.extend(records)", + " next_url = data.get('nextLink') or data.get('@odata.nextLink')", + "", + "df = spark.createDataFrame(all_records)", + "target_table = dbutils.widgets.get('target_table')", + "df.write.format('delta').mode('overwrite').saveAsTable(target_table)", + "print(f'Ingested {len(all_records)} records')", + ] + + +def _auto_loader_file_notification_motif_body(task_key: str) -> list[str]: + return [ + "# Auto Loader with file notification -- replaces GetMetadata/ForEach/Copy/Delete chain", + "source_path = dbutils.widgets.get('source_path')", + "target_table = dbutils.widgets.get('target_table')", + f"checkpoint_path = '/tmp/checkpoints/{task_key}'", + "", + "df = (", + ' spark.readStream.format("cloudFiles")', + ' .option("cloudFiles.format", "parquet")', + ' .option("cloudFiles.useNotifications", "true")', + ' .option("cloudFiles.schemaLocation", checkpoint_path + "/_schema")', + " .load(source_path)", + ")", + "", + "(", + ' df.writeStream.format("delta")', + ' .option("checkpointLocation", checkpoint_path)', + " .outputMode('append')", + " .trigger(availableNow=True)", + " .toTable(target_table)", + ")", + ] diff --git a/src/orchestra/preparer/workflow_preparer.py b/src/orchestra/preparer/workflow_preparer.py new file mode 100644 index 0000000..e3f4034 --- /dev/null +++ b/src/orchestra/preparer/workflow_preparer.py @@ -0,0 +1,281 @@ +"""Converts a translated Pipeline IR into a PreparedWorkflow.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from flowx.models.dab import DabNotebook, SecretInstruction, SetupTask +from flowx.models.ir import ( + Activity, + AppendVariableActivity, + CopyActivity, + DeleteActivity, + ExecutePipelineActivity, + FilterActivity, + ForEachActivity, + IfConditionActivity, + LookupActivity, + MotifActivity, + NotebookActivity, + Pipeline, + PlaceholderActivity, + RunJobActivity, + SetVariableActivity, + SparkJarActivity, + SparkPythonActivity, + SwitchActivity, + UnsupportedActivity, + WaitActivity, + WebActivity, +) + + +@dataclass(slots=True, kw_only=True) +class PreparedActivity: + """Result of preparing a single activity for DAB deployment.""" + + task: dict[str, Any] + extra_tasks: list[dict[str, Any]] = field(default_factory=list) + notebooks: list[DabNotebook] = field(default_factory=list) + secrets: list[SecretInstruction] = field(default_factory=list) + setup_tasks: list[SetupTask] = field(default_factory=list) + inner_workflows: list[PreparedWorkflow] = field(default_factory=list) + # Switch renames its first case from ```` to + # ``_case_``; ``prepare_workflow`` reads this map to + # rewrite ``depends_on`` edges that referenced the original key. + task_key_remap: dict[str, str] = field(default_factory=dict) + + +@dataclass(slots=True, kw_only=True) +class PreparedWorkflow: + """A fully prepared workflow ready for DAB bundle generation.""" + + name: str + tasks: list[dict[str, Any]] + notebooks: list[DabNotebook] + secrets: list[SecretInstruction] + setup_tasks: list[SetupTask] + inner_workflows: list[PreparedWorkflow] = field(default_factory=list) + parameters: list[dict[str, Any]] = field(default_factory=list) + cluster_hints: list[dict[str, Any]] = field(default_factory=list) + + +def run_if_from_adf_outcomes(outcomes: list[str | None]) -> str | None: + """Maps a set of ADF dependency-edge outcomes to a single DAB ``run_if``.""" + normalised = [outcome for outcome in outcomes if outcome] + if not normalised: + return None + if any(outcome in ("Completed", "Skipped") for outcome in normalised): + return "ALL_DONE" + if any(outcome == "Failed" for outcome in normalised): + return "AT_LEAST_ONE_FAILED" + return None + + +def build_common_task_fields(activity: Activity) -> dict[str, Any]: + """Builds the task-level fields shared by every DAB task type. + + Returns: + A dict with ``task_key``, ``depends_on``, ``timeout_seconds``, and + retry fields populated from the activity. + """ + task: dict[str, Any] = {"task_key": activity.task_key} + + if activity.depends_on: + task["depends_on"] = [{"task_key": dep.task_key} for dep in activity.depends_on] + run_if = run_if_from_adf_outcomes([dep.outcome for dep in activity.depends_on]) + if run_if: + task["run_if"] = run_if + + if activity.timeout_seconds is not None and activity.timeout_seconds > 0: + task["timeout_seconds"] = activity.timeout_seconds + + if activity.max_retries is not None and activity.max_retries > 0: + task["retry_on_timeout"] = True + task["max_retries"] = activity.max_retries + if activity.min_retry_interval_millis is not None: + task["min_retry_interval_millis"] = activity.min_retry_interval_millis + + if activity.description: + task["description"] = activity.description + + return task + + +def prepare_activity( + activity: Activity, + *, + scope: str = "", + variable_task_keys: dict[str, str] | None = None, +) -> PreparedActivity: + """Dispatches to the appropriate activity preparer based on activity type.""" + from flowx.preparer.activity_preparers import ( + append_variable, + copy, + databricks_job, + delete, + execute_pipeline, + filter, + for_each, + if_condition, + lookup, + motif, + notebook, + set_variable, + spark_jar, + spark_python, + switch, + wait, + web_activity, + ) + + dispatch: dict[type, Any] = { + NotebookActivity: notebook.prepare, + SparkJarActivity: spark_jar.prepare, + SparkPythonActivity: spark_python.prepare, + CopyActivity: copy.prepare, + LookupActivity: lookup.prepare, + WebActivity: web_activity.prepare, + DeleteActivity: delete.prepare, + SetVariableActivity: set_variable.prepare, + FilterActivity: filter.prepare, + AppendVariableActivity: append_variable.prepare, + ForEachActivity: for_each.prepare, + IfConditionActivity: if_condition.prepare, + ExecutePipelineActivity: execute_pipeline.prepare, + RunJobActivity: databricks_job.prepare, + SwitchActivity: switch.prepare, + WaitActivity: wait.prepare, + MotifActivity: motif.prepare, + } + + preparer_fn = dispatch.get(type(activity)) + if preparer_fn is None: + if isinstance(activity, (PlaceholderActivity, UnsupportedActivity)): + return _prepare_placeholder(activity) + raise ValueError( + f"No preparer registered for activity type {type(activity).__name__} (task_key={activity.task_key!r})" + ) + + # NotebookActivity rewrites ``@variables()`` references; AppendVariable + # reads the prior writer's task_key to find the value to append to. + if type(activity) is NotebookActivity: + return preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) + if type(activity) is AppendVariableActivity: + return preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) + return preparer_fn(activity, scope=scope) + + +def _prepare_placeholder(activity: Activity) -> PreparedActivity: + """Returns a PreparedActivity with a stub notebook for an unsupported activity.""" + task = build_common_task_fields(activity) + + if isinstance(activity, PlaceholderActivity): + comment = activity.comment or "This activity requires manual implementation." + original_type = activity.original_type + elif isinstance(activity, UnsupportedActivity): + comment = activity.reason or "This activity type is not supported." + original_type = activity.original_type + else: + comment = "Unknown activity type." + original_type = type(activity).__name__ + + notebook_name = f"{activity.task_key}.py" + notebook_path = f"notebooks/{notebook_name}" + + content = ( + "# Databricks notebook source\n" + "# MAGIC %md\n" + f"# MAGIC # Placeholder: {activity.name}\n" + "# MAGIC\n" + f"# MAGIC Original ADF activity type: **{original_type}**\n" + "# MAGIC\n" + f"# MAGIC {comment}\n" + "\n# COMMAND ----------\n\n" + f"raise NotImplementedError(\"Activity '{activity.name}' ({original_type}) requires manual implementation.\")\n" + ) + + task["notebook_task"] = { + "notebook_path": f"../src/{notebook_path}", + } + + notebook = DabNotebook( + relative_path=notebook_path, + content=content, + ) + + return PreparedActivity(task=task, notebooks=[notebook]) + + +@dataclass(frozen=True, slots=True) +class PreparedArtifacts: + """Immutable accumulator for the four artifact lists a workflow collects.""" + + notebooks: tuple[DabNotebook, ...] = () + secrets: tuple[SecretInstruction, ...] = () + setup_tasks: tuple[SetupTask, ...] = () + inner_workflows: tuple[PreparedWorkflow, ...] = () + + +def merge_prepared_artifacts( + artifacts: PreparedArtifacts, + prepared: PreparedActivity, +) -> PreparedArtifacts: + """Return a new :class:`PreparedArtifacts` extended with *prepared*'s artifacts.""" + return PreparedArtifacts( + notebooks=artifacts.notebooks + tuple(prepared.notebooks), + secrets=artifacts.secrets + tuple(prepared.secrets), + setup_tasks=artifacts.setup_tasks + tuple(prepared.setup_tasks), + inner_workflows=artifacts.inner_workflows + tuple(prepared.inner_workflows), + ) + + +def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: + """Converts a Pipeline IR into a PreparedWorkflow ready for the DAB bundle writer.""" + all_tasks: list[dict[str, Any]] = [] + artifacts = PreparedArtifacts() + cluster_hints: list[dict[str, Any]] = [] + task_key_remap: dict[str, str] = {} + + scope = pipeline.name + # Updated in pipeline-declaration order so AppendVariable sees the + # most-recent prior writer of each variable. + variable_task_keys_map: dict[str, str] = {} + + for activity in pipeline.tasks: + prepared = prepare_activity(activity, scope=scope, variable_task_keys=variable_task_keys_map) + all_tasks.append(prepared.task) + all_tasks.extend(prepared.extra_tasks) + artifacts = merge_prepared_artifacts(artifacts, prepared) + task_key_remap.update(prepared.task_key_remap) + if activity.cluster: + cluster_hints.append(dict(activity.cluster)) + + if isinstance(activity, (SetVariableActivity, AppendVariableActivity)): + variable_task_keys_map[activity.variable_name] = activity.task_key + + if task_key_remap: + for task in all_tasks: + for dep in task.get("depends_on", []) or []: + original_key = dep.get("task_key") + if original_key in task_key_remap: + dep["task_key"] = task_key_remap[original_key] + + seen_secrets: set[tuple[str, str]] = set() + unique_secrets: list[SecretInstruction] = [] + for secret in artifacts.secrets: + secret_id = (secret.scope, secret.key) + if secret_id not in seen_secrets: + seen_secrets.add(secret_id) + unique_secrets.append(secret) + + return PreparedWorkflow( + name=pipeline.name, + tasks=all_tasks, + notebooks=list(artifacts.notebooks), + secrets=unique_secrets, + setup_tasks=list(artifacts.setup_tasks), + inner_workflows=list(artifacts.inner_workflows), + cluster_hints=cluster_hints, + ) diff --git a/src/orchestra/preparer/workspace_downloader.py b/src/orchestra/preparer/workspace_downloader.py new file mode 100644 index 0000000..fcf0503 --- /dev/null +++ b/src/orchestra/preparer/workspace_downloader.py @@ -0,0 +1,210 @@ +"""Download workspace artifacts (notebooks, scripts, JARs) via Databricks SDK.""" + +from __future__ import annotations + +import base64 +import configparser +import logging +import os +from pathlib import Path + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Module-level state — resolved once per process, reused across calls +# --------------------------------------------------------------------------- + +_resolved_profile: str | None = None +_profile_resolved: bool = False + + +def _get_databrickscfg_path() -> Path: + """Return the path to the Databricks CLI config file.""" + override = os.environ.get("DATABRICKS_CONFIG_FILE") + if override: + return Path(override) + return Path.home() / ".databrickscfg" + + +def _list_profiles() -> list[str]: + """Parses ``~/.databrickscfg`` and return available profile names. + + Returns: + Sorted list of profile section names. Empty list if the file + does not exist or cannot be parsed. + """ + cfg_path = _get_databrickscfg_path() + if not cfg_path.exists(): + return [] + + config = configparser.ConfigParser(default_section="__no_default__") + try: + config.read(str(cfg_path), encoding="utf-8") + except configparser.Error: + logger.warning("Failed to parse %s", cfg_path) + return [] + + return sorted(config.sections()) + + +def _resolve_profile() -> str | None: + """Determines which ``~/.databrickscfg`` profile to use. + + Returns: + Profile name string, or ``None`` to let the SDK use its own + default resolution. + """ + global _resolved_profile, _profile_resolved # noqa: PLW0603 + + if _profile_resolved: + return _resolved_profile + + _profile_resolved = True + + env_profile = os.environ.get("DATABRICKS_CONFIG_PROFILE") + if env_profile: + logger.info("Using profile from DATABRICKS_CONFIG_PROFILE: %s", env_profile) + _resolved_profile = env_profile + return _resolved_profile + + profiles = _list_profiles() + + if not profiles: + _resolved_profile = None + return _resolved_profile + + if len(profiles) == 1: + _resolved_profile = profiles[0] + logger.info("Using sole .databrickscfg profile: %s", _resolved_profile) + return _resolved_profile + + if "DEFAULT" in profiles: + _resolved_profile = "DEFAULT" + logger.info("Multiple profiles found; using DEFAULT") + return _resolved_profile + + _resolved_profile = _prompt_for_profile(profiles) + return _resolved_profile + + +def _prompt_for_profile(profiles: list[str]) -> str: + """Interactively prompt the user to select a profile. + + Args: + profiles: Available profile names. + + Returns: + Selected profile name. + """ + cfg_path = _get_databrickscfg_path() + + config = configparser.ConfigParser(default_section="__no_default__") + try: + config.read(str(cfg_path), encoding="utf-8") + except configparser.Error: + pass + + print(f"\nMultiple Databricks profiles found in {cfg_path}:") + for i, name in enumerate(profiles, 1): + host = config.get(name, "host", fallback="") + print(f" [{i}] {name:<30s} {host}") + + while True: + try: + choice = input("\nSelect a profile number (or name): ").strip() + except (EOFError, KeyboardInterrupt): + # Non-interactive context — fall back to first profile + print(f"\nNon-interactive; defaulting to '{profiles[0]}'") + return profiles[0] + + if choice.isdigit(): + idx = int(choice) - 1 + if 0 <= idx < len(profiles): + selected = profiles[idx] + print(f"Using profile: {selected}") + return selected + + if choice in profiles: + print(f"Using profile: {choice}") + return choice + + print(f"Invalid selection: '{choice}'. Try again.") + + +def set_profile(profile: str | None) -> None: + """Explicitly set the profile to use, bypassing auto-resolution. + + Args: + profile: Profile name, or ``None`` to reset to auto-resolution. + """ + global _resolved_profile, _profile_resolved # noqa: PLW0603 + _resolved_profile = profile + _profile_resolved = profile is not None + + +def _get_workspace_client(): + """Return a ``WorkspaceClient`` configured with the resolved profile. + + Returns: + A ``WorkspaceClient`` instance. + + Raises: + ImportError: If ``databricks-sdk`` is not installed. + """ + from databricks.sdk import WorkspaceClient # type: ignore[import-not-found] + + profile = _resolve_profile() + if profile: + return WorkspaceClient(profile=profile) + return WorkspaceClient() + + +# --------------------------------------------------------------------------- +# Public download API +# --------------------------------------------------------------------------- + + +def download_notebook(workspace_path: str) -> str | None: + """Download a notebook from Databricks workspace. + + Args: + workspace_path: Workspace path (e.g., ``"/Shared/flowx/transform"``). + + Returns: + Notebook source code as a string, or ``None`` if download failed. + """ + try: + from databricks.sdk.service.workspace import ExportFormat # type: ignore[import-not-found] + + w = _get_workspace_client() + response = w.workspace.export(path=workspace_path, format=ExportFormat.SOURCE) + if response.content: + return base64.b64decode(response.content).decode("utf-8") + except ImportError: + logger.info("databricks-sdk not installed; skipping notebook download for %s", workspace_path) + except Exception as e: + logger.warning("Failed to download notebook %s: %s", workspace_path, e) + return None + + +def download_dbfs_file(dbfs_path: str) -> bytes | None: + """Download a file from DBFS. + + Args: + dbfs_path: DBFS path (e.g., ``"dbfs:/scripts/process.py"`` or + ``"dbfs:/jars/app.jar"``). + + Returns: + File content as bytes, or ``None`` if download failed. + """ + try: + w = _get_workspace_client() + # Strip "dbfs:" prefix for the SDK call + path = dbfs_path.replace("dbfs:", "", 1) + with w.dbfs.open(path, read=True) as f: + return f.read() + except ImportError: + logger.info("databricks-sdk not installed; skipping DBFS download for %s", dbfs_path) + except Exception as e: + logger.warning("Failed to download DBFS file %s: %s", dbfs_path, e) + return None diff --git a/src/orchestra/translator/__init__.py b/src/orchestra/translator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/orchestra/translator/activity_translators/__init__.py b/src/orchestra/translator/activity_translators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/orchestra/translator/activity_translators/append_variable.py b/src/orchestra/translator/activity_translators/append_variable.py new file mode 100644 index 0000000..44a0529 --- /dev/null +++ b/src/orchestra/translator/activity_translators/append_variable.py @@ -0,0 +1,63 @@ +"""Translates ADF AppendVariable activities to Databricks AppendVariableActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, AppendVariableActivity, TranslationContext +from flowx.parser.expression_parser import resolve_expression + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> tuple[Activity, TranslationContext]: + """Translates an AppendVariable activity and register the variable in context. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + + Returns: + Tuple of ``(AppendVariableActivity, updated_context)`` where the context + now maps the variable name to this activity's task key. + """ + type_properties = activity.type_properties or {} + + variable_name = type_properties.get("variableName", "") + value_raw = type_properties.get("value", "") + + expr_result = resolve_expression(value_raw, context) + + required_parameters: dict[str, str] = {} + if expr_result is not None: + append_value = expr_result.value + value_kind = expr_result.kind + notebook_code = expr_result.value if expr_result.kind == "notebook_code" else None + notebook_imports = list(expr_result.imports) if expr_result.kind == "notebook_code" else [] + required_parameters = dict(expr_result.required_parameters) + else: + # Fall back to raw string representation + append_value = value_raw if isinstance(value_raw, str) else str(value_raw) + value_kind = "literal" + notebook_code = None + notebook_imports = [] + + append_var_activity = AppendVariableActivity( + **base_kwargs, + variable_name=variable_name, + append_value=append_value, + value_kind=value_kind, + notebook_code=notebook_code, + notebook_imports=notebook_imports, + required_parameters=required_parameters, + ) + + new_context = context.with_variable(variable_name, base_kwargs["task_key"]) + + return append_var_activity, new_context diff --git a/src/orchestra/translator/activity_translators/copy.py b/src/orchestra/translator/activity_translators/copy.py new file mode 100644 index 0000000..d8a519b --- /dev/null +++ b/src/orchestra/translator/activity_translators/copy.py @@ -0,0 +1,368 @@ +"""Translates ADF Copy activities to Databricks CopyActivity IR.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, CopyActivity, TranslationContext +from flowx.parser.expression_parser import ( + resolve_expression, + resolve_interpolated_string, + resolve_interpolated_string_for_notebook, +) + +_DATASET_TYPE_TO_SPARK_FORMAT: dict[str, str] = { + "DelimitedText": "csv", + "Parquet": "parquet", + "Json": "json", + "Avro": "avro", + "Orc": "orc", + "Binary": "binaryFile", + "DeltaLakeDataset": "delta", +} + +# Map ADF dataset location types to (uri-scheme, host-template) pairs used +# when constructing the external-volume URL. ``{account}`` is replaced with +# the storage account name (or a ``${var.storage_account}`` placeholder when +# the linked service does not expose one) and ``{bucket}`` with the bucket +# name for AWS / GCS sinks. +_LOCATION_URL_TEMPLATE: dict[str, str] = { + "AzureBlobFSLocation": "abfss://{container}@{account}.dfs.core.windows.net/", + "AzureBlobStorageLocation": "abfss://{container}@{account}.dfs.core.windows.net/", + "AmazonS3Location": "s3://{bucket}/", + "GoogleCloudStorageLocation": "gs://{bucket}/", +} + +# Regex to pull AccountName=... from an Azure storage connection string when +# the secret value is plaintext (rare in az exports, but supported). +_ACCOUNT_NAME_RE = re.compile(r"AccountName=([A-Za-z0-9]+)", re.IGNORECASE) +_DATASET_PARAM_RE = re.compile(r"^@dataset\(\)\.([A-Za-z_][A-Za-z0-9_]*)$") + + +@dataclass(slots=True) +class SinkPathInfo: + """Resolved sink dataset location for an external UC volume. + + Attributes: + location_type: ADF location type (``AzureBlobStorageLocation``, ...). + container: Storage container or bucket name (e.g. ``exports``). + folder: Folder path inside the container (already expression-resolved). + filename: File name at the leaf, may be empty for "folder of files". + storage_account: Account name when the linked service exposed it, + otherwise ``None`` so the bundler emits a ``${var.storage_account}`` + placeholder. + external_location_url: ``abfss://...`` / ``s3://...`` / ``gs://...`` + URL for the external location and volume. + volume_name: Sanitised UC volume name (typically the container name). + volume_relative_path: Path inside the volume root, ready to append + to ``/Volumes////``. + uc_volume_path: Full ``/Volumes/${var.catalog}/${var.schema}/...`` + path the generated notebook should write to. + """ + + location_type: str + container: str + folder: str + filename: str + storage_account: str | None + external_location_url: str + volume_name: str + volume_relative_path: str + uc_volume_path: str + + +def _dataset_props(dataset_ref: Any, definitions: AdfDefinitions) -> dict[str, Any] | None: + """Return the ``properties`` dict for an input/output dataset reference.""" + dataset = definitions.datasets.get(dataset_ref.reference_name) + if not dataset: + return None + return dict(dataset.properties or {}) + + +def _sanitize_volume_name(value: str) -> str: + """Sanitises an ADF container name for use as a UC volume name.""" + cleaned = re.sub(r"[^A-Za-z0-9_]", "_", value or "default_volume").strip("_") + return cleaned or "default_volume" + + +def _resolve_param_value( + raw: Any, + dataset_params: dict[str, Any], + context: TranslationContext, + *, + for_notebook: bool = False, +) -> str: + """Resolves a single ADF location field to a string.""" + if raw is None: + return "" + if isinstance(raw, dict) and raw.get("type") == "Expression": + raw = raw.get("value", "") + if not isinstance(raw, str): + return str(raw) + text = raw + + match = _DATASET_PARAM_RE.match(text.strip()) + if match: + param_name = match.group(1) + return _resolve_param_value( + dataset_params.get(param_name, ""), dataset_params, context, for_notebook=for_notebook + ) + + if "@{" in text: + if for_notebook: + return resolve_interpolated_string_for_notebook(text, context) + return resolve_interpolated_string(text, context) + + if text.startswith("@"): + result = resolve_expression(text, context) + if result is not None and result.kind in ("literal", "dab_ref"): + return result.value + return text + + return text + + +def _resolve_storage_account(linked_service: Any) -> str | None: + """Tries to pull a storage account name out of a linked service, if present.""" + if linked_service is None: + return None + type_props = linked_service.properties.get("typeProperties") or linked_service.properties + + url = type_props.get("url") or "" + if isinstance(url, str) and url: + host = url.replace("https://", "").split("/", 1)[0] + host_no_port = host.split(":", 1)[0] + if "." in host_no_port: + return host_no_port.split(".", 1)[0] + + sas_uri = type_props.get("sasUri") or "" + if isinstance(sas_uri, str) and sas_uri: + host = sas_uri.split("?", 1)[0].replace("https://", "").split("/", 1)[0] + if "." in host: + return host.split(".", 1)[0] + + # Plaintext connection string (rare in az exports — usually masked). + conn_string = type_props.get("connectionString") + if isinstance(conn_string, str): + match = _ACCOUNT_NAME_RE.search(conn_string) + if match: + return match.group(1) + if isinstance(conn_string, dict): + value = conn_string.get("value", "") + match = _ACCOUNT_NAME_RE.search(value) + if match: + return match.group(1) + + # AWS — bucket name lives on the dataset, account is implicit. + # Nothing useful to return at the linked-service level for S3/GCS. + return None + + +def _resolve_dataset_path(dataset_props: dict[str, Any], definitions: AdfDefinitions) -> str | None: + """Resolves a dataset's storage path using its location + linked service.""" + type_props = dataset_props.get("typeProperties") or dataset_props + location = type_props.get("location") or {} + + file_system = location.get("fileSystem") or location.get("container") or "" + folder_path = location.get("folderPath") or "" + if isinstance(file_system, dict) or isinstance(folder_path, dict): + return None # parameterised; caller handles via _resolve_path_info + + linked_service_ref = dataset_props.get("linkedServiceName") or {} + if isinstance(linked_service_ref, dict): + linked_service_name = linked_service_ref.get("referenceName", "") + else: + linked_service_name = str(linked_service_ref) + linked_service = definitions.linked_services.get(linked_service_name) if linked_service_name else None + account = _resolve_storage_account(linked_service) + if not account: + return None + + return f"abfss://{file_system}@{account}.dfs.core.windows.net/{folder_path}".rstrip("/") + + +def _resolve_path_info( + dataset_ref: Any, + dataset_props: dict[str, Any], + definitions: AdfDefinitions, + context: TranslationContext, +) -> SinkPathInfo | None: + """Resolves a (possibly parameterised) file-on-cloud-storage dataset.""" + type_props = dataset_props.get("typeProperties") or dataset_props + location = type_props.get("location") or {} + location_type = location.get("type") + if not location_type or location_type not in _LOCATION_URL_TEMPLATE: + return None + + declared = dataset_props.get("parameters") or {} + effective: dict[str, Any] = {} + for name, spec in declared.items(): + if isinstance(spec, dict) and "defaultValue" in spec: + effective[name] = spec["defaultValue"] + if dataset_ref is not None and getattr(dataset_ref, "parameters", None): + effective.update(dict(dataset_ref.parameters)) + + # Container name is used in the volume URL (no expressions allowed); the + # other path components flow into the notebook write call as f-string + # fragments so date/time expressions evaluate at runtime. + container = _resolve_param_value( + location.get("container") or location.get("fileSystem") or location.get("bucketName"), + effective, + context, + ) + folder = _resolve_param_value(location.get("folderPath"), effective, context, for_notebook=True).strip("/") + filename = _resolve_param_value(location.get("fileName"), effective, context, for_notebook=True).strip("/") + + if not container: + return None + + linked_service_ref = dataset_props.get("linkedServiceName") or {} + if isinstance(linked_service_ref, dict): + linked_service_name = linked_service_ref.get("referenceName", "") + else: + linked_service_name = str(linked_service_ref) + linked_service = definitions.linked_services.get(linked_service_name) if linked_service_name else None + storage_account = _resolve_storage_account(linked_service) if linked_service else None + + if location_type in ("AmazonS3Location", "GoogleCloudStorageLocation"): + external_url = _LOCATION_URL_TEMPLATE[location_type].format(bucket=container) + else: + account_token = storage_account or "${var.storage_account}" + external_url = _LOCATION_URL_TEMPLATE[location_type].format(container=container, account=account_token) + + volume_name = _sanitize_volume_name(container) + parts = [folder, filename] + relative = "/".join(part for part in parts if part) + volume_path = f"/Volumes/${{var.catalog}}/${{var.schema}}/{volume_name}" + if relative: + volume_path = f"{volume_path}/{relative}" + + return SinkPathInfo( + location_type=location_type, + container=container, + folder=folder, + filename=filename, + storage_account=storage_account, + external_location_url=external_url, + volume_name=volume_name, + volume_relative_path=relative, + uc_volume_path=volume_path, + ) + + +def _resolve_source_path(activity: AdfActivity, definitions: AdfDefinitions) -> str | None: + """Resolves the full storage path from the activity's input dataset.""" + if not activity.inputs: + return None + props = _dataset_props(activity.inputs[0], definitions) + if not props: + return None + return _resolve_dataset_path(props, definitions) + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a Copy activity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing datasets. + + Returns: + A :class:`CopyActivity` IR node. + """ + type_properties = activity.type_properties or {} + + source_raw = type_properties.get("source", {}) + source_type = source_raw.get("type") + source_properties = {k: v for k, v in source_raw.items() if k != "type"} if source_raw else {} + + resolved_path = _resolve_source_path(activity, definitions) + if resolved_path: + source_properties["resolved_path"] = resolved_path + + sink_raw = type_properties.get("sink", {}) + sink_type = sink_raw.get("type") + sink_properties = {k: v for k, v in sink_raw.items() if k != "type"} if sink_raw else {} + + column_mapping: list[dict[str, str]] = [] + translator_raw = type_properties.get("translator") + if translator_raw and isinstance(translator_raw, dict): + mappings = translator_raw.get("mappings", []) + for mapping in mappings: + source_col = mapping.get("source", {}) + sink_col = mapping.get("sink", {}) + if source_col and sink_col: + column_mapping.append( + { + "source_name": source_col.get("name", ""), + "source_type": source_col.get("type", ""), + "sink_name": sink_col.get("name", ""), + "sink_type": sink_col.get("type", ""), + } + ) + + # Resolve sink dataset metadata so the code generator can write to the + # actual target format and location instead of always defaulting to Delta. + sink_dataset_type: str | None = None + sink_format: str | None = None + sink_resolved_path: str | None = None + sink_table_name: str | None = None + if activity.outputs: + sink_dataset_ref = activity.outputs[0] + sink_dataset_props = _dataset_props(sink_dataset_ref, definitions) + if sink_dataset_props: + sink_dataset_type = sink_dataset_props.get("type") + sink_format = _DATASET_TYPE_TO_SPARK_FORMAT.get(sink_dataset_type or "") + + # File-on-cloud-storage sinks: compose a UC external volume + # path so the notebook writes through Unity Catalog and the + # bundler can emit the matching SetupTask (storage credential + # + external location + external volume). + sink_path_info = _resolve_path_info(sink_dataset_ref, sink_dataset_props, definitions, context) + if sink_path_info is not None: + sink_resolved_path = sink_path_info.uc_volume_path + sink_properties = { + **sink_properties, + "volume_name": sink_path_info.volume_name, + "volume_external_location": sink_path_info.external_location_url, + "volume_relative_path": sink_path_info.volume_relative_path, + "volume_location_type": sink_path_info.location_type, + } + if sink_path_info.storage_account: + sink_properties["volume_storage_account"] = sink_path_info.storage_account + else: + # Non-file sinks: fall back to the simpler resolver (returns + # an abfss:// path or None for tables). + sink_resolved_path = _resolve_dataset_path(sink_dataset_props, definitions) + + type_props = sink_dataset_props.get("typeProperties") or sink_dataset_props + sink_table_name = ( + type_props.get("tableName") or type_props.get("table") or sink_dataset_props.get("tableName") + ) + + if sink_table_name: + sink_properties = {**sink_properties, "table": sink_table_name} + if sink_resolved_path: + sink_properties = {**sink_properties, "resolved_path": sink_resolved_path} + + return CopyActivity( + **base_kwargs, + source_type=source_type, + sink_type=sink_type, + source_properties=source_properties, + sink_properties=sink_properties, + sink_dataset_type=sink_dataset_type, + sink_format=sink_format, + sink_resolved_path=sink_resolved_path, + column_mapping=column_mapping if column_mapping else None, + ) diff --git a/src/orchestra/translator/activity_translators/databricks_job.py b/src/orchestra/translator/activity_translators/databricks_job.py new file mode 100644 index 0000000..f39f084 --- /dev/null +++ b/src/orchestra/translator/activity_translators/databricks_job.py @@ -0,0 +1,44 @@ +"""Translates ADF DatabricksJob activities to Databricks RunJobActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, RunJobActivity, TranslationContext +from flowx.translator.activity_translators.resolve import resolve_dict_values, resolve_field + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a DatabricksJob activity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + + Returns: + A :class:`RunJobActivity` IR node. + """ + type_properties = activity.type_properties or {} + + job_name_raw = type_properties.get("jobName") or type_properties.get("jobId") + job_name = resolve_field(job_name_raw, context) if job_name_raw else activity.name + existing_job_id = type_properties.get("jobId") + job_parameters = ( + resolve_dict_values(type_properties.get("jobParameters") or type_properties.get("baseParameters"), context) + or None + ) + + return RunJobActivity( + **base_kwargs, + job_name=job_name, + existing_job_id=str(existing_job_id) if existing_job_id else None, + job_parameters=job_parameters, + ) diff --git a/src/orchestra/translator/activity_translators/delete.py b/src/orchestra/translator/activity_translators/delete.py new file mode 100644 index 0000000..e133339 --- /dev/null +++ b/src/orchestra/translator/activity_translators/delete.py @@ -0,0 +1,56 @@ +"""Translates ADF Delete activities to Databricks DeleteActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, DeleteActivity, TranslationContext +from flowx.translator.activity_translators.resolve import resolve_field + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a Delete activity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing datasets. + + Returns: + A :class:`DeleteActivity` IR node. + """ + type_properties = activity.type_properties or {} + + dataset_name = "" + if activity.inputs: + dataset_name = activity.inputs[0].reference_name + + recursive = type_properties.get("recursive", True) + folder_path_raw = ( + type_properties.get("dataset", {}).get("folderPath") + if isinstance(type_properties.get("dataset"), dict) + else None + ) + folder_path = resolve_field(folder_path_raw, context) if folder_path_raw is not None else None + + store_settings = type_properties.get("storeSettings", {}) + wildcard_folder_path_raw = store_settings.get("wildcardFolderPath") + wildcard_folder_path = ( + resolve_field(wildcard_folder_path_raw, context) if wildcard_folder_path_raw is not None else None + ) + + effective_folder = folder_path or wildcard_folder_path + + return DeleteActivity( + **base_kwargs, + dataset_name=dataset_name, + folder_path=effective_folder, + recursive=recursive, + ) diff --git a/src/orchestra/translator/activity_translators/execute_pipeline.py b/src/orchestra/translator/activity_translators/execute_pipeline.py new file mode 100644 index 0000000..6c2eef1 --- /dev/null +++ b/src/orchestra/translator/activity_translators/execute_pipeline.py @@ -0,0 +1,46 @@ +"""Translates ADF ExecutePipeline activities to Databricks ExecutePipelineActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, ExecutePipelineActivity, TranslationContext +from flowx.translator.activity_translators.resolve import resolve_dict_values, resolve_field + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates an ExecutePipeline activity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing pipelines. + + Returns: + An :class:`ExecutePipelineActivity` IR node. + """ + type_properties = activity.type_properties or {} + + pipeline_ref = type_properties.get("pipeline", {}) + pipeline_name = ( + resolve_field(pipeline_ref.get("referenceName", ""), context) + if isinstance(pipeline_ref, dict) + else str(pipeline_ref) + ) + + parameters = resolve_dict_values(type_properties.get("parameters"), context) or {} + wait_on_completion = type_properties.get("waitOnCompletion", True) + + return ExecutePipelineActivity( + **base_kwargs, + pipeline_name=pipeline_name, + parameters=parameters, + wait_on_completion=wait_on_completion, + ) diff --git a/src/orchestra/translator/activity_translators/filter.py b/src/orchestra/translator/activity_translators/filter.py new file mode 100644 index 0000000..2f3a104 --- /dev/null +++ b/src/orchestra/translator/activity_translators/filter.py @@ -0,0 +1,69 @@ +"""Translates ADF Filter activities to Databricks FilterActivity IR.""" + +from __future__ import annotations + +import re +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, FilterActivity, TranslationContext +from flowx.parser.expression_parser import resolve_expression + +# The expression resolver translates ``item().X`` into +# ``dbutils.widgets.get('X')`` because ``{{input.X}}`` is the DAB ref it +# emits for ForEach-iteration item access. Inside a Filter notebook the +# items array is iterated locally with a Python ``item`` dict per +# iteration, so we rewrite each widget read to a dict lookup. +_WIDGET_ITEM_ACCESS_RE = re.compile(r"""dbutils\.widgets\.get\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\)""") + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a Filter activity, pre-resolving the condition where safe.""" + type_properties = activity.type_properties or {} + items_raw = type_properties.get("items", {}) + condition_raw = type_properties.get("condition", {}) + + items_result = resolve_expression(items_raw, context) + if items_result is not None: + items_expression = items_result.value + else: + items_expression = items_raw.get("value", "") if isinstance(items_raw, dict) else str(items_raw) + + # Preserve the original ADF expression text in ``condition_expression`` + # so the notebook can show it as a documentation comment. The + # *resolved* form lives separately in ``condition_code``. + condition_expression = condition_raw.get("value", "") if isinstance(condition_raw, dict) else str(condition_raw) + + condition_result = resolve_expression(condition_raw, context) + condition_code, condition_imports = _resolve_condition_code(condition_result) + + return FilterActivity( + **base_kwargs, + items_expression=items_expression, + condition_expression=condition_expression, + condition_code=condition_code, + condition_imports=condition_imports, + ) + + +def _resolve_condition_code(condition_result: Any) -> tuple[str | None, list[str]]: + """Returns (python_expression, imports) for an ADF Filter condition. + + Returns ``(None, [])`` when the condition cannot be safely lowered to + Python -- the generator falls back to a TODO placeholder notebook. + A condition is unsafe to lower when the resolver returned ``None``, + when the result kind is not ``notebook_code``, or when the resolved + text still carries unresolved DAB-syntax markers (``{{...}}``) the + generator can't legally evaluate at notebook runtime. + """ + if condition_result is None or condition_result.kind != "notebook_code": + return None, [] + rewritten = _WIDGET_ITEM_ACCESS_RE.sub(r"item.get('\1')", condition_result.value) + if "{{" in rewritten: + return None, [] + return rewritten, list(condition_result.imports or []) diff --git a/src/orchestra/translator/activity_translators/for_each.py b/src/orchestra/translator/activity_translators/for_each.py new file mode 100644 index 0000000..54146b3 --- /dev/null +++ b/src/orchestra/translator/activity_translators/for_each.py @@ -0,0 +1,77 @@ +"""Translates ADF ForEach activities to Databricks ForEachActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, ForEachActivity, TranslationContext +from flowx.parser.expression_parser import resolve_expression +from flowx.translator.activity_translators.resolve import resolve_field_int + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, + *, + translate_activities_fn: Any = None, +) -> tuple[Activity, TranslationContext]: + """Translates a ForEach activity with recursive inner translation. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + translate_activities_fn: Callback to translate inner activities. + Signature: ``(activities, context, definitions) -> (list[Activity], TranslationContext)``. + + Returns: + Tuple of ``(ForEachActivity, updated_context)``. + """ + type_properties = activity.type_properties or {} + + items_raw = type_properties.get("items") + expr_result = resolve_expression(items_raw, context) if items_raw is not None else None + if expr_result is not None and expr_result.kind in ("dab_ref", "literal"): + items_expression = expr_result.value + else: + # Fallback: extract raw string + if isinstance(items_raw, dict) and items_raw.get("type") == "Expression": + items_expression = items_raw.get("value", "") + elif isinstance(items_raw, str): + items_expression = items_raw + else: + items_expression = "" + + is_sequential = type_properties.get("isSequential", False) + default_batch = 1 if is_sequential else 20 + batch_count_raw = type_properties.get("batchCount") + batch_count = ( + resolve_field_int(batch_count_raw, context, default=default_batch) + if batch_count_raw is not None + else default_batch + ) + + inner_activities: list[Activity] = [] + child_adf_activities = activity.activities or [] + + if translate_activities_fn and child_adf_activities: + child_context = TranslationContext( + activity_cache=context.activity_cache, + registry=context.registry, + variable_cache=context.variable_cache, + variable_value_cache=context.variable_value_cache, + ) + inner_activities, _ = translate_activities_fn(child_adf_activities, child_context, definitions) + + foreach_activity = ForEachActivity( + **base_kwargs, + items_expression=items_expression, + inner_activities=inner_activities, + concurrency=batch_count, + ) + + return foreach_activity, context diff --git a/src/orchestra/translator/activity_translators/if_condition.py b/src/orchestra/translator/activity_translators/if_condition.py new file mode 100644 index 0000000..c450f1d --- /dev/null +++ b/src/orchestra/translator/activity_translators/if_condition.py @@ -0,0 +1,242 @@ +"""Translates ADF IfCondition activities to Databricks IfConditionActivity IR. + +References: +- https://docs.databricks.com/aws/en/jobs/conditional-tasks +""" + +from __future__ import annotations + +import re +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, IfConditionActivity, TranslationContext +from flowx.parser.expression_parser import resolve_expression + +# --------------------------------------------------------------------------- +# ADF comparison function -> Databricks condition_task op mapping +# --------------------------------------------------------------------------- + +_OP_MAP: dict[str, str] = { + "equals": "EQUAL_TO", + "greater": "GREATER_THAN", + "greaterorequals": "GREATER_THAN_OR_EQUAL", + "less": "LESS_THAN", + "lessorequals": "LESS_THAN_OR_EQUAL", + "not": "NOT_EQUAL", +} + +_COMPARISON_RE = re.compile( + r"(equals|greater|greaterOrEquals|less|lessOrEquals|not)\s*\((.+)\)", + re.IGNORECASE | re.DOTALL, +) + +_NOT_COMPARISON_RE = re.compile( + r"not\s*\(\s*(equals|greater|greaterOrEquals|less|lessOrEquals)\s*\((.+)\)\s*\)", + re.IGNORECASE | re.DOTALL, +) + +_NEGATE_OP_MAP: dict[str, str] = { + "equals": "NOT_EQUAL", + "greater": "LESS_THAN_OR_EQUAL", + "greaterorequals": "LESS_THAN", + "less": "GREATER_THAN_OR_EQUAL", + "lessorequals": "GREATER_THAN", +} + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, + *, + translate_activities_fn: Any = None, +) -> tuple[Activity, TranslationContext]: + """Translates an IfCondition activity with recursive branch translation. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + translate_activities_fn: Callback to translate branch activities. + Signature: ``(activities, context, definitions) -> (list[Activity], TranslationContext)``. + + Returns: + Tuple of ``(IfConditionActivity, updated_context)``. + """ + type_properties = activity.type_properties or {} + + expression_raw = type_properties.get("expression", {}) + op, left, right = _parse_condition(expression_raw, context) + + if_true_activities: list[Activity] = [] + if_true_adf = activity.if_true_activities or [] + if translate_activities_fn and if_true_adf: + if_true_activities, _ = translate_activities_fn(if_true_adf, context, definitions) + + if_false_activities: list[Activity] = [] + if_false_adf = activity.if_false_activities or [] + if translate_activities_fn and if_false_adf: + if_false_activities, _ = translate_activities_fn(if_false_adf, context, definitions) + + if_activity = IfConditionActivity( + **base_kwargs, + op=op, + left=left, + right=right, + if_true_activities=if_true_activities, + if_false_activities=if_false_activities, + ) + + return if_activity, context + + +def _parse_condition(expression: dict[str, Any] | str, context: TranslationContext) -> tuple[str, str, str]: + """Parses an ADF IfCondition expression into ``(op, left, right)``. + + Args: + expression: Raw ADF expression dict or string. + context: Translation context for resolving variables. + + Returns: + Tuple of ``(databricks_op, left_operand, right_operand)``. + """ + expr_str = "" + if isinstance(expression, dict): + expr_str = expression.get("value", "") + elif isinstance(expression, str): + expr_str = expression + + if expr_str.startswith("@"): + expr_str = expr_str[1:] + + m_not = _NOT_COMPARISON_RE.match(expr_str.strip()) + if m_not: + inner_op_name = m_not.group(1).lower() + op = _NEGATE_OP_MAP.get(inner_op_name, "NOT_EQUAL") + args = _split_args(m_not.group(2).strip()) + left = _resolve_operand(args[0], context) if len(args) > 0 else "" + right = _resolve_operand(args[1], context) if len(args) > 1 else "" + return op, left, right + + m = _COMPARISON_RE.match(expr_str.strip()) + if m: + adf_op = m.group(1).lower() + op = _OP_MAP.get(adf_op, adf_op.upper()) + + if adf_op == "not": + inner = m.group(2).strip() + resolved = _resolve_operand(inner, context) + return "NOT_EQUAL", resolved, "" + + args = _split_args(m.group(2).strip()) + left = _resolve_operand(args[0], context) if len(args) > 0 else "" + right = _resolve_operand(args[1], context) if len(args) > 1 else "" + return op, left, right + + # Fallback: treat the whole expression as a truthy check + resolved = _resolve_operand(expr_str, context) + return "NOT_EQUAL", resolved, "0" + + +def _resolve_operand(operand: str, context: TranslationContext) -> str: + """Converts an ADF expression operand to a Databricks task value reference. + + Examples:: + + activity('Lookup').output.firstRow.cnt + -> {{tasks.Lookup.values.cnt}} + + activity('Lookup').output.value + -> {{tasks.Lookup.values.result}} + + 0 -> 0 (literal) + 'active' -> active (string literal) + null -> "" (null literal) + + Args: + operand: A single operand string from the parsed condition. + context: Translation context for resolving variables. + + Returns: + A DAB dynamic value reference or literal string. + """ + operand = operand.strip() + + if operand.lower() == "null": + return "" + + if operand.startswith("'") and operand.endswith("'"): + return operand[1:-1] + + if operand.lstrip("-").replace(".", "", 1).isdigit(): + return operand + + inner = _unwrap_functions(operand) + + # Try unified expression resolution with @ prefix + result = resolve_expression("@" + inner, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + + if inner != operand: + result = resolve_expression("@" + operand, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + + return operand + + +def _unwrap_functions(expr: str) -> str: + """Strips wrapping ADF functions like ``int(...)`` to expose the inner expression. + + Args: + expr: Expression that may be wrapped in a type-casting function. + + Returns: + The inner expression, or the original if no wrapping detected. + """ + m = re.match(r"(?:int|string|float|bool)\s*\((.+)\)\s*$", expr, re.IGNORECASE) + if m: + return m.group(1).strip() + return expr + + +def _split_args(args_str: str) -> list[str]: + """Splits function arguments respecting nested parentheses and quotes. + + Args: + args_str: Comma-separated argument string. + + Returns: + List of argument strings, stripped of leading/trailing whitespace. + """ + parts: list[str] = [] + depth = 0 + current: list[str] = [] + in_quote = False + + for ch in args_str: + if ch == "'" and depth == 0: + in_quote = not in_quote + current.append(ch) + elif in_quote: + current.append(ch) + elif ch == "(": + depth += 1 + current.append(ch) + elif ch == ")": + depth -= 1 + current.append(ch) + elif ch == "," and depth == 0: + parts.append("".join(current).strip()) + current = [] + else: + current.append(ch) + + if current: + parts.append("".join(current).strip()) + + return parts diff --git a/src/orchestra/translator/activity_translators/lookup.py b/src/orchestra/translator/activity_translators/lookup.py new file mode 100644 index 0000000..3620df8 --- /dev/null +++ b/src/orchestra/translator/activity_translators/lookup.py @@ -0,0 +1,48 @@ +"""Translates ADF Lookup activities to Databricks LookupActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, LookupActivity, TranslationContext +from flowx.translator.activity_translators.resolve import resolve_field + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a Lookup activity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing datasets. + + Returns: + A :class:`LookupActivity` IR node. + """ + type_properties = activity.type_properties or {} + + source_raw = type_properties.get("source", {}) + source_type = source_raw.get("type") + source_properties = {k: v for k, v in source_raw.items() if k != "type"} if source_raw else {} + + source_query_raw = ( + source_raw.get("query") or source_raw.get("sqlReaderQuery") or source_raw.get("sqlReaderStoredProcedureName") + ) + source_query = resolve_field(source_query_raw, context) if source_query_raw is not None else None + + first_row_only = type_properties.get("firstRowOnly", True) + + return LookupActivity( + **base_kwargs, + source_type=source_type, + source_properties=source_properties, + first_row_only=first_row_only, + source_query=source_query, + ) diff --git a/src/orchestra/translator/activity_translators/notebook.py b/src/orchestra/translator/activity_translators/notebook.py new file mode 100644 index 0000000..3fc5897 --- /dev/null +++ b/src/orchestra/translator/activity_translators/notebook.py @@ -0,0 +1,51 @@ +"""Translates ADF DatabricksNotebook activities to Databricks NotebookActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, NotebookActivity, TranslationContext +from flowx.parser.expression_parser import resolve_expression +from flowx.translator.activity_translators.resolve import resolve_field + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a DatabricksNotebook activity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + + Returns: + A :class:`NotebookActivity` IR node. + """ + type_properties = activity.type_properties or {} + + notebook_path = resolve_field(type_properties.get("notebookPath", ""), context) + raw_params = type_properties.get("baseParameters") or {} + + # Resolve base_parameters at translate time so ADF expressions like + # @variables('runTimestamp') are inlined to DAB refs while the full + # translation context (with variable_value_cache) is available. + resolved_params: dict[str, Any] = {} + for key, value in raw_params.items(): + result = resolve_expression(value, context) + if result is not None and result.kind in ("dab_ref", "literal"): + resolved_params[key] = result.value + else: + # Keep original for downstream handling (notebook_code or unresolvable) + resolved_params[key] = value + + return NotebookActivity( + **base_kwargs, + notebook_path=notebook_path, + base_parameters=resolved_params, + ) diff --git a/src/orchestra/translator/activity_translators/resolve.py b/src/orchestra/translator/activity_translators/resolve.py new file mode 100644 index 0000000..fd6f496 --- /dev/null +++ b/src/orchestra/translator/activity_translators/resolve.py @@ -0,0 +1,64 @@ +"""Shared field resolution helper for activity translators.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.ir import TranslationContext +from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string + + +def resolve_field(value: Any, context: TranslationContext) -> str: + """Resolves a field value that may contain an ADF expression. + + Args: + value: The raw field value from ADF type properties. + context: Translation context for variable resolution. + + Returns: + Resolved string value. + """ + if value is None: + return "" + result = resolve_expression(value, context) + if result is not None: + return result.value + # Fallback: unwrap expression dicts, return raw string + if isinstance(value, dict) and value.get("type") == "Expression": + return value.get("value", "") + if isinstance(value, str) and "@{" in value: + return resolve_interpolated_string(value, context) + return str(value) if not isinstance(value, str) else value + + +def resolve_field_int(value: Any, context: TranslationContext, default: int = 0) -> int: + """Resolves a field to an integer, handling expressions that resolve to literals. + + Args: + value: The raw field value from ADF type properties. + context: Translation context for variable resolution. + default: Fallback value when conversion fails. + + Returns: + Resolved integer value. + """ + resolved = resolve_field(value, context) + try: + return int(resolved) + except (ValueError, TypeError): + return default + + +def resolve_dict_values(d: dict[str, Any] | None, context: TranslationContext) -> dict[str, str]: + """Resolves all values in a dict that may contain ADF expressions. + + Args: + d: Dict of field name to raw values. + context: Translation context for variable resolution. + + Returns: + Dict with all values resolved to strings. + """ + if not d: + return {} + return {k: resolve_field(v, context) for k, v in d.items()} diff --git a/src/orchestra/translator/activity_translators/set_variable.py b/src/orchestra/translator/activity_translators/set_variable.py new file mode 100644 index 0000000..6ddc3e1 --- /dev/null +++ b/src/orchestra/translator/activity_translators/set_variable.py @@ -0,0 +1,77 @@ +"""Translates ADF SetVariable activities to Databricks SetVariableActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, SetVariableActivity, TranslationContext +from flowx.parser.expression_parser import resolve_expression + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> tuple[Activity, TranslationContext]: + """Translates a SetVariable activity and register the variable in context. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + + Returns: + Tuple of ``(SetVariableActivity, updated_context)`` where the context + now maps the variable name to this activity's task key. + """ + type_properties = activity.type_properties or {} + + variable_name = type_properties.get("variableName", "") + value_raw = type_properties.get("value", "") + + expr_result = resolve_expression(value_raw, context) + + required_parameters: dict[str, str] = {} + if expr_result is not None: + variable_value = expr_result.value + value_kind = expr_result.kind + notebook_code = expr_result.value if expr_result.kind == "notebook_code" else None + notebook_imports = list(expr_result.imports) if expr_result.kind == "notebook_code" else [] + required_parameters = dict(expr_result.required_parameters) + else: + # Fallback: unwrap expression-type dicts to at least preserve the string + if isinstance(value_raw, dict) and value_raw.get("type") == "Expression": + variable_value = value_raw.get("value", "") + elif isinstance(value_raw, str): + variable_value = value_raw + else: + variable_value = str(value_raw) + value_kind = "literal" + notebook_code = None + notebook_imports = [] + + set_var_activity = SetVariableActivity( + **base_kwargs, + variable_name=variable_name, + variable_value=variable_value, + value_kind=value_kind, + notebook_code=notebook_code, + notebook_imports=notebook_imports, + required_parameters=required_parameters, + ) + + # Register variable -> task_key mapping in context. + # When the value is a DAB ref (e.g. {{job.start_time.iso_datetime}} from + # @utcNow()), store it so downstream @variables() calls can inline it + # instead of routing through the task value. + dab_ref_value = variable_value if value_kind == "dab_ref" else None + new_context = context.with_variable( + variable_name, + base_kwargs["task_key"], + dab_ref_value=dab_ref_value, + ) + + return set_var_activity, new_context diff --git a/src/orchestra/translator/activity_translators/spark_jar.py b/src/orchestra/translator/activity_translators/spark_jar.py new file mode 100644 index 0000000..4da5c6f --- /dev/null +++ b/src/orchestra/translator/activity_translators/spark_jar.py @@ -0,0 +1,70 @@ +"""Translates ADF DatabricksSparkJar activities to Databricks SparkJarActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, SparkJarActivity, TranslationContext +from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string +from flowx.translator.activity_translators.resolve import resolve_field + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a DatabricksSparkJar activity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + + Returns: + A :class:`SparkJarActivity` IR node. + """ + type_properties = activity.type_properties or {} + + main_class_name = resolve_field(type_properties.get("mainClassName", ""), context) + raw_parameters = type_properties.get("parameters") or [] + libraries = type_properties.get("libraries") or [] + + parameters = [_resolve_parameter(param, context) for param in raw_parameters] + + return SparkJarActivity( + **base_kwargs, + main_class_name=main_class_name, + parameters=parameters, + libraries=libraries, + ) + + +def _resolve_parameter(param: Any, context: TranslationContext) -> str: + """Resolves a single ADF parameter to a DAB value string. + + Args: + param: A parameter string or ``{"type": "Expression", "value": "..."}`` dict. + context: Translation context for variable resolution. + + Returns: + Resolved parameter string. + """ + if isinstance(param, dict) and param.get("type") == "Expression": + return _resolve_parameter(param["value"], context) + + if not isinstance(param, str): + return str(param) + + if "@{" in param: + return resolve_interpolated_string(param, context) + + if param.startswith("@"): + result = resolve_expression(param, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + + return param diff --git a/src/orchestra/translator/activity_translators/spark_python.py b/src/orchestra/translator/activity_translators/spark_python.py new file mode 100644 index 0000000..8dc8892 --- /dev/null +++ b/src/orchestra/translator/activity_translators/spark_python.py @@ -0,0 +1,65 @@ +"""Translates ADF DatabricksSparkPython activities to Databricks SparkPythonActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, SparkPythonActivity, TranslationContext +from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string +from flowx.translator.activity_translators.resolve import resolve_field + + +def _resolve_parameter(param: str, context: TranslationContext) -> str: + """Resolves a single ADF parameter string to a DAB value. + + Args: + param: A parameter string that may contain ADF expressions. + context: Translation context for variable resolution. + + Returns: + Resolved parameter string. + """ + if not isinstance(param, str): + return param + + if "@{" in param: + return resolve_interpolated_string(param, context) + + if param.startswith("@"): + result = resolve_expression(param, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + + return param + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a DatabricksSparkPython activity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + + Returns: + A :class:`SparkPythonActivity` IR node. + """ + type_properties = activity.type_properties or {} + + python_file = resolve_field(type_properties.get("pythonFile", ""), context) + raw_parameters = type_properties.get("parameters") or [] + + parameters = [_resolve_parameter(p, context) for p in raw_parameters] + + return SparkPythonActivity( + **base_kwargs, + python_file=python_file, + parameters=parameters, + ) diff --git a/src/orchestra/translator/activity_translators/switch.py b/src/orchestra/translator/activity_translators/switch.py new file mode 100644 index 0000000..095840d --- /dev/null +++ b/src/orchestra/translator/activity_translators/switch.py @@ -0,0 +1,121 @@ +"""Translates ADF Switch activities to Databricks SwitchActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, SwitchActivity, SwitchCase, TranslationContext +from flowx.parser.adf_loader import parse_activity +from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string +from flowx.translator.activity_translators.resolve import resolve_field + + +def _resolve_on_expression(on_expression: str, context: TranslationContext) -> str: + """Resolves the ``on`` expression to a DAB dynamic value ref. + + Args: + on_expression: Raw ADF on-expression string. + context: Translation context for resolving variables. + + Returns: + Resolved DAB ref string, or the original if unresolvable. + """ + if "@{" in on_expression: + return resolve_interpolated_string(on_expression, context) + + if on_expression.startswith("@"): + result = resolve_expression(on_expression, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + + return on_expression + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, + *, + translate_activities_fn: Any = None, +) -> tuple[Activity, TranslationContext]: + """Translates a Switch activity with recursive branch translation. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + translate_activities_fn: Callback to translate branch activities. + Signature: ``(activities, context, definitions) -> (list[Activity], TranslationContext)``. + + Returns: + Tuple of ``(SwitchActivity, updated_context)``. + """ + type_properties = activity.type_properties or {} + + on_raw = type_properties.get("on", {}) + if isinstance(on_raw, dict): + on_expression_raw = on_raw.get("value", "") + elif isinstance(on_raw, str): + on_expression_raw = on_raw + else: + on_expression_raw = str(on_raw) if on_raw else "" + + on_expression = _resolve_on_expression(on_expression_raw, context) + + cases: list[SwitchCase] = [] + raw_cases = type_properties.get("cases", []) + for raw_case in raw_cases: + case_value = resolve_field(raw_case.get("value", ""), context) + case_activities_raw = raw_case.get("activities", []) + + case_adf_activities = _ensure_adf_activities(case_activities_raw) + + case_translated: list[Activity] = [] + if translate_activities_fn and case_adf_activities: + case_translated, _ = translate_activities_fn( + case_adf_activities, + context, + definitions, + ) + + cases.append(SwitchCase(value=case_value, activities=case_translated)) + + default_activities: list[Activity] = [] + default_raw = type_properties.get("defaultActivities", []) + default_adf_activities = _ensure_adf_activities(default_raw) + if translate_activities_fn and default_adf_activities: + default_activities, _ = translate_activities_fn( + default_adf_activities, + context, + definitions, + ) + + switch_activity = SwitchActivity( + **base_kwargs, + on_expression=on_expression, + cases=cases, + default_activities=default_activities, + ) + + return switch_activity, context + + +def _ensure_adf_activities(raw_activities: list[Any]) -> list[AdfActivity]: + """Ensure a list of activities are AdfActivity instances. + + Args: + raw_activities: List that may contain AdfActivity instances or raw dicts. + + Returns: + List of AdfActivity instances. + """ + result: list[AdfActivity] = [] + for item in raw_activities: + if isinstance(item, AdfActivity): + result.append(item) + elif isinstance(item, dict): + result.append(parse_activity(item)) + return result diff --git a/src/orchestra/translator/activity_translators/wait.py b/src/orchestra/translator/activity_translators/wait.py new file mode 100644 index 0000000..6577f06 --- /dev/null +++ b/src/orchestra/translator/activity_translators/wait.py @@ -0,0 +1,36 @@ +"""Translates ADF Wait activities to Databricks WaitActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, TranslationContext, WaitActivity +from flowx.translator.activity_translators.resolve import resolve_field_int + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a Wait activity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + + Returns: + A :class:`WaitActivity` IR node. + """ + type_properties = activity.type_properties or {} + + wait_time_seconds = resolve_field_int(type_properties.get("waitTimeInSeconds", 0), context, default=0) + + return WaitActivity( + **base_kwargs, + wait_time_seconds=wait_time_seconds, + ) diff --git a/src/orchestra/translator/activity_translators/web_activity.py b/src/orchestra/translator/activity_translators/web_activity.py new file mode 100644 index 0000000..7f33c6b --- /dev/null +++ b/src/orchestra/translator/activity_translators/web_activity.py @@ -0,0 +1,104 @@ +"""Translates ADF WebActivity activities to Databricks WebActivity IR.""" + +from __future__ import annotations + +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, TranslationContext +from flowx.models.ir import WebActivity as WebActivityIR +from flowx.parser.expression_parser import resolve_expression +from flowx.translator.activity_translators.resolve import resolve_dict_values, resolve_field + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a WebActivity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + + Returns: + A :class:`WebActivity` IR node. + """ + type_properties = activity.type_properties or {} + + url = resolve_field(type_properties.get("url", ""), context) + method = type_properties.get("method", "GET") + headers = resolve_dict_values(type_properties.get("headers"), context) or None + body = _resolve_body(type_properties.get("body"), context) + authentication = type_properties.get("authentication") + disable_cert_validation = type_properties.get("disableCertValidation", False) + http_request_timeout = type_properties.get("httpRequestTimeout") + + timeout_seconds: int | None = None + if http_request_timeout and isinstance(http_request_timeout, str): + timeout_seconds = _parse_timeout_to_seconds(http_request_timeout) + + return WebActivityIR( + **base_kwargs, + url=url, + method=method, + body=body, + headers=headers, + authentication=authentication, + disable_cert_validation=disable_cert_validation, + http_request_timeout_seconds=timeout_seconds, + ) + + +def _resolve_body(body: Any, context: TranslationContext) -> Any: + """Pre-resolve ADF expressions in the request body at translate time. + + Args: + body: Raw body from the ADF typeProperties. + context: Current translation context with variable caches. + + Returns: + Resolved body — either a Python code string (for notebook_code), + the original body dict, or ``None``. + """ + if body is None: + return None + + if isinstance(body, dict) and body.get("type") == "Expression" and "value" in body: + result = resolve_expression(body, context) + if result is not None and result.kind == "notebook_code": + return result.value + if result is not None and result.kind == "literal": + return result.value + + return body + + +def _parse_timeout_to_seconds(timeout_str: str) -> int | None: + """Parses an ADF timeout string to seconds. + + Args: + timeout_str: Timeout in ``"d.hh:mm:ss"`` or ``"hh:mm:ss"`` format. + + Returns: + Total seconds, or ``None`` if the format is unrecognised. + """ + try: + parts = timeout_str.split(".") + if len(parts) == 2: + days = int(parts[0]) + time_part = parts[1] + else: + days = 0 + time_part = parts[0] + time_parts = time_part.split(":") + hours = int(time_parts[0]) if len(time_parts) > 0 else 0 + minutes = int(time_parts[1]) if len(time_parts) > 1 else 0 + seconds = int(time_parts[2]) if len(time_parts) > 2 else 0 + return days * 86400 + hours * 3600 + minutes * 60 + seconds + except (ValueError, IndexError): + return None diff --git a/src/orchestra/translator/engine.py b/src/orchestra/translator/engine.py new file mode 100644 index 0000000..a9fa8b6 --- /dev/null +++ b/src/orchestra/translator/engine.py @@ -0,0 +1,807 @@ +"""Core translation engine for ADF-to-Databricks pipeline conversion.""" + +from __future__ import annotations + +import argparse +import json +import logging +import re +from collections import defaultdict +from dataclasses import asdict +from pathlib import Path +from types import MappingProxyType +from typing import Any, Callable + +from flowx.models.adf_ast import ( + AdfActivity, + AdfDefinitions, + AdfPipeline, + TranslationStrategy, +) +from flowx.models.ir import ( + Activity, + AgenticGap, + AppendVariableActivity, + CopyActivity, + DeleteActivity, + Dependency, + ExecutePipelineActivity, + FilterActivity, + ForEachActivity, + IfConditionActivity, + LookupActivity, + MotifActivity, + NotebookActivity, + Pipeline, + PlaceholderActivity, + RunJobActivity, + SetVariableActivity, + SparkJarActivity, + SparkPythonActivity, + SwitchActivity, + TranslationContext, + TranslationReport, + UnsupportedActivity, + WaitActivity, + WebActivity, +) +from flowx.motifs.collapser import collapse_motifs +from flowx.motifs.detector import detect_motifs +from flowx.parser.adf_loader import classify_activity, load_adf_definitions +from flowx.translator.activity_translators import ( + append_variable, + copy, + databricks_job, + delete, + execute_pipeline, + filter, + for_each, + if_condition, + lookup, + notebook, + set_variable, + spark_jar, + spark_python, + switch, + wait, + web_activity, +) + +logger = logging.getLogger(__name__) + +TRANSLATOR_REGISTRY: dict[str, Callable[..., Activity]] = { + "Copy": copy.translate, + "DatabricksNotebook": notebook.translate, + "DatabricksSparkJar": spark_jar.translate, + "DatabricksSparkPython": spark_python.translate, + "Lookup": lookup.translate, + "WebActivity": web_activity.translate, + "Delete": delete.translate, + "ExecutePipeline": execute_pipeline.translate, + "DatabricksJob": databricks_job.translate, + "Wait": wait.translate, + "Filter": filter.translate, +} + + +def translate_pipeline(pipeline: AdfPipeline, definitions: AdfDefinitions) -> TranslationReport: + """Translates an ADF pipeline into a Databricks pipeline IR. + + Args: + pipeline: Parsed ADF pipeline AST. + definitions: Full ADF definitions for cross-referencing datasets, + linked services, etc. + + Returns: + :class:`TranslationReport` containing the translated :class:`Pipeline` + and any gaps encountered. + """ + context = TranslationContext( + activity_cache=MappingProxyType({}), + registry=MappingProxyType(TRANSLATOR_REGISTRY), + variable_cache=MappingProxyType({}), + ) + + gaps: list[AgenticGap] = [] + warnings: list[str] = [] + + sorted_activities = _topological_visit(pipeline.activities) + translated_activities: list[Activity] = [] + deterministic_count = 0 + agentic_count = 0 + unsupported_count = 0 + + for adf_activity in sorted_activities: + activity_ir, context = _dispatch_activity(adf_activity, context, definitions) + translated_activities.append(activity_ir) + + strategy, skill = classify_activity(adf_activity.type) + if strategy is TranslationStrategy.DETERMINISTIC: + deterministic_count += 1 + elif strategy is TranslationStrategy.AGENTIC: + agentic_count += 1 + gaps.append( + AgenticGap( + activity_name=adf_activity.name, + activity_type=adf_activity.type, + recommended_skill=skill, + raw_definition=adf_activity.type_properties, + ) + ) + else: + unsupported_count += 1 + gaps.append( + AgenticGap( + activity_name=adf_activity.name, + activity_type=adf_activity.type, + recommended_skill=None, + raw_definition=adf_activity.type_properties, + ) + ) + warnings.append(f"Activity '{adf_activity.name}' (type={adf_activity.type}) has no translation path.") + + parameters: dict[str, Any] = {} + if pipeline.parameters: + for param_name, param_def in pipeline.parameters.items(): + parameters[param_name] = param_def.default_value + + pipeline_ir = Pipeline( + name=pipeline.name, + parameters=[{"name": param_name, "default": param_value} for param_name, param_value in parameters.items()] + if parameters + else None, + tasks=translated_activities, + tags={"source": "adf", "pipeline": pipeline.name}, + ) + + # Motif detection and collapsing: scan for known multi-activity patterns + # and replace matched groups with single MotifActivity nodes. + detected_motifs = detect_motifs(pipeline, definitions) + if detected_motifs: + pipeline_ir = collapse_motifs(pipeline_ir, detected_motifs) + for motif in detected_motifs: + logger.info( + "Collapsed motif '%s': %d activities -> %s", + motif.definition.display_name, + len(motif.matched_activities), + motif.definition.databricks_replacement, + ) + + return TranslationReport( + pipeline=pipeline_ir, + deterministic_count=deterministic_count, + agentic_count=agentic_count, + unsupported_count=unsupported_count, + gaps=gaps, + warnings=warnings, + ) + + +def _dispatch_activity( + activity: AdfActivity, + context: TranslationContext, + definitions: AdfDefinitions, +) -> tuple[Activity, TranslationContext]: + """Dispatch a single activity to its translator. + + Args: + activity: ADF activity AST node. + context: Current translation context. + definitions: Full ADF definitions. + + Returns: + Tuple of ``(translated_activity, updated_context)``. + """ + base_kwargs = _build_base_kwargs(activity, definitions) + + match activity.type: + case "ForEach": + result, context = for_each.translate( + activity, + base_kwargs, + context, + definitions, + translate_activities_fn=_translate_activity_list, + ) + context = context.with_activity(activity.name, result) + return result, context + + case "IfCondition": + result, context = if_condition.translate( + activity, + base_kwargs, + context, + definitions, + translate_activities_fn=_translate_activity_list, + ) + context = context.with_activity(activity.name, result) + return result, context + + case "SetVariable": + result, context = set_variable.translate( + activity, + base_kwargs, + context, + definitions, + ) + context = context.with_activity(activity.name, result) + return result, context + + case "AppendVariable": + result, context = append_variable.translate( + activity, + base_kwargs, + context, + definitions, + ) + context = context.with_activity(activity.name, result) + return result, context + + case "Switch": + result, context = switch.translate( + activity, + base_kwargs, + context, + definitions, + translate_activities_fn=_translate_activity_list, + ) + context = context.with_activity(activity.name, result) + return result, context + + case activity_type if activity_type in TRANSLATOR_REGISTRY: + translator_fn = TRANSLATOR_REGISTRY[activity_type] + result = translator_fn(activity, base_kwargs, context, definitions) + context = context.with_activity(activity.name, result) + return result, context + + case _: + strategy, skill = classify_activity(activity.type) + reason = f"Agentic skill: {skill}" if skill else f"No translator for type '{activity.type}'" + placeholder = PlaceholderActivity( + **base_kwargs, + original_type=activity.type, + comment=reason, + ) + context = context.with_activity(activity.name, placeholder) + return placeholder, context + + +def _translate_activity_list( + activities: list[AdfActivity], + context: TranslationContext, + definitions: AdfDefinitions, +) -> tuple[list[Activity], TranslationContext]: + """Translates a list of ADF activities, threading context through each. + + Args: + activities: List of ADF activity AST nodes. + context: Current translation context. + definitions: Full ADF definitions. + + Returns: + Tuple of ``(translated_activities, final_context)``. + """ + sorted_activities = _topological_visit(activities) + results: list[Activity] = [] + + for adf_activity in sorted_activities: + activity_ir, context = _dispatch_activity(adf_activity, context, definitions) + results.append(activity_ir) + + return results, context + + +def _topological_visit(activities: list[AdfActivity]) -> list[AdfActivity]: + """Return activities in dependency-first (topological) order. + + Args: + activities: Flat list of ADF activities at one nesting level. + + Returns: + Activities reordered so that dependencies are visited first. + """ + if not activities: + return [] + + name_to_activity: dict[str, AdfActivity] = {act.name: act for act in activities} + in_degree: dict[str, int] = {act.name: 0 for act in activities} + dependents: dict[str, list[str]] = defaultdict(list) + + for activity in activities: + if activity.depends_on: + for dependency in activity.depends_on: + dependency_name = dependency.activity + if dependency_name in name_to_activity: + in_degree[activity.name] += 1 + dependents[dependency_name].append(activity.name) + + queue: list[str] = [name for name, degree in in_degree.items() if degree == 0] + result: list[AdfActivity] = [] + + while queue: + queue.sort() + current = queue.pop(0) + result.append(name_to_activity[current]) + for dependent in dependents.get(current, []): + in_degree[dependent] -= 1 + if in_degree[dependent] == 0: + queue.append(dependent) + + if len(result) < len(activities): + visited = {act.name for act in result} + for activity in activities: + if activity.name not in visited: + logger.warning("Cycle detected: activity '%s' has unresolved dependencies.", activity.name) + result.append(activity) + + return result + + +_TIMEOUT_RE = re.compile(r"(?:(\d+)\.)?(\d{2}):(\d{2}):(\d{2})") + + +def _build_base_kwargs(activity: AdfActivity, definitions: AdfDefinitions) -> dict[str, Any]: + """Extracts common fields shared by all Activity IR subclasses. + + Args: + activity: ADF activity AST node. + definitions: Full ADF definitions for linked-service cluster config. + + Returns: + Dictionary with keys: ``name``, ``task_key``, ``timeout_seconds``, + ``max_retries``, ``depends_on``, ``cluster``. + """ + task_key = _sanitize_task_key(activity.name) + + timeout_seconds: int | None = None + if activity.policy and activity.policy.timeout: + timeout_seconds = _parse_adf_timeout(activity.policy.timeout) + + max_retries: int | None = None + if activity.policy and activity.policy.retry is not None: + max_retries = activity.policy.retry + + min_retry_interval_millis: int | None = None + if activity.policy and activity.policy.retry_interval_in_seconds is not None: + min_retry_interval_millis = activity.policy.retry_interval_in_seconds * 1000 + + depends_on: list[Dependency] | None = None + if activity.depends_on: + depends_on = [] + for dependency in activity.depends_on: + outcome = dependency.dependency_conditions[0] if dependency.dependency_conditions else None + depends_on.append( + Dependency( + task_key=_sanitize_task_key(dependency.activity), + outcome=outcome, + ) + ) + + cluster: dict[str, Any] | None = None + if activity.linked_service_name: + linked_service_name = activity.linked_service_name.reference_name + linked_service_def = definitions.linked_services.get(linked_service_name) + if linked_service_def: + cluster = _extract_cluster_config(linked_service_def.properties) + + return { + "name": activity.name, + "task_key": task_key, + "description": None, + "timeout_seconds": timeout_seconds, + "max_retries": max_retries, + "min_retry_interval_millis": min_retry_interval_millis, + "depends_on": depends_on, + "cluster": cluster, + } + + +def _sanitize_task_key(name: str) -> str: + """Converts an ADF activity name to a valid Databricks task key. + + Args: + name: ADF activity name. + + Returns: + Sanitised task key string. + """ + key = re.sub(r"[^a-zA-Z0-9_-]", "_", name) + key = re.sub(r"_+", "_", key) + key = key.strip("_") + return key or "unnamed" + + +def _parse_adf_timeout(timeout_str: str) -> int | None: + """Parses an ADF timeout string to total seconds. + + Args: + timeout_str: Timeout in ``"d.hh:mm:ss"`` or ``"hh:mm:ss"`` format. + + Returns: + Total seconds, or ``None`` if the format is unrecognised. + """ + match = _TIMEOUT_RE.match(timeout_str) + if not match: + return None + days = int(match.group(1) or 0) + hours = int(match.group(2)) + minutes = int(match.group(3)) + seconds = int(match.group(4)) + return days * 86400 + hours * 3600 + minutes * 60 + seconds + + +def _extract_cluster_config(ls_properties: dict[str, Any]) -> dict[str, Any] | None: + """Extracts Databricks cluster configuration from a linked-service properties dict. + + Args: + ls_properties: Full properties bag from the linked service JSON. + + Returns: + Cluster configuration dict, or ``None`` if no Databricks cluster + details are present. + """ + nested = ls_properties.get("typeProperties") or {} + # Merge: nested values win over flat ones when both exist (matches ARM + # template precedence). + fields: dict[str, Any] = {**ls_properties, **nested} + + config: dict[str, Any] = {} + + existing_cluster_id = fields.get("existingClusterId") + if existing_cluster_id: + config["existing_cluster_id"] = existing_cluster_id + + new_cluster = fields.get("newClusterVersion") or fields.get("newClusterSparkVersion") + if new_cluster: + config["spark_version"] = new_cluster + num_workers_raw = fields.get("newClusterNumOfWorker", 1) + try: + config["num_workers"] = int(num_workers_raw) + except (TypeError, ValueError): + config["num_workers"] = num_workers_raw + config["node_type_id"] = fields.get("newClusterNodeType", "Standard_DS3_v2") + spark_conf = fields.get("newClusterSparkConf") + if spark_conf: + config["spark_conf"] = spark_conf + + return config if config else None + + +def _pipeline_to_dict(pipeline: Pipeline) -> dict[str, Any]: + """Serialise a Pipeline IR to a JSON-friendly dictionary. + + Args: + pipeline: The translated pipeline IR. + + Returns: + Dictionary suitable for ``json.dumps``. + """ + return { + "name": pipeline.name, + "parameters": pipeline.parameters, + "schedule": pipeline.schedule, + "tags": pipeline.tags, + "tasks": [_activity_to_dict(task) for task in pipeline.tasks], + } + + +def _activity_to_dict(task: Activity) -> dict[str, Any]: + """Serialise a single Activity IR node to a JSON-friendly dictionary. + + Args: + task: Any Activity IR node. + + Returns: + Dictionary suitable for ``json.dumps``. + """ + task_dict: dict[str, Any] = { + "name": task.name, + "task_key": task.task_key, + "type": type(task).__name__, + } + if task.description: + task_dict["description"] = task.description + if task.timeout_seconds: + task_dict["timeout_seconds"] = task.timeout_seconds + if task.max_retries: + task_dict["max_retries"] = task.max_retries + if task.min_retry_interval_millis: + task_dict["min_retry_interval_millis"] = task.min_retry_interval_millis + if task.depends_on: + task_dict["depends_on"] = [ + {"task_key": dependency.task_key, "outcome": dependency.outcome} for dependency in task.depends_on + ] + if task.cluster: + task_dict["cluster"] = task.cluster + + extra = _activity_extra_fields(task) + task_dict.update(extra) + return task_dict + + +def _activity_extra_fields(activity: Activity) -> dict[str, Any]: + """Extracts type-specific fields from an Activity subclass. + + Args: + activity: Any Activity IR node. + + Returns: + Dictionary of extra fields beyond the base Activity. + """ + extra: dict[str, Any] = {} + + match activity: + case NotebookActivity(): + extra["notebook_path"] = activity.notebook_path + if activity.base_parameters: + extra["base_parameters"] = activity.base_parameters + case CopyActivity(): + extra["source_type"] = activity.source_type + extra["sink_type"] = activity.sink_type + if activity.source_properties: + extra["source_properties"] = activity.source_properties + if activity.sink_properties: + extra["sink_properties"] = activity.sink_properties + if activity.sink_dataset_type: + extra["sink_dataset_type"] = activity.sink_dataset_type + if activity.sink_format: + extra["sink_format"] = activity.sink_format + if activity.sink_resolved_path: + extra["sink_resolved_path"] = activity.sink_resolved_path + if activity.column_mapping: + extra["column_mapping"] = activity.column_mapping + case ForEachActivity(): + extra["items_expression"] = activity.items_expression + extra["concurrency"] = activity.concurrency + extra["inner_activities"] = [_activity_to_dict(inner) for inner in activity.inner_activities] + case IfConditionActivity(): + extra["op"] = activity.op + extra["left"] = activity.left + extra["right"] = activity.right + extra["if_true_activities"] = [_activity_to_dict(inner) for inner in activity.if_true_activities] + extra["if_false_activities"] = [_activity_to_dict(inner) for inner in activity.if_false_activities] + case LookupActivity(): + extra["source_type"] = activity.source_type + if activity.source_properties: + extra["source_properties"] = activity.source_properties + extra["first_row_only"] = activity.first_row_only + if activity.source_query: + extra["source_query"] = activity.source_query + case SetVariableActivity(): + extra["variable_name"] = activity.variable_name + extra["variable_value"] = activity.variable_value + extra["value_kind"] = activity.value_kind + if activity.notebook_code: + extra["notebook_code"] = activity.notebook_code + if activity.notebook_imports: + extra["notebook_imports"] = activity.notebook_imports + if activity.required_parameters: + extra["required_parameters"] = dict(activity.required_parameters) + case FilterActivity(): + extra["items_expression"] = activity.items_expression + extra["condition_expression"] = activity.condition_expression + if activity.condition_code is not None: + extra["condition_code"] = activity.condition_code + if activity.condition_imports: + extra["condition_imports"] = list(activity.condition_imports) + case AppendVariableActivity(): + extra["variable_name"] = activity.variable_name + extra["append_value"] = activity.append_value + extra["value_kind"] = activity.value_kind + if activity.notebook_code: + extra["notebook_code"] = activity.notebook_code + if activity.notebook_imports: + extra["notebook_imports"] = activity.notebook_imports + if activity.required_parameters: + extra["required_parameters"] = dict(activity.required_parameters) + case SwitchActivity(): + extra["on_expression"] = activity.on_expression + extra["cases"] = [ + {"value": case_item.value, "activities": [_activity_to_dict(inner) for inner in case_item.activities]} + for case_item in activity.cases + ] + extra["default_activities"] = [_activity_to_dict(inner) for inner in activity.default_activities] + case WaitActivity(): + extra["wait_time_seconds"] = activity.wait_time_seconds + case SparkJarActivity(): + extra["main_class_name"] = activity.main_class_name + if activity.parameters: + extra["parameters"] = activity.parameters + if activity.libraries: + extra["libraries"] = activity.libraries + case SparkPythonActivity(): + extra["python_file"] = activity.python_file + if activity.parameters: + extra["parameters"] = activity.parameters + case WebActivity(): + extra["url"] = activity.url + extra["method"] = activity.method + if activity.body is not None: + extra["body"] = activity.body + if activity.headers: + extra["headers"] = activity.headers + if activity.authentication: + extra["authentication"] = activity.authentication + case DeleteActivity(): + extra["dataset_name"] = activity.dataset_name + if activity.folder_path: + extra["folder_path"] = activity.folder_path + extra["recursive"] = activity.recursive + case ExecutePipelineActivity(): + extra["pipeline_name"] = activity.pipeline_name + extra["wait_on_completion"] = activity.wait_on_completion + if activity.parameters: + extra["parameters"] = activity.parameters + case RunJobActivity(): + extra["job_name"] = activity.job_name + if activity.existing_job_id: + extra["existing_job_id"] = activity.existing_job_id + if activity.job_parameters: + extra["job_parameters"] = activity.job_parameters + case MotifActivity(): + extra["motif_id"] = activity.motif_id + extra["display_name"] = activity.display_name + extra["databricks_replacement"] = activity.databricks_replacement + extra["matched_activity_names"] = activity.matched_activity_names + if activity.source_type_hint: + extra["source_type_hint"] = activity.source_type_hint + if activity.confidence_notes: + extra["confidence_notes"] = activity.confidence_notes + if activity.notebook_template: + extra["notebook_template"] = activity.notebook_template + if activity.motif_config: + extra["motif_config"] = activity.motif_config + case PlaceholderActivity(): + extra["original_type"] = activity.original_type + extra["comment"] = activity.comment + case UnsupportedActivity(): + extra["original_type"] = activity.original_type + extra["reason"] = activity.reason + + return extra + + +def _activity_to_debug_dict(activity: Activity) -> dict[str, Any]: + """Serialise an Activity to a full debug dict showing all dataclass fields. + + Args: + activity: Any Activity IR node. + + Returns: + Dict with ``__class__`` plus every dataclass field. + """ + result: dict[str, Any] = {"__class__": type(activity).__name__} + + for field in activity.__dataclass_fields__: + value = getattr(activity, field) + + if isinstance(value, Activity): + result[field] = _activity_to_debug_dict(value) + elif isinstance(value, list) and value and isinstance(value[0], Activity): + result[field] = [_activity_to_debug_dict(inner) for inner in value] + elif isinstance(value, list) and value and hasattr(value[0], "__dataclass_fields__"): + result[field] = [_dataclass_to_debug_dict(item) for item in value] + else: + result[field] = value + + return result + + +def _dataclass_to_debug_dict(obj: Any) -> dict[str, Any]: + """Serialise a generic dataclass (SwitchCase, Dependency, etc.) to a debug dict. + + Args: + obj: A dataclass instance. + + Returns: + Dict with ``__class__`` plus every dataclass field. + """ + result: dict[str, Any] = {"__class__": type(obj).__name__} + + for field in obj.__dataclass_fields__: + value = getattr(obj, field) + + if isinstance(value, Activity): + result[field] = _activity_to_debug_dict(value) + elif isinstance(value, list) and value and isinstance(value[0], Activity): + result[field] = [_activity_to_debug_dict(inner) for inner in value] + else: + result[field] = value + + return result + + +def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: + """Serialise a Pipeline IR to a full debug dict. + + Args: + pipeline: The translated pipeline IR. + + Returns: + Dict with every field fully expanded. + """ + return { + "__class__": "Pipeline", + "name": pipeline.name, + "parameters": pipeline.parameters, + "schedule": pipeline.schedule, + "tags": pipeline.tags, + "tasks": [_activity_to_debug_dict(task) for task in pipeline.tasks], + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Translate ADF pipelines to Databricks IR.") + parser.add_argument("--source-dir", required=True, type=Path, help="Root directory containing ADF JSON exports.") + parser.add_argument( + "--output-dir", + type=Path, + default=Path("./orchestra_output/translate"), + help="Directory to write translation results into.", + ) + parser.add_argument( + "--pipeline", + type=str, + default=None, + help="Translate only the named pipeline (default: all).", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Write a full debug IR dump alongside the normal output.", + ) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + definitions = load_adf_definitions(args.source_dir) + logger.info("Loaded %d pipeline(s) from %s", len(definitions.pipelines), args.source_dir) + + output_dir: Path = args.output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + total_deterministic = 0 + total_agentic = 0 + total_unsupported = 0 + all_gaps: list[dict[str, Any]] = [] + + for pipeline in definitions.pipelines: + if args.pipeline and pipeline.name != args.pipeline: + continue + + report = translate_pipeline(pipeline, definitions) + total_deterministic += report.deterministic_count + total_agentic += report.agentic_count + total_unsupported += report.unsupported_count + + pipeline_file = output_dir / f"{_sanitize_task_key(pipeline.name)}.json" + pipeline_dict = _pipeline_to_dict(report.pipeline) + pipeline_file.write_text(json.dumps(pipeline_dict, indent=2, default=str), encoding="utf-8") + logger.info("Wrote pipeline IR to %s", pipeline_file) + + # Write debug IR if requested + if args.debug: + debug_file = output_dir / f"{_sanitize_task_key(pipeline.name)}.debug.json" + debug_dict = _pipeline_to_debug_dict(report.pipeline) + debug_file.write_text(json.dumps(debug_dict, indent=2, default=str), encoding="utf-8") + logger.info("Wrote debug IR to %s", debug_file) + + for gap in report.gaps: + all_gaps.append(asdict(gap)) + + if report.warnings: + for warning in report.warnings: + logger.warning(warning) + + if all_gaps: + gaps_file = output_dir / "gaps.json" + gaps_file.write_text(json.dumps(all_gaps, indent=2, default=str), encoding="utf-8") + logger.info("Wrote %d gap(s) to %s", len(all_gaps), gaps_file) + + total = total_deterministic + total_agentic + total_unsupported + print("\nTranslation Summary") + print("===================") + print(f"Deterministic: {total_deterministic}") + print(f"Agentic: {total_agentic}") + print(f"Unsupported: {total_unsupported}") + print(f"Total: {total}") diff --git a/src/orchestra/utils.py b/src/orchestra/utils.py new file mode 100644 index 0000000..fbbd882 --- /dev/null +++ b/src/orchestra/utils.py @@ -0,0 +1,129 @@ +"""Shared utilities for the flowx translation pipeline.""" + +from __future__ import annotations + +import re +from typing import Any + +from flowx.models.adf_ast import AdfPolicy + +# --------------------------------------------------------------------------- +# Default ADF timeout (12 hours) used when a timeout string cannot be parsed. +# --------------------------------------------------------------------------- +DEFAULT_TIMEOUT_SECONDS = 43_200 + +# --------------------------------------------------------------------------- +# Case conversion +# --------------------------------------------------------------------------- + +_CAMEL_RE_1 = re.compile(r"(.)([A-Z][a-z]+)") +_CAMEL_RE_2 = re.compile(r"([a-z0-9])([A-Z])") + + +def camel_to_snake(name: str) -> str: + """Converts a camelCase or PascalCase string to snake_case. + + Args: + name: Identifier in camelCase or PascalCase. + + Returns: + Same identifier in snake_case. + """ + substituted = _CAMEL_RE_1.sub(r"\1_\2", name) + return _CAMEL_RE_2.sub(r"\1_\2", substituted).lower() + + +def recursive_camel_to_snake(obj: Any) -> Any: + """Recursively convert all dict keys from camelCase to snake_case. + + Args: + obj: Nested structure of dicts, lists, and primitives (e.g. ADF JSON). + + Returns: + New structure with dict keys in snake_case. + """ + if isinstance(obj, dict): + return {camel_to_snake(k): recursive_camel_to_snake(v) for k, v in obj.items()} + if isinstance(obj, list): + return [recursive_camel_to_snake(item) for item in obj] + return obj + + +# --------------------------------------------------------------------------- +# Task-key normalisation +# --------------------------------------------------------------------------- + +_TASK_KEY_RE = re.compile(r"[^a-z0-9_]") + + +def normalize_task_key(name: str) -> str: + """Sanitises a display name for use as a Databricks task key. + + Args: + name: Original activity or pipeline name. + + Returns: + Cleaned task key string. + """ + lowered = name.strip().lower() + replaced = _TASK_KEY_RE.sub("_", lowered) + collapsed = re.sub(r"_+", "_", replaced).strip("_") + return collapsed + + +# --------------------------------------------------------------------------- +# Timeout parsing +# --------------------------------------------------------------------------- + +_TIMEOUT_PATTERN = re.compile(r"^(?:(\d+)\.)?((\d{1,2}):(\d{2}):(\d{2}))$") + + +def parse_timeout(timeout_str: str | None) -> int | None: + """Parses an ADF timeout string into total seconds. + + Args: + timeout_str: Timeout string from the ADF activity policy, or ``None``. + + Returns: + Total seconds, ``DEFAULT_TIMEOUT_SECONDS`` on parse failure, or ``None`` + when no timeout is specified. + """ + if timeout_str is None: + return None + + match = _TIMEOUT_PATTERN.match(timeout_str) + if not match: + return DEFAULT_TIMEOUT_SECONDS + + days = int(match.group(1)) if match.group(1) is not None else 0 + hours = int(match.group(3)) + minutes = int(match.group(4)) + seconds = int(match.group(5)) + + total = days * 86_400 + hours * 3_600 + minutes * 60 + seconds + if total <= 0: + return DEFAULT_TIMEOUT_SECONDS + return total + + +# --------------------------------------------------------------------------- +# Retry policy extraction +# --------------------------------------------------------------------------- + + +def parse_retry_policy(policy: AdfPolicy | None) -> tuple[int | None, int | None]: + """Extracts retry count and interval from an ADF policy. + + Args: + policy: Parsed ``AdfPolicy``, or ``None``. + + Returns: + Tuple of ``(max_retries, retry_interval_seconds)``. Either or both + values may be ``None`` when the policy does not specify them. + """ + if policy is None: + return None, None + + retries = policy.retry + interval = policy.retry_interval_in_seconds + return retries, interval diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..3be5be4 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,22 @@ +import sys +from pathlib import Path + +import pytest + +# Add src to path +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +FIXTURES_DIR = Path(__file__).parent / "resources" / "json" + + +@pytest.fixture +def fixtures_dir(): + return FIXTURES_DIR + + +@pytest.fixture +def adf_definitions(): + """Load all ADF definitions from the test fixtures directory.""" + from flowx.parser.adf_loader import load_adf_definitions + + return load_adf_definitions(FIXTURES_DIR) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..388dc4b --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,28 @@ +# Integration test configuration — shared fixtures are in tests/conftest.py. +import pytest + + +@pytest.fixture +def pipeline_by_name(adf_definitions): + """Return a lookup helper that finds a pipeline by name.""" + + def _find(name: str): + for p in adf_definitions.pipelines: + if p.name == name: + return p + raise KeyError(f"Pipeline '{name}' not found in fixtures") + + return _find + + +@pytest.fixture +def live_pipeline_by_name(live_definitions): + """Return a lookup helper that finds a live ADF pipeline by name.""" + + def _find(name: str): + for p in live_definitions.pipelines: + if p.name == name: + return p + raise KeyError(f"Pipeline '{name}' not found in live ADF export") + + return _find diff --git a/tests/integration/test_adf_live.py b/tests/integration/test_adf_live.py new file mode 100644 index 0000000..107d73b --- /dev/null +++ b/tests/integration/test_adf_live.py @@ -0,0 +1,377 @@ +"""Integration tests against a live Azure Data Factory. + +These tests export pipeline definitions from a real ADF factory, run +them through the flowx translation pipeline, and validate that +the output is correct. + +Requires: +- Azure CLI authenticated (``az login``) +- Access to subscription 00000000-0000-0000-0000-000000000000 +""" + +from __future__ import annotations + +import ast +import json +import re +import subprocess + +import pytest +import yaml + +from flowx.bundler.dab_writer import write_bundle +from flowx.preparer.workflow_preparer import PreparedWorkflow, prepare_workflow +from flowx.translator.engine import translate_pipeline + +SUBSCRIPTION = "00000000-0000-0000-0000-000000000000" +RESOURCE_GROUP = "flowx-rg" +FACTORY_NAME = "flowx-adf" +API_VERSION = "2018-06-01" + + +def _az_authenticated() -> bool: + """Check if Azure CLI is authenticated.""" + try: + result = subprocess.run( + ["az", "account", "show", "--query", "id", "-o", "tsv"], + capture_output=True, + text=True, + timeout=10, + ) + return result.returncode == 0 + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + not _az_authenticated(), + reason="Azure CLI not authenticated -- run 'az login' first", + ), +] + + +# --------------------------------------------------------------------------- +# ADF REST helpers +# --------------------------------------------------------------------------- + + +def _export_resource(resource_type: str, name: str) -> dict: + """Export a single ADF resource via Azure REST API.""" + url = ( + f"https://management.azure.com/subscriptions/{SUBSCRIPTION}" + f"/resourceGroups/{RESOURCE_GROUP}/providers/Microsoft.DataFactory" + f"/factories/{FACTORY_NAME}/{resource_type}/{name}" + f"?api-version={API_VERSION}" + ) + result = subprocess.run( + ["az", "rest", "--method", "get", "--url", url, "-o", "json"], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, f"Failed to export {resource_type}/{name}: {result.stderr}" + return json.loads(result.stdout) + + +def _list_resources(resource_type: str) -> list[str]: + """List all ADF resources of a given type.""" + url = ( + f"https://management.azure.com/subscriptions/{SUBSCRIPTION}" + f"/resourceGroups/{RESOURCE_GROUP}/providers/Microsoft.DataFactory" + f"/factories/{FACTORY_NAME}/{resource_type}" + f"?api-version={API_VERSION}" + ) + result = subprocess.run( + ["az", "rest", "--method", "get", "--url", url, "-o", "json"], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, f"Failed to list {resource_type}: {result.stderr}" + data = json.loads(result.stdout) + return [item["name"] for item in data.get("value", [])] + + +# --------------------------------------------------------------------------- +# Module-scoped fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def adf_export_dir(tmp_path_factory): + """Export all ADF definitions to a temp directory.""" + export_dir = tmp_path_factory.mktemp("adf_live_export") + + # Export pipelines + pipelines_dir = export_dir / "pipelines" + pipelines_dir.mkdir() + for name in _list_resources("pipelines"): + data = _export_resource("pipelines", name) + (pipelines_dir / f"{name}.json").write_text(json.dumps(data, indent=2)) + + # Export datasets + datasets_dir = export_dir / "datasets" + datasets_dir.mkdir() + for name in _list_resources("datasets"): + data = _export_resource("datasets", name) + (datasets_dir / f"{name}.json").write_text(json.dumps(data, indent=2)) + + # Export linked services + ls_dir = export_dir / "linked_services" + ls_dir.mkdir() + for name in _list_resources("linkedservices"): + data = _export_resource("linkedservices", name) + (ls_dir / f"{name}.json").write_text(json.dumps(data, indent=2)) + + # Export triggers + triggers_dir = export_dir / "triggers" + triggers_dir.mkdir() + for name in _list_resources("triggers"): + data = _export_resource("triggers", name) + (triggers_dir / f"{name}.json").write_text(json.dumps(data, indent=2)) + + return export_dir + + +@pytest.fixture(scope="module") +def live_definitions(adf_export_dir): + """Load all exported ADF definitions.""" + from flowx.parser.adf_loader import load_adf_definitions + + return load_adf_definitions(adf_export_dir) + + +# --------------------------------------------------------------------------- +# TestAdfExport -- validate that we can export and parse all ADF definitions +# --------------------------------------------------------------------------- + + +class TestAdfExport: + """Validate that we can export and parse all ADF definitions.""" + + def test_export_loads_all_pipelines(self, live_definitions): + """All 18 pipelines are loaded.""" + assert len(live_definitions.pipelines) >= 18 + + def test_export_loads_datasets(self, live_definitions): + """All 9 datasets are loaded.""" + assert len(live_definitions.datasets) >= 9 + + def test_export_loads_linked_services(self, live_definitions): + """All 5 linked services are loaded.""" + assert len(live_definitions.linked_services) >= 5 + + def test_export_loads_triggers(self, live_definitions): + """All 2 triggers are loaded.""" + assert len(live_definitions.triggers) >= 2 + + +# --------------------------------------------------------------------------- +# TestAdfTranslation -- validate translation of all live ADF pipelines +# --------------------------------------------------------------------------- + + +class TestAdfTranslation: + """Validate translation of all live ADF pipelines.""" + + def test_translate_all_pipelines(self, live_definitions): + """All pipelines translate without errors.""" + for pl in live_definitions.pipelines: + report = translate_pipeline(pl, live_definitions) + assert report.pipeline is not None, f"Pipeline {pl.name} produced no IR" + total = report.deterministic_count + report.agentic_count + report.unsupported_count + assert total > 0, f"Pipeline {pl.name} has no translated activities" + + def test_prepare_all_pipelines(self, live_definitions): + """All translated pipelines produce valid PreparedWorkflows.""" + for pl in live_definitions.pipelines: + report = translate_pipeline(pl, live_definitions) + wf = prepare_workflow(report.pipeline) + assert isinstance(wf, PreparedWorkflow), f"Pipeline {pl.name} did not produce PreparedWorkflow" + assert len(wf.tasks) > 0, f"Pipeline {pl.name} produced no tasks" + for task in wf.tasks: + assert "task_key" in task, f"Task missing task_key in pipeline {pl.name}" + assert task["task_key"], f"Empty task_key in pipeline {pl.name}" + + def test_no_python_code_in_parameters(self, live_definitions): + """No base_parameters contain executable Python code.""" + forbidden = ["__import__", "dbutils.jobs.taskValues.get", "eval(", "exec("] + + for pl in live_definitions.pipelines: + report = translate_pipeline(pl, live_definitions) + wf = prepare_workflow(report.pipeline) + for task in wf.tasks: + params = task.get("notebook_task", {}).get("base_parameters", {}) + for key, val in params.items(): + val_str = str(val) + for pattern in forbidden: + assert pattern not in val_str, ( + f"Pipeline {pl.name}, task {task.get('task_key')}, " + f"param {key} contains forbidden pattern '{pattern}': {val_str}" + ) + + def test_dab_refs_are_well_formed(self, live_definitions): + """All DAB dynamic value references use valid syntax.""" + dab_ref_re = re.compile(r"\{\{[a-zA-Z0-9_.]+\}\}") + + for pl in live_definitions.pipelines: + report = translate_pipeline(pl, live_definitions) + wf = prepare_workflow(report.pipeline) + for task in wf.tasks: + params = task.get("notebook_task", {}).get("base_parameters", {}) + for key, val in params.items(): + val_str = str(val) + if "{{" in val_str: + assert dab_ref_re.match(val_str), ( + f"Malformed DAB ref in {pl.name}/{task.get('task_key')}/{key}: {val_str}" + ) + + +# --------------------------------------------------------------------------- +# TestAdfBundleOutput -- validate DAB bundle output from live ADF pipelines +# --------------------------------------------------------------------------- + + +class TestAdfBundleOutput: + """Validate DAB bundle output from live ADF pipelines.""" + + def test_generate_bundles_for_all_pipelines(self, live_definitions, tmp_path): + """All pipelines generate valid DAB bundles.""" + for pl in live_definitions.pipelines: + report = translate_pipeline(pl, live_definitions) + wf = prepare_workflow(report.pipeline) + bundle_dir = tmp_path / pl.name + write_bundle(wf, bundle_dir, catalog="main", schema="bronze") + + # databricks.yml must exist + assert (bundle_dir / "databricks.yml").exists(), f"Missing databricks.yml for {pl.name}" + + # At least one resource YAML + resource_files = list((bundle_dir / "resources").glob("*.yml")) + assert len(resource_files) > 0, f"No resource YAML for {pl.name}" + + def test_generated_notebooks_are_valid_python(self, live_definitions, tmp_path): + """All generated notebooks parse as valid Python.""" + for pl in live_definitions.pipelines: + report = translate_pipeline(pl, live_definitions) + wf = prepare_workflow(report.pipeline) + bundle_dir = tmp_path / f"{pl.name}_nb" + write_bundle(wf, bundle_dir, catalog="main", schema="bronze") + + for nb_file in bundle_dir.rglob("*.py"): + content = nb_file.read_text() + lines = [] + for line in content.split("\n"): + stripped = line.lstrip() + if stripped.startswith("# MAGIC"): + continue + if stripped.startswith("# COMMAND"): + continue + if stripped == "# Databricks notebook source": + continue + lines.append(line) + try: + ast.parse("\n".join(lines)) + except SyntaxError as e: + pytest.fail(f"Notebook {nb_file} in pipeline '{pl.name}' has syntax error: {e}") + + def test_yaml_is_parseable(self, live_definitions, tmp_path): + """All generated YAML files are valid.""" + for pl in live_definitions.pipelines: + report = translate_pipeline(pl, live_definitions) + wf = prepare_workflow(report.pipeline) + bundle_dir = tmp_path / f"{pl.name}_yaml" + write_bundle(wf, bundle_dir, catalog="main", schema="bronze") + + for yml_file in bundle_dir.rglob("*.yml"): + content = yml_file.read_text() + try: + yaml.safe_load(content) + except yaml.YAMLError as e: + pytest.fail(f"YAML parse error in {yml_file} for pipeline '{pl.name}': {e}") + + +# --------------------------------------------------------------------------- +# TestSpecificPipelines -- targeted tests for specific ADF pipeline patterns +# --------------------------------------------------------------------------- + + +class TestSpecificPipelines: + """Targeted tests for specific ADF pipeline patterns.""" + + def test_copy_csv_resolves_source_path(self, live_definitions): + """Copy CSV pipeline resolves source path to a volume path.""" + pl = next((p for p in live_definitions.pipelines if p.name == "pl_copy_csv_to_delta"), None) + if pl is None: + pytest.skip("pl_copy_csv_to_delta not found") + report = translate_pipeline(pl, live_definitions) + wf = prepare_workflow(report.pipeline) + + params = wf.tasks[0].get("notebook_task", {}).get("base_parameters", {}) + source_path = params.get("source_path", "") + assert "/Volumes/" in source_path, f"Expected volume path, got: {source_path}" + + def test_foreach_uses_task_value_inputs(self, live_definitions): + """ForEach pipeline uses task value references for inputs.""" + pl = next((p for p in live_definitions.pipelines if p.name == "pl_foreach_copy_tables"), None) + if pl is None: + pytest.skip("pl_foreach_copy_tables not found") + report = translate_pipeline(pl, live_definitions) + wf = prepare_workflow(report.pipeline) + + foreach_task = next((t for t in wf.tasks if "for_each_task" in t), None) + assert foreach_task is not None, "No for_each_task found in workflow" + inputs = foreach_task["for_each_task"]["inputs"] + assert "{{tasks." in inputs and ".values." in inputs, f"Expected task value ref in inputs: {inputs}" + + def test_if_condition_uses_structured_op(self, live_definitions): + """IfCondition pipeline uses structured op/left/right.""" + pl = next((p for p in live_definitions.pipelines if p.name == "pl_if_condition_branch"), None) + if pl is None: + pytest.skip("pl_if_condition_branch not found") + report = translate_pipeline(pl, live_definitions) + wf = prepare_workflow(report.pipeline) + + cond_task = next((t for t in wf.tasks if "condition_task" in t), None) + assert cond_task is not None, "No condition_task found in workflow" + ct = cond_task["condition_task"] + assert "op" in ct, f"condition_task missing 'op': {ct}" + assert "left" in ct, f"condition_task missing 'left': {ct}" + assert "right" in ct, f"condition_task missing 'right': {ct}" + valid_ops = ( + "EQUAL_TO", + "NOT_EQUAL", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + ) + assert ct["op"] in valid_ops, f"Unexpected op '{ct['op']}', expected one of {valid_ops}" + + def test_switch_generates_condition_chain(self, live_definitions): + """Switch pipeline generates condition task chain.""" + pl = next((p for p in live_definitions.pipelines if p.name == "pl_switch_environment"), None) + if pl is None: + pytest.skip("pl_switch_environment not found") + report = translate_pipeline(pl, live_definitions) + wf = prepare_workflow(report.pipeline) + assert len(wf.tasks) >= 1, "Switch pipeline produced no tasks" + + def test_set_variable_no_code_injection(self, live_definitions): + """SetVariable pipeline has no Python code in parameters.""" + pl = next((p for p in live_definitions.pipelines if p.name == "pl_set_variable_chain"), None) + if pl is None: + pytest.skip("pl_set_variable_chain not found") + report = translate_pipeline(pl, live_definitions) + wf = prepare_workflow(report.pipeline) + + for task in wf.tasks: + params = task.get("notebook_task", {}).get("base_parameters", {}) + for key, val in params.items(): + val_str = str(val) + assert "__import__" not in val_str, f"Code injection in {task['task_key']}/{key}: {val_str}" + assert "dbutils.jobs.taskValues.get" not in val_str, ( + f"Code in param {task['task_key']}/{key}: {val_str}" + ) diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py new file mode 100644 index 0000000..f7744fb --- /dev/null +++ b/tests/integration/test_end_to_end.py @@ -0,0 +1,363 @@ +"""End-to-end integration tests for the flowx translation pipeline. + +These tests exercise the full ingest -> translate -> prepare -> bundle pipeline +against realistic ADF fixture files, simulating what happens when a user +invokes the flowx skills. +""" + +from __future__ import annotations + +import ast + +import pytest +import yaml + +from flowx.bundler.dab_writer import write_bundle +from flowx.models.adf_ast import AdfDefinitions, TranslationStrategy +from flowx.models.ir import ( + CopyActivity, + ForEachActivity, + PlaceholderActivity, + SwitchActivity, +) +from flowx.parser.adf_loader import build_inventory +from flowx.preparer.workflow_preparer import PreparedWorkflow, prepare_workflow +from flowx.translator.engine import translate_pipeline + +# --------------------------------------------------------------------------- +# TestTranslateAllPipelines — simulates "translate all pipelines" +# --------------------------------------------------------------------------- + + +class TestTranslateAllPipelines: + """Tests simulating 'translate all pipelines' prompt.""" + + def test_ingest_all_pipelines(self, adf_definitions): + """All pipelines load successfully from the fixture directory.""" + assert isinstance(adf_definitions, AdfDefinitions) + assert len(adf_definitions.pipelines) >= 4 + + def test_translate_all_pipelines(self, adf_definitions): + """All pipelines translate without errors.""" + for pipeline in adf_definitions.pipelines: + report = translate_pipeline(pipeline, adf_definitions) + assert report.pipeline is not None + assert report.pipeline.name == pipeline.name + total = report.deterministic_count + report.agentic_count + report.unsupported_count + assert total > 0, f"Pipeline {pipeline.name} has no translated activities" + + def test_prepare_all_pipelines(self, adf_definitions): + """All translated pipelines produce valid PreparedWorkflows.""" + for pipeline in adf_definitions.pipelines: + report = translate_pipeline(pipeline, adf_definitions) + wf = prepare_workflow(report.pipeline) + assert isinstance(wf, PreparedWorkflow) + assert len(wf.tasks) > 0, f"Pipeline {pipeline.name} produced no tasks" + # Every task must have a task_key + for task in wf.tasks: + assert "task_key" in task, f"Task missing task_key in pipeline {pipeline.name}" + assert task["task_key"], f"Empty task_key in pipeline {pipeline.name}" + + def test_bundle_all_pipelines(self, adf_definitions, tmp_path): + """All pipelines produce valid DAB output files.""" + for i, pipeline in enumerate(adf_definitions.pipelines): + report = translate_pipeline(pipeline, adf_definitions) + wf = prepare_workflow(report.pipeline) + output_dir = tmp_path / f"bundle_{i}" + created = write_bundle(wf, output_dir) + assert len(created) > 0 + + # Verify databricks.yml exists and is valid YAML + dby = output_dir / "databricks.yml" + assert dby.exists(), f"databricks.yml missing for {pipeline.name}" + parsed = yaml.safe_load(dby.read_text()) + assert "bundle" in parsed + + # Verify resources/*.yml exist and are valid + resource_files = list((output_dir / "resources").glob("*.yml")) + assert len(resource_files) >= 1, f"No resource YAML for {pipeline.name}" + for rf in resource_files: + content = yaml.safe_load(rf.read_text()) + assert content is not None + + +# --------------------------------------------------------------------------- +# TestTranslateSpecificPipeline — simulates "translate a specific pipeline" +# --------------------------------------------------------------------------- + + +class TestTranslateSpecificPipeline: + """Tests simulating 'translate a specific pipeline' prompt.""" + + def test_translate_copy_csv_pipeline(self, adf_definitions, pipeline_by_name): + """Copy CSV to Delta pipeline translates correctly.""" + pipeline = pipeline_by_name("pipeline_copy_csv_to_delta") + report = translate_pipeline(pipeline, adf_definitions) + assert report.deterministic_count == 1 + assert report.agentic_count == 0 + assert isinstance(report.pipeline.tasks[0], CopyActivity) + copy_task = report.pipeline.tasks[0] + assert copy_task.source_type == "DelimitedTextSource" + assert copy_task.sink_type == "DeltaSink" + assert copy_task.column_mapping is not None + assert len(copy_task.column_mapping) == 5 + + def test_translate_notebook_basic_pipeline(self, adf_definitions, pipeline_by_name): + """Basic notebook pipeline translates with notebook path preserved.""" + pipeline = pipeline_by_name("pipeline_notebook_basic") + report = translate_pipeline(pipeline, adf_definitions) + assert report.deterministic_count == 1 + from flowx.models.ir import NotebookActivity + + nb = report.pipeline.tasks[0] + assert isinstance(nb, NotebookActivity) + assert nb.notebook_path == "/Shared/ETL/transform_customers" + + def test_translate_copy_sql_pipeline(self, adf_definitions, pipeline_by_name): + """SQL to Delta pipeline preserves source query and column mappings.""" + pipeline = pipeline_by_name("pipeline_copy_sql_to_delta") + report = translate_pipeline(pipeline, adf_definitions) + assert report.deterministic_count == 1 + copy_task = report.pipeline.tasks[0] + assert isinstance(copy_task, CopyActivity) + assert copy_task.source_type == "AzureSqlSource" + assert copy_task.column_mapping is not None + assert len(copy_task.column_mapping) == 6 + + def test_translate_complex_etl_pipeline(self, adf_definitions, pipeline_by_name): + """Complex multi-activity ETL pipeline translates all activities.""" + pipeline = pipeline_by_name("pipeline_complex_etl") + report = translate_pipeline(pipeline, adf_definitions) + # Should have at least 5 deterministic activities + assert report.deterministic_count >= 5 + task_types = {type(t).__name__ for t in report.pipeline.tasks} + assert len(task_types) > 1 + + def test_translate_foreach_switch_pipeline(self, adf_definitions, pipeline_by_name): + """ForEach + Switch pipeline preserves control flow structure.""" + pipeline = pipeline_by_name("pipeline_foreach_switch") + report = translate_pipeline(pipeline, adf_definitions) + # Should have Lookup + ForEach + SetVariable at top level + task_names = {t.name for t in report.pipeline.tasks} + assert "Get Table List" in task_names + assert "Process Each Table" in task_names + assert "Set Completion Status" in task_names + + def test_translate_all_activity_types_pipeline(self, adf_definitions, pipeline_by_name): + """Pipeline with all 16 deterministic types translates fully.""" + pipeline = pipeline_by_name("pipeline_all_activity_types") + report = translate_pipeline(pipeline, adf_definitions) + # All 13 top-level activities should be deterministic + assert report.deterministic_count == 13 + assert report.agentic_count == 0 + assert report.unsupported_count == 0 + + def test_translate_mixed_agentic_pipeline(self, adf_definitions, pipeline_by_name): + """Mixed pipeline has both deterministic and agentic/unsupported items.""" + pipeline = pipeline_by_name("pipeline_mixed_agentic") + report = translate_pipeline(pipeline, adf_definitions) + assert report.deterministic_count >= 1 # Copy + assert (report.agentic_count + report.unsupported_count) >= 1 # ExecuteDataFlow, etc. + # Should have placeholders for agentic types + placeholders = [t for t in report.pipeline.tasks if isinstance(t, PlaceholderActivity)] + assert len(placeholders) >= 1 + + +# --------------------------------------------------------------------------- +# TestActivityTypeTranslation — specific activity translation accuracy +# --------------------------------------------------------------------------- + + +class TestActivityTypeTranslation: + """Tests for specific activity type translation accuracy.""" + + def test_copy_activity_source_sink(self, adf_definitions, pipeline_by_name): + """Copy activity preserves source/sink properties.""" + pipeline = pipeline_by_name("pipeline_copy_csv_to_delta") + report = translate_pipeline(pipeline, adf_definitions) + copy = report.pipeline.tasks[0] + assert isinstance(copy, CopyActivity) + assert copy.source_type is not None + assert copy.sink_type is not None + # Source properties should contain storeSettings, formatSettings, etc. + assert copy.source_properties is not None + + def test_foreach_items_expression(self, adf_definitions, pipeline_by_name): + """ForEach items expression is parsed correctly.""" + pipeline = pipeline_by_name("pipeline_foreach_switch") + report = translate_pipeline(pipeline, adf_definitions) + foreach_tasks = [t for t in report.pipeline.tasks if isinstance(t, ForEachActivity)] + assert len(foreach_tasks) == 1 + fe = foreach_tasks[0] + assert "{{tasks.Get_Table_List.values.result}}" == fe.items_expression + + def test_switch_cases_translation(self, adf_definitions, pipeline_by_name): + """Switch cases produce correct case branches.""" + pipeline = pipeline_by_name("pipeline_foreach_switch") + report = translate_pipeline(pipeline, adf_definitions) + foreach_tasks = [t for t in report.pipeline.tasks if isinstance(t, ForEachActivity)] + assert len(foreach_tasks) == 1 + switch_children = [a for a in foreach_tasks[0].inner_activities if isinstance(a, SwitchActivity)] + assert len(switch_children) == 1 + child = switch_children[0] + assert len(child.cases) == 2 + assert child.cases[0].value == "full" + assert child.cases[1].value == "incremental" + + def test_dependency_conditions_preserved(self, adf_definitions, pipeline_by_name): + """All dependency conditions (Succeeded, Failed, Completed) are preserved.""" + pipeline = pipeline_by_name("pipeline_all_activity_types") + report = translate_pipeline(pipeline, adf_definitions) + # Most tasks depend on the previous one via Succeeded + for task in report.pipeline.tasks: + if task.depends_on: + for dep in task.depends_on: + assert dep.outcome in ("Succeeded", "Failed", "Completed", "Skipped", None) + + +# --------------------------------------------------------------------------- +# TestBundleOutput — DAB bundle output validity +# --------------------------------------------------------------------------- + + +class TestBundleOutput: + """Tests for DAB bundle output validity.""" + + def test_databricks_yml_structure(self, adf_definitions, tmp_path): + """Generated databricks.yml has correct structure for every pipeline.""" + for i, pipeline in enumerate(adf_definitions.pipelines): + report = translate_pipeline(pipeline, adf_definitions) + wf = prepare_workflow(report.pipeline) + out = tmp_path / f"bundle_{i}" + write_bundle(wf, out, catalog="prod", schema="analytics") + content = yaml.safe_load((out / "databricks.yml").read_text()) + assert content["variables"]["catalog"]["default"] == "prod" + assert content["variables"]["schema"]["default"] == "analytics" + assert "targets" in content + assert set(content["targets"].keys()) == {"dev", "staging", "prod"} + + def test_job_yaml_task_keys_match_activities(self, adf_definitions, pipeline_by_name, tmp_path): + """Job YAML has unique task keys matching the pipeline activities.""" + pipeline = pipeline_by_name("pipeline_all_activity_types") + report = translate_pipeline(pipeline, adf_definitions) + wf = prepare_workflow(report.pipeline) + write_bundle(wf, tmp_path) + + resource_file = list((tmp_path / "resources").glob("*.yml"))[0] + content = yaml.safe_load(resource_file.read_text()) + job = list(content["resources"]["jobs"].values())[0] + task_keys = [t["task_key"] for t in job["tasks"]] + + # All task keys should be unique + assert len(task_keys) == len(set(task_keys)) + # Should have as many tasks as activities + assert len(task_keys) == len(pipeline.activities) + + def test_notebooks_are_valid_python(self, adf_definitions, tmp_path): + """All generated notebooks are syntactically valid Python.""" + for i, pipeline in enumerate(adf_definitions.pipelines): + report = translate_pipeline(pipeline, adf_definitions) + wf = prepare_workflow(report.pipeline) + out = tmp_path / f"bundle_{i}" + write_bundle(wf, out) + + src_dir = out / "src" + if not src_dir.exists(): + continue + for nb_file in src_dir.rglob("*.py"): + content = nb_file.read_text() + # Strip Databricks magic comments + lines = [] + for line in content.split("\n"): + stripped = line.lstrip() + if stripped.startswith("# MAGIC") or stripped.startswith("# COMMAND"): + continue + if stripped == "# Databricks notebook source": + continue + lines.append(line) + python_code = "\n".join(lines) + try: + ast.parse(python_code) + except SyntaxError as exc: + pytest.fail( + f"Notebook {nb_file.relative_to(out)} in pipeline '{pipeline.name}' has invalid Python: {exc}" + ) + + def test_setup_notebooks_generated(self, adf_definitions, pipeline_by_name, tmp_path): + """Setup notebooks are created for activities requiring secrets/volumes.""" + # pipeline_copy_sql_to_delta uses AzureSqlSource which needs JDBC secrets + pipeline = pipeline_by_name("pipeline_copy_sql_to_delta") + report = translate_pipeline(pipeline, adf_definitions) + wf = prepare_workflow(report.pipeline) + write_bundle(wf, tmp_path) + + if wf.secrets: + setup_dir = tmp_path / "src" / "setup" + assert setup_dir.exists() + assert any(setup_dir.glob("*.py")) + + +# --------------------------------------------------------------------------- +# TestInventoryAccuracy — activity classification accuracy +# --------------------------------------------------------------------------- + + +class TestInventoryAccuracy: + """Tests for activity classification accuracy.""" + + def test_deterministic_count(self, adf_definitions): + """Inventory correctly counts deterministic activities.""" + inv = build_inventory(adf_definitions) + assert inv.deterministic_count > 0 + det_items = [i for i in inv.items if i.strategy is TranslationStrategy.DETERMINISTIC] + assert len(det_items) == inv.deterministic_count + + def test_agentic_activities_identified(self, adf_definitions): + """Agentic activities are identified with correct skill mapping.""" + inv = build_inventory(adf_definitions) + agentic_items = [i for i in inv.items if i.strategy is TranslationStrategy.AGENTIC] + assert len(agentic_items) == inv.agentic_count + for item in agentic_items: + assert item.agentic_skill is not None + # All agentic skills should reference a known skill + assert "adf-to-databricks" in item.agentic_skill + + def test_mixed_pipeline_classification(self, adf_definitions): + """Mixed pipeline has both deterministic and agentic items.""" + inv = build_inventory(adf_definitions) + # pipeline_mixed_agentic has Copy (deterministic) + ExecuteDataFlow (agentic) + more + mixed_items = [i for i in inv.items if i.pipeline_name == "pipeline_mixed_agentic"] + strategies = {i.strategy for i in mixed_items} + assert TranslationStrategy.DETERMINISTIC in strategies + assert TranslationStrategy.AGENTIC in strategies or TranslationStrategy.UNSUPPORTED in strategies + + def test_unsupported_type_counted(self, adf_definitions): + """Unknown activity types are counted as unsupported.""" + inv = build_inventory(adf_definitions) + # pipeline_mixed_agentic has SomeFutureActivity + unsupported = [i for i in inv.items if i.strategy is TranslationStrategy.UNSUPPORTED] + assert len(unsupported) == inv.unsupported_count + future_items = [i for i in unsupported if i.activity_type == "SomeFutureActivity"] + assert len(future_items) >= 1 + + def test_inventory_pipeline_count_matches(self, adf_definitions): + """Inventory pipeline count equals the number of loaded pipelines.""" + inv = build_inventory(adf_definitions) + assert inv.pipeline_count == len(adf_definitions.pipelines) + + def test_recursive_classification_includes_children(self, adf_definitions): + """Activities nested inside ForEach/IfCondition are also classified. + + Note: The inventory classifies activities recursively through + if_true_activities, if_false_activities, and activities (ForEach children), + but Switch case activities remain in typeProperties and are not recursively + classified at the inventory level (they are translated by the switch translator). + """ + inv = build_inventory(adf_definitions) + # pipeline_foreach_switch has activities inside ForEach > Switch + foreach_pipeline_items = [i for i in inv.items if i.pipeline_name == "pipeline_foreach_switch"] + activity_types = {i.activity_type for i in foreach_pipeline_items} + assert "ForEach" in activity_types + assert "Switch" in activity_types + # ForEach child activities are classified recursively + assert "Lookup" in activity_types or "SetVariable" in activity_types diff --git a/tests/integration/test_golden_output.py b/tests/integration/test_golden_output.py new file mode 100644 index 0000000..2c235e4 --- /dev/null +++ b/tests/integration/test_golden_output.py @@ -0,0 +1,524 @@ +"""Golden-file integration tests for the 15 pl_test_* coverage pipelines. + +These tests exercise the full load -> translate -> prepare -> write pipeline +against the 15 coverage test pipelines exported from ADF, asserting structural +properties of the output: correct task counts, unique task keys, clean +parameters, valid Python notebooks, and pipeline-specific patterns. +""" + +from __future__ import annotations + +import ast +import re + +import pytest +import yaml + +from flowx.bundler.dab_writer import write_bundle +from flowx.models.ir import ( + CopyActivity, + SetVariableActivity, +) +from flowx.parser.adf_loader import load_adf_definitions +from flowx.preparer.workflow_preparer import PreparedWorkflow, prepare_workflow +from flowx.translator.engine import translate_pipeline + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +FIXTURES_DIR = pytest.importorskip("pathlib").Path(__file__).parent.parent / "resources" / "json" + +_PL_TEST_NAMES = [ + "pl_test_copy_coverage", + "pl_test_notebook_coverage", + "pl_test_sparkjar_coverage", + "pl_test_sparkpython_coverage", + "pl_test_foreach_coverage", + "pl_test_ifcondition_coverage", + "pl_test_setvariable_coverage", + "pl_test_appendvariable_coverage", + "pl_test_switch_coverage", + "pl_test_lookup_coverage", + "pl_test_webactivity_coverage", + "pl_test_delete_coverage", + "pl_test_executepipeline_coverage", + "pl_test_wait_coverage", + "pl_test_filter_coverage", +] + +# Forbidden patterns that must NEVER appear in base_parameters values +FORBIDDEN_PATTERNS = [ + "__import__", + "dbutils.jobs.taskValues.get", + "eval(", + "exec(", +] + +# ADF expression patterns that should be resolved before reaching parameters +ADF_EXPRESSION_RE = re.compile(r"@(?:activity|pipeline|variables|item|concat|equals|greater|utcNow)\(") +DAB_REF_RE = re.compile(r"\{\{[a-zA-Z0-9_.]+\}\}") + + +@pytest.fixture(scope="module") +def all_definitions(): + """Load all ADF definitions from the test fixtures.""" + return load_adf_definitions(FIXTURES_DIR) + + +@pytest.fixture(scope="module") +def test_pipelines(all_definitions): + """Return only the pl_test_* coverage pipelines, keyed by name.""" + result = {} + for pl in all_definitions.pipelines: + if pl.name in _PL_TEST_NAMES: + result[pl.name] = pl + return result + + +@pytest.fixture(scope="module") +def translated_pipelines(test_pipelines, all_definitions): + """Translate all test pipelines and return (name -> TranslationReport).""" + return {name: translate_pipeline(pl, all_definitions) for name, pl in test_pipelines.items()} + + +@pytest.fixture(scope="module") +def prepared_workflows(translated_pipelines): + """Prepare all translated pipelines and return (name -> PreparedWorkflow).""" + return {name: prepare_workflow(rpt.pipeline) for name, rpt in translated_pipelines.items()} + + +@pytest.fixture(scope="module") +def bundle_dirs(prepared_workflows, tmp_path_factory): + """Write all bundles and return (name -> bundle_path).""" + result = {} + for name, wf in prepared_workflows.items(): + out = tmp_path_factory.mktemp(name) + write_bundle(wf, out, catalog="test_catalog", schema="test_schema") + result[name] = out + return result + + +def _strip_magic(content: str) -> str: + """Strip Databricks magic comments for Python syntax checking.""" + lines = [] + for line in content.split("\n"): + stripped = line.lstrip() + if stripped.startswith("# MAGIC") or stripped.startswith("# COMMAND"): + continue + if stripped == "# Databricks notebook source": + continue + lines.append(line) + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Structural tests across all 15 pipelines +# --------------------------------------------------------------------------- + + +class TestAllPipelinesLoad: + """Verify that all 15 test pipelines load from fixtures.""" + + def test_all_15_pipelines_found(self, test_pipelines): + missing = set(_PL_TEST_NAMES) - set(test_pipelines.keys()) + assert not missing, f"Missing test pipelines: {missing}" + + def test_all_15_pipelines_translate(self, translated_pipelines): + for name, report in translated_pipelines.items(): + assert report.pipeline is not None, f"{name}: translation produced no pipeline" + total = report.deterministic_count + report.agentic_count + report.unsupported_count + assert total > 0, f"{name}: no translated activities" + + def test_all_15_pipelines_prepare(self, prepared_workflows): + for name, wf in prepared_workflows.items(): + assert isinstance(wf, PreparedWorkflow), f"{name}: not a PreparedWorkflow" + assert len(wf.tasks) > 0, f"{name}: no tasks" + + +class TestBundleStructure: + """Verify bundle output structure for every test pipeline.""" + + def test_databricks_yml_exists(self, bundle_dirs): + for name, path in bundle_dirs.items(): + assert (path / "databricks.yml").exists(), f"{name}: missing databricks.yml" + + def test_databricks_yml_has_correct_bundle_name(self, bundle_dirs): + for name, path in bundle_dirs.items(): + content = yaml.safe_load((path / "databricks.yml").read_text()) + assert "bundle" in content + assert content["bundle"]["name"] is not None + + def test_job_resource_yml_exists(self, bundle_dirs): + for name, path in bundle_dirs.items(): + resources = list((path / "resources").glob("*.yml")) + assert len(resources) >= 1, f"{name}: no resource YAMLs" + + def test_task_keys_unique_within_each_job(self, bundle_dirs): + for name, path in bundle_dirs.items(): + for rf in (path / "resources").glob("*.yml"): + content = yaml.safe_load(rf.read_text()) + if not content or "resources" not in content: + continue + for job in content["resources"].get("jobs", {}).values(): + task_keys = _collect_all_task_keys(job.get("tasks", [])) + assert len(task_keys) == len(set(task_keys)), ( + f"{name}: duplicate task keys in {rf.name}: {task_keys}" + ) + + +class TestCleanParameters: + """Verify no forbidden patterns in base_parameters across all pipelines.""" + + def test_no_code_injection_in_parameters(self, prepared_workflows): + for name, wf in prepared_workflows.items(): + for task in wf.tasks: + params = task.get("notebook_task", {}).get("base_parameters", {}) + for key, val in params.items(): + val_str = str(val) + for pattern in FORBIDDEN_PATTERNS: + assert pattern not in val_str, ( + f"{name}/{task.get('task_key')}/{key}: forbidden pattern '{pattern}' found: {val_str}" + ) + + def test_no_raw_adf_expressions_in_parameters(self, prepared_workflows): + """No unresolved ADF @expression() calls in parameter values.""" + for name, wf in prepared_workflows.items(): + for task in wf.tasks: + params = task.get("notebook_task", {}).get("base_parameters", {}) + for key, val in params.items(): + val_str = str(val) + # Skip values that are deliberately ADF expressions for + # notebook interpretation (items_expression, condition_expression) + if key in ("items_expression", "condition_expression"): + continue + assert not ADF_EXPRESSION_RE.search(val_str), ( + f"{name}/{task.get('task_key')}/{key}: unresolved ADF expression found: {val_str}" + ) + + def test_dab_refs_well_formed(self, prepared_workflows): + """All {{...}} references use valid DAB syntax.""" + for name, wf in prepared_workflows.items(): + for task in wf.tasks: + params = task.get("notebook_task", {}).get("base_parameters", {}) + for key, val in params.items(): + val_str = str(val) + if "{{" in val_str: + assert DAB_REF_RE.search(val_str), ( + f"{name}/{task.get('task_key')}/{key}: malformed DAB ref: {val_str}" + ) + + +class TestNotebookValidity: + """Verify all generated notebooks pass ast.parse().""" + + def test_all_notebooks_valid_python(self, bundle_dirs): + for name, path in bundle_dirs.items(): + src_dir = path / "src" + if not src_dir.exists(): + continue + for nb_file in src_dir.rglob("*.py"): + content = nb_file.read_text() + python_code = _strip_magic(content) + try: + ast.parse(python_code) + except SyntaxError as exc: + pytest.fail(f"{name}/{nb_file.relative_to(path)}: syntax error: {exc}") + + +# --------------------------------------------------------------------------- +# Pipeline-specific structural assertions +# --------------------------------------------------------------------------- + + +class TestCopyCoverage: + """pl_test_copy_coverage: file source + SQL source copies.""" + + def test_has_two_copy_tasks(self, translated_pipelines): + report = translated_pipelines.get("pl_test_copy_coverage") + if report is None: + pytest.skip("pl_test_copy_coverage not in fixtures") + copies = [t for t in report.pipeline.tasks if isinstance(t, CopyActivity)] + assert len(copies) == 2 + + def test_notebook_has_cloudfiles_or_jdbc(self, bundle_dirs): + path = bundle_dirs.get("pl_test_copy_coverage") + if path is None: + pytest.skip("pl_test_copy_coverage not in bundle_dirs") + notebooks = list((path / "src" / "notebooks").glob("*.py")) + assert len(notebooks) >= 2 + all_content = " ".join(nb.read_text() for nb in notebooks) + # At least one should use cloudFiles (Auto Loader) or JDBC + has_auto_loader = "cloudFiles" in all_content + has_jdbc = "jdbc" in all_content + assert has_auto_loader or has_jdbc, "Expected at least one cloudFiles or JDBC notebook" + + +class TestForEachCoverage: + """pl_test_foreach_coverage: ForEach with task value inputs.""" + + def test_has_foreach_task(self, prepared_workflows): + wf = prepared_workflows.get("pl_test_foreach_coverage") + if wf is None: + pytest.skip("pl_test_foreach_coverage not in prepared_workflows") + foreach_tasks = [t for t in wf.tasks if "for_each_task" in t] + assert len(foreach_tasks) >= 1, "Expected at least one for_each_task" + + def test_foreach_has_inputs(self, prepared_workflows): + wf = prepared_workflows.get("pl_test_foreach_coverage") + if wf is None: + pytest.skip("pl_test_foreach_coverage not in prepared_workflows") + foreach_task = next((t for t in wf.tasks if "for_each_task" in t), None) + assert foreach_task is not None + inputs = foreach_task["for_each_task"].get("inputs", "") + # Inputs should reference a parameter or task value + assert inputs, "for_each_task inputs should not be empty" + + +class TestIfConditionCoverage: + """pl_test_ifcondition_coverage: structured op/left/right conditions.""" + + def test_has_condition_tasks(self, prepared_workflows): + wf = prepared_workflows.get("pl_test_ifcondition_coverage") + if wf is None: + pytest.skip("pl_test_ifcondition_coverage not in prepared_workflows") + cond_tasks = [t for t in wf.tasks if "condition_task" in t] + assert len(cond_tasks) >= 1, "Expected at least one condition_task" + + def test_condition_uses_op_left_right(self, prepared_workflows): + wf = prepared_workflows.get("pl_test_ifcondition_coverage") + if wf is None: + pytest.skip("pl_test_ifcondition_coverage not in prepared_workflows") + for task in wf.tasks: + if "condition_task" not in task: + continue + ct = task["condition_task"] + assert "op" in ct, f"condition_task missing 'op': {ct}" + assert "left" in ct, f"condition_task missing 'left': {ct}" + assert "right" in ct, f"condition_task missing 'right': {ct}" + valid_ops = ( + "EQUAL_TO", + "NOT_EQUAL", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + ) + assert ct["op"] in valid_ops, f"Unexpected op '{ct['op']}'" + + +class TestSwitchCoverage: + """pl_test_switch_coverage: chained condition_task with unique keys.""" + + def test_has_condition_task(self, prepared_workflows): + wf = prepared_workflows.get("pl_test_switch_coverage") + if wf is None: + pytest.skip("pl_test_switch_coverage not in prepared_workflows") + cond_tasks = [t for t in wf.tasks if "condition_task" in t] + assert len(cond_tasks) >= 1, "Expected at least one condition_task from Switch" + + def test_condition_chain_unique_keys(self, bundle_dirs): + """All task keys in the switch output are unique.""" + path = bundle_dirs.get("pl_test_switch_coverage") + if path is None: + pytest.skip("pl_test_switch_coverage not in bundle_dirs") + for rf in (path / "resources").glob("*.yml"): + content = yaml.safe_load(rf.read_text()) + if not content or "resources" not in content: + continue + for job in content["resources"].get("jobs", {}).values(): + all_keys = _collect_all_task_keys(job.get("tasks", [])) + assert len(all_keys) == len(set(all_keys)), f"Duplicate keys in switch: {all_keys}" + + +class TestSetVariableCoverage: + """pl_test_setvariable_coverage: literal, notebook_code, dab_ref kinds.""" + + def test_translates_all_five(self, translated_pipelines): + report = translated_pipelines.get("pl_test_setvariable_coverage") + if report is None: + pytest.skip("pl_test_setvariable_coverage not in translated_pipelines") + svs = [t for t in report.pipeline.tasks if isinstance(t, SetVariableActivity)] + assert len(svs) == 5 + + def test_utcnow_uses_notebook_code(self, translated_pipelines): + report = translated_pipelines.get("pl_test_setvariable_coverage") + if report is None: + pytest.skip("pl_test_setvariable_coverage not in translated_pipelines") + utcnow_sv = next( + ( + t + for t in report.pipeline.tasks + if isinstance(t, SetVariableActivity) and t.variable_name == "runTimestamp" + ), + None, + ) + assert utcnow_sv is not None, "Expected SetVariable for runTimestamp" + assert utcnow_sv.value_kind == "notebook_code" + + def test_pipeline_param_uses_dab_ref(self, translated_pipelines): + report = translated_pipelines.get("pl_test_setvariable_coverage") + if report is None: + pytest.skip("pl_test_setvariable_coverage not in translated_pipelines") + param_sv = next( + ( + t + for t in report.pipeline.tasks + if isinstance(t, SetVariableActivity) and t.variable_name == "environment" + ), + None, + ) + assert param_sv is not None, "Expected SetVariable for environment" + assert param_sv.value_kind == "dab_ref" + assert "job.parameters" in param_sv.variable_value + + +class TestWebActivityCoverage: + """pl_test_webactivity_coverage: resolved headers and auth.""" + + def test_no_expression_dicts_in_params(self, prepared_workflows): + wf = prepared_workflows.get("pl_test_webactivity_coverage") + if wf is None: + pytest.skip("pl_test_webactivity_coverage not in prepared_workflows") + for task in wf.tasks: + params = task.get("notebook_task", {}).get("base_parameters", {}) + for key, val in params.items(): + assert not isinstance(val, dict), f"Expression dict leaked into params: {key}={val}" + + +class TestNotebookCoverage: + """pl_test_notebook_coverage: base_parameters resolved to DAB refs.""" + + def test_params_are_resolved(self, prepared_workflows): + wf = prepared_workflows.get("pl_test_notebook_coverage") + if wf is None: + pytest.skip("pl_test_notebook_coverage not in prepared_workflows") + for task in wf.tasks: + params = task.get("notebook_task", {}).get("base_parameters", {}) + for key, val in params.items(): + assert not isinstance(val, dict), f"Expression dict in params: {key}={val}" + + +class TestLookupCoverage: + """pl_test_lookup_coverage: lookup generates notebooks with task values.""" + + def test_has_two_lookup_tasks(self, translated_pipelines): + report = translated_pipelines.get("pl_test_lookup_coverage") + if report is None: + pytest.skip("pl_test_lookup_coverage not in translated_pipelines") + from flowx.models.ir import LookupActivity + + lookups = [t for t in report.pipeline.tasks if isinstance(t, LookupActivity)] + assert len(lookups) == 2 + + +class TestDeleteCoverage: + """pl_test_delete_coverage: delete generates notebook.""" + + def test_has_delete_task(self, translated_pipelines): + report = translated_pipelines.get("pl_test_delete_coverage") + if report is None: + pytest.skip("pl_test_delete_coverage not in translated_pipelines") + from flowx.models.ir import DeleteActivity + + deletes = [t for t in report.pipeline.tasks if isinstance(t, DeleteActivity)] + assert len(deletes) == 1 + + +class TestWaitCoverage: + """pl_test_wait_coverage: wait generates notebooks.""" + + def test_has_two_wait_tasks(self, translated_pipelines): + report = translated_pipelines.get("pl_test_wait_coverage") + if report is None: + pytest.skip("pl_test_wait_coverage not in translated_pipelines") + from flowx.models.ir import WaitActivity + + waits = [t for t in report.pipeline.tasks if isinstance(t, WaitActivity)] + assert len(waits) == 2 + + +class TestFilterCoverage: + """pl_test_filter_coverage: filter generates notebooks.""" + + def test_has_filter_tasks(self, translated_pipelines): + report = translated_pipelines.get("pl_test_filter_coverage") + if report is None: + pytest.skip("pl_test_filter_coverage not in translated_pipelines") + from flowx.models.ir import FilterActivity + + filters = [t for t in report.pipeline.tasks if isinstance(t, FilterActivity)] + assert len(filters) >= 1 + + +class TestAppendVariableCoverage: + """pl_test_appendvariable_coverage: append variable generates notebooks.""" + + def test_has_append_tasks(self, translated_pipelines): + report = translated_pipelines.get("pl_test_appendvariable_coverage") + if report is None: + pytest.skip("pl_test_appendvariable_coverage not in translated_pipelines") + from flowx.models.ir import AppendVariableActivity + + appends = [t for t in report.pipeline.tasks if isinstance(t, AppendVariableActivity)] + assert len(appends) == 3 + + +class TestExecutePipelineCoverage: + """pl_test_executepipeline_coverage: generates run_job_task.""" + + def test_has_run_job_tasks(self, prepared_workflows): + wf = prepared_workflows.get("pl_test_executepipeline_coverage") + if wf is None: + pytest.skip("pl_test_executepipeline_coverage not in prepared_workflows") + run_jobs = [t for t in wf.tasks if "run_job_task" in t] + assert len(run_jobs) >= 1 + + +class TestSparkJarCoverage: + """pl_test_sparkjar_coverage: generates spark_jar_task.""" + + def test_has_spark_jar_task(self, prepared_workflows): + wf = prepared_workflows.get("pl_test_sparkjar_coverage") + if wf is None: + pytest.skip("pl_test_sparkjar_coverage not in prepared_workflows") + jar_tasks = [t for t in wf.tasks if "spark_jar_task" in t] + assert len(jar_tasks) >= 1 + + +class TestSparkPythonCoverage: + """pl_test_sparkpython_coverage: generates spark_python_task.""" + + def test_has_spark_python_task(self, prepared_workflows): + wf = prepared_workflows.get("pl_test_sparkpython_coverage") + if wf is None: + pytest.skip("pl_test_sparkpython_coverage not in prepared_workflows") + py_tasks = [t for t in wf.tasks if "spark_python_task" in t] + assert len(py_tasks) >= 1 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _collect_all_task_keys(tasks: list[dict]) -> list[str]: + """Recursively collect all task_keys from tasks including nested condition chains.""" + keys = [] + for t in tasks: + if "task_key" in t: + keys.append(t["task_key"]) + # Recurse into condition_task branches + ct = t.get("condition_task", {}) + if ct: + for branch_name in ("if_true", "if_false"): + branch = ct.get(branch_name, []) + if isinstance(branch, list): + keys.extend(_collect_all_task_keys(branch)) + # Recurse into for_each_task + fe = t.get("for_each_task", {}) + if fe: + inner_tasks = fe.get("tasks", []) + if isinstance(inner_tasks, list): + keys.extend(_collect_all_task_keys(inner_tasks)) + return keys diff --git a/tests/resources/json/datasets/dataset_avro_adls.json b/tests/resources/json/datasets/dataset_avro_adls.json new file mode 100644 index 0000000..b32fff1 --- /dev/null +++ b/tests/resources/json/datasets/dataset_avro_adls.json @@ -0,0 +1,43 @@ +{ + "name": "ds_avro_adls_kafka_events", + "properties": { + "type": "Avro", + "linkedServiceName": { + "referenceName": "ls_adls_gen2", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "folderPath": { + "value": "@dataset().folderPath", + "type": "Expression" + }, + "fileSystem": "landing" + }, + "avroCompressionCodec": "snappy" + }, + "parameters": { + "folderPath": { + "type": "String", + "defaultValue": "kafka-events/topic-orders" + } + }, + "schema": [ + { "name": "key", "type": "bytes" }, + { "name": "value", "type": "bytes" }, + { "name": "topic", "type": "string" }, + { "name": "partition", "type": "int" }, + { "name": "offset", "type": "long" }, + { "name": "timestamp", "type": "long" } + ], + "annotations": [ + "avro", + "kafka", + "events" + ], + "folder": { + "name": "Landing/Avro" + } + } +} diff --git a/tests/resources/json/datasets/dataset_azure_blob.json b/tests/resources/json/datasets/dataset_azure_blob.json new file mode 100644 index 0000000..0d29656 --- /dev/null +++ b/tests/resources/json/datasets/dataset_azure_blob.json @@ -0,0 +1,47 @@ +{ + "name": "ds_azure_blob_staging", + "properties": { + "type": "Binary", + "linkedServiceName": { + "referenceName": "ls_azure_blob", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "location": { + "type": "AzureBlobStorageLocation", + "container": "staging", + "folderPath": { + "value": "@dataset().folderPath", + "type": "Expression" + }, + "fileName": { + "value": "@dataset().fileName", + "type": "Expression" + } + }, + "compression": { + "type": "ZipDeflate", + "level": "Optimal" + } + }, + "parameters": { + "folderPath": { + "type": "String", + "defaultValue": "uploads/daily" + }, + "fileName": { + "type": "String", + "defaultValue": "*" + } + }, + "schema": [], + "annotations": [ + "blob", + "staging", + "binary" + ], + "folder": { + "name": "Staging/Blob" + } + } +} diff --git a/tests/resources/json/datasets/dataset_azure_sql_table.json b/tests/resources/json/datasets/dataset_azure_sql_table.json new file mode 100644 index 0000000..90c5062 --- /dev/null +++ b/tests/resources/json/datasets/dataset_azure_sql_table.json @@ -0,0 +1,73 @@ +{ + "name": "ds_azure_sql_orders", + "properties": { + "type": "AzureSqlTable", + "linkedServiceName": { + "referenceName": "ls_azure_sql", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "schema": "dbo", + "table": "orders" + }, + "parameters": {}, + "schema": [ + { + "name": "order_id", + "type": "int", + "precision": 10, + "scale": 0 + }, + { + "name": "customer_id", + "type": "int", + "precision": 10, + "scale": 0 + }, + { + "name": "order_date", + "type": "datetime2", + "precision": 27, + "scale": 7 + }, + { + "name": "total_amount", + "type": "decimal", + "precision": 18, + "scale": 2 + }, + { + "name": "status", + "type": "nvarchar", + "precision": 0, + "scale": 0 + }, + { + "name": "shipping_address", + "type": "nvarchar", + "precision": 0, + "scale": 0 + }, + { + "name": "created_at", + "type": "datetime2", + "precision": 27, + "scale": 7 + }, + { + "name": "updated_at", + "type": "datetime2", + "precision": 27, + "scale": 7 + } + ], + "annotations": [ + "sql", + "orders", + "source" + ], + "folder": { + "name": "Source/SQL" + } + } +} diff --git a/tests/resources/json/datasets/dataset_csv_adls.json b/tests/resources/json/datasets/dataset_csv_adls.json new file mode 100644 index 0000000..e29472d --- /dev/null +++ b/tests/resources/json/datasets/dataset_csv_adls.json @@ -0,0 +1,59 @@ +{ + "name": "ds_csv_adls_customers", + "properties": { + "type": "DelimitedText", + "linkedServiceName": { + "referenceName": "ls_adls_gen2", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@dataset().fileName", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().folderPath", + "type": "Expression" + }, + "fileSystem": "raw" + }, + "columnDelimiter": ",", + "rowDelimiter": "\n", + "encodingName": "UTF-8", + "escapeChar": "\\", + "quoteChar": "\"", + "firstRowAsHeader": true, + "compressionCodec": "none", + "nullValue": "" + }, + "parameters": { + "folderPath": { + "type": "String", + "defaultValue": "customers" + }, + "fileName": { + "type": "String", + "defaultValue": "*.csv" + } + }, + "schema": [ + { "name": "customer_id", "type": "String" }, + { "name": "first_name", "type": "String" }, + { "name": "last_name", "type": "String" }, + { "name": "email", "type": "String" }, + { "name": "signup_date", "type": "String" }, + { "name": "region", "type": "String" }, + { "name": "tier", "type": "String" } + ], + "annotations": [ + "csv", + "customers", + "raw" + ], + "folder": { + "name": "Raw/CSV" + } + } +} diff --git a/tests/resources/json/datasets/dataset_delta_table.json b/tests/resources/json/datasets/dataset_delta_table.json new file mode 100644 index 0000000..c1cacb8 --- /dev/null +++ b/tests/resources/json/datasets/dataset_delta_table.json @@ -0,0 +1,49 @@ +{ + "name": "ds_delta_customers", + "properties": { + "type": "AzureDatabricksDeltaLakeDataset", + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "database": { + "value": "@dataset().databaseName", + "type": "Expression" + }, + "table": { + "value": "@dataset().tableName", + "type": "Expression" + } + }, + "parameters": { + "databaseName": { + "type": "String", + "defaultValue": "curated" + }, + "tableName": { + "type": "String", + "defaultValue": "customers" + } + }, + "schema": [ + { "name": "customer_id", "type": "string" }, + { "name": "first_name", "type": "string" }, + { "name": "last_name", "type": "string" }, + { "name": "email", "type": "string" }, + { "name": "signup_date", "type": "timestamp" }, + { "name": "region", "type": "string" }, + { "name": "tier", "type": "string" }, + { "name": "_load_timestamp", "type": "timestamp" }, + { "name": "_source_file", "type": "string" } + ], + "annotations": [ + "delta", + "curated", + "customers" + ], + "folder": { + "name": "Curated/Delta" + } + } +} diff --git a/tests/resources/json/datasets/dataset_json_adls.json b/tests/resources/json/datasets/dataset_json_adls.json new file mode 100644 index 0000000..be65c6f --- /dev/null +++ b/tests/resources/json/datasets/dataset_json_adls.json @@ -0,0 +1,41 @@ +{ + "name": "ds_json_adls_api_responses", + "properties": { + "type": "Json", + "linkedServiceName": { + "referenceName": "ls_adls_gen2", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "folderPath": { + "value": "@dataset().folderPath", + "type": "Expression" + }, + "fileSystem": "landing" + }, + "encodingName": "UTF-8", + "documentForm": "documentPerLine", + "compressionProperties": { + "type": "GZipReadSettings", + "compressionCodec": "gzip" + } + }, + "parameters": { + "folderPath": { + "type": "String", + "defaultValue": "api-responses/daily" + } + }, + "schema": {}, + "annotations": [ + "json", + "api-responses", + "landing" + ], + "folder": { + "name": "Landing/JSON" + } + } +} diff --git a/tests/resources/json/datasets/dataset_mysql_table.json b/tests/resources/json/datasets/dataset_mysql_table.json new file mode 100644 index 0000000..dcc180c --- /dev/null +++ b/tests/resources/json/datasets/dataset_mysql_table.json @@ -0,0 +1,68 @@ +{ + "name": "ds_mysql_products", + "properties": { + "type": "AzureMySqlTable", + "linkedServiceName": { + "referenceName": "ls_mysql", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "tableName": "products" + }, + "parameters": {}, + "schema": [ + { + "name": "product_id", + "type": "int", + "precision": 11 + }, + { + "name": "product_name", + "type": "varchar", + "precision": 200 + }, + { + "name": "category", + "type": "varchar", + "precision": 100 + }, + { + "name": "price", + "type": "decimal", + "precision": 10, + "scale": 2 + }, + { + "name": "stock_quantity", + "type": "int", + "precision": 11 + }, + { + "name": "sku", + "type": "varchar", + "precision": 50 + }, + { + "name": "is_active", + "type": "tinyint", + "precision": 1 + }, + { + "name": "created_at", + "type": "datetime" + }, + { + "name": "updated_at", + "type": "datetime" + } + ], + "annotations": [ + "mysql", + "products", + "source" + ], + "folder": { + "name": "Source/MySQL" + } + } +} diff --git a/tests/resources/json/datasets/dataset_orc_adls.json b/tests/resources/json/datasets/dataset_orc_adls.json new file mode 100644 index 0000000..939601f --- /dev/null +++ b/tests/resources/json/datasets/dataset_orc_adls.json @@ -0,0 +1,43 @@ +{ + "name": "ds_orc_adls_hive_tables", + "properties": { + "type": "Orc", + "linkedServiceName": { + "referenceName": "ls_adls_gen2", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "folderPath": { + "value": "@dataset().folderPath", + "type": "Expression" + }, + "fileSystem": "warehouse" + }, + "orcCompressionCodec": "zlib" + }, + "parameters": { + "folderPath": { + "type": "String", + "defaultValue": "hive/warehouse/sales_db/transactions" + } + }, + "schema": [ + { "name": "transaction_id", "type": "bigint" }, + { "name": "customer_id", "type": "bigint" }, + { "name": "amount", "type": "decimal(18,2)" }, + { "name": "currency", "type": "string" }, + { "name": "transaction_date", "type": "timestamp" }, + { "name": "category", "type": "string" } + ], + "annotations": [ + "orc", + "hive", + "warehouse" + ], + "folder": { + "name": "Warehouse/ORC" + } + } +} diff --git a/tests/resources/json/datasets/dataset_parquet_adls.json b/tests/resources/json/datasets/dataset_parquet_adls.json new file mode 100644 index 0000000..48e8e59 --- /dev/null +++ b/tests/resources/json/datasets/dataset_parquet_adls.json @@ -0,0 +1,44 @@ +{ + "name": "ds_parquet_adls_events", + "properties": { + "type": "Parquet", + "linkedServiceName": { + "referenceName": "ls_adls_gen2", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "folderPath": { + "value": "@dataset().folderPath", + "type": "Expression" + }, + "fileSystem": "raw" + }, + "compressionCodec": "snappy" + }, + "parameters": { + "folderPath": { + "type": "String", + "defaultValue": "events/2024" + } + }, + "schema": [ + { "name": "event_id", "type": "UTF8", "physicalType": "UTF8" }, + { "name": "event_type", "type": "UTF8", "physicalType": "UTF8" }, + { "name": "user_id", "type": "UTF8", "physicalType": "UTF8" }, + { "name": "event_timestamp", "type": "INT96", "physicalType": "INT96" }, + { "name": "payload", "type": "UTF8", "physicalType": "UTF8" }, + { "name": "source_system", "type": "UTF8", "physicalType": "UTF8" }, + { "name": "partition_date", "type": "UTF8", "physicalType": "UTF8" } + ], + "annotations": [ + "parquet", + "events", + "raw" + ], + "folder": { + "name": "Raw/Parquet" + } + } +} diff --git a/tests/resources/json/datasets/dataset_postgresql_table.json b/tests/resources/json/datasets/dataset_postgresql_table.json new file mode 100644 index 0000000..b94d3df --- /dev/null +++ b/tests/resources/json/datasets/dataset_postgresql_table.json @@ -0,0 +1,55 @@ +{ + "name": "ds_postgresql_users", + "properties": { + "type": "AzurePostgreSqlTable", + "linkedServiceName": { + "referenceName": "ls_postgresql", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "schema": "public", + "table": "users" + }, + "parameters": {}, + "schema": [ + { + "name": "user_id", + "type": "uuid" + }, + { + "name": "username", + "type": "varchar", + "precision": 255 + }, + { + "name": "email", + "type": "varchar", + "precision": 255 + }, + { + "name": "created_at", + "type": "timestamptz" + }, + { + "name": "last_login", + "type": "timestamptz" + }, + { + "name": "is_active", + "type": "boolean" + }, + { + "name": "metadata", + "type": "jsonb" + } + ], + "annotations": [ + "postgresql", + "users", + "source" + ], + "folder": { + "name": "Source/PostgreSQL" + } + } +} diff --git a/tests/resources/json/datasets/ds_csv_adls_customers.json b/tests/resources/json/datasets/ds_csv_adls_customers.json new file mode 100644 index 0000000..1e1c43e --- /dev/null +++ b/tests/resources/json/datasets/ds_csv_adls_customers.json @@ -0,0 +1,22 @@ +{ + "name": "ds_csv_adls_customers", + "properties": { + "type": "DelimitedText", + "linkedServiceName": { + "referenceName": "ls_adls_gen2", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "folderPath": "raw/customers", + "fileSystem": "data" + }, + "columnDelimiter": ",", + "rowDelimiter": "\n", + "quoteChar": "\"", + "escapeChar": "\\", + "firstRowAsHeader": true + } + } +} diff --git a/tests/resources/json/datasets/ds_delta_customers.json b/tests/resources/json/datasets/ds_delta_customers.json new file mode 100644 index 0000000..f0d5395 --- /dev/null +++ b/tests/resources/json/datasets/ds_delta_customers.json @@ -0,0 +1,13 @@ +{ + "name": "ds_delta_customers", + "properties": { + "type": "AzureDatabricksDeltaLakeDataset", + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + }, + "typeProperties": { + "table": "main.default.customers" + } + } +} diff --git a/tests/resources/json/linked_services/ls_adls_gen2.json b/tests/resources/json/linked_services/ls_adls_gen2.json new file mode 100644 index 0000000..50e12e2 --- /dev/null +++ b/tests/resources/json/linked_services/ls_adls_gen2.json @@ -0,0 +1,26 @@ +{ + "name": "ls_adls_gen2", + "properties": { + "type": "AzureBlobFS", + "typeProperties": { + "url": "https://contosodatalake.dfs.core.windows.net", + "accountKey": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "ls_key_vault", + "type": "LinkedServiceReference" + }, + "secretName": "adls-account-key" + } + }, + "annotations": [ + "adls", + "gen2", + "storage" + ], + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + } + } +} diff --git a/tests/resources/json/linked_services/ls_azure_blob.json b/tests/resources/json/linked_services/ls_azure_blob.json new file mode 100644 index 0000000..2f5212f --- /dev/null +++ b/tests/resources/json/linked_services/ls_azure_blob.json @@ -0,0 +1,26 @@ +{ + "name": "ls_azure_blob", + "properties": { + "type": "AzureBlobStorage", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "ls_key_vault", + "type": "LinkedServiceReference" + }, + "secretName": "blob-storage-connection-string" + }, + "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCn0=" + }, + "annotations": [ + "blob", + "storage", + "staging" + ], + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + } + } +} diff --git a/tests/resources/json/linked_services/ls_azure_sql.json b/tests/resources/json/linked_services/ls_azure_sql.json new file mode 100644 index 0000000..038b5fe --- /dev/null +++ b/tests/resources/json/linked_services/ls_azure_sql.json @@ -0,0 +1,26 @@ +{ + "name": "ls_azure_sql", + "properties": { + "type": "AzureSqlDatabase", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "ls_key_vault", + "type": "LinkedServiceReference" + }, + "secretName": "azure-sql-connection-string" + }, + "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCn0=" + }, + "annotations": [ + "sql", + "azure-sql", + "database" + ], + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + } + } +} diff --git a/tests/resources/json/linked_services/ls_databricks_existing_cluster.json b/tests/resources/json/linked_services/ls_databricks_existing_cluster.json new file mode 100644 index 0000000..cd3fd51 --- /dev/null +++ b/tests/resources/json/linked_services/ls_databricks_existing_cluster.json @@ -0,0 +1,29 @@ +{ + "name": "ls_databricks_existing_cluster", + "properties": { + "type": "AzureDatabricks", + "typeProperties": { + "domain": "https://adb-1234567890123456.7.azuredatabricks.net", + "accessToken": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "ls_key_vault", + "type": "LinkedServiceReference" + }, + "secretName": "databricks-access-token" + }, + "existingClusterId": "0123-456789-abcde123", + "authentication": "MSI", + "workspaceResourceId": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-data-platform/providers/Microsoft.Databricks/workspaces/contoso-databricks-prod" + }, + "annotations": [ + "databricks", + "existing-cluster", + "interactive" + ], + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + } + } +} diff --git a/tests/resources/json/linked_services/ls_databricks_new_cluster.json b/tests/resources/json/linked_services/ls_databricks_new_cluster.json new file mode 100644 index 0000000..4d885d6 --- /dev/null +++ b/tests/resources/json/linked_services/ls_databricks_new_cluster.json @@ -0,0 +1,52 @@ +{ + "name": "ls_databricks_new_cluster", + "properties": { + "type": "AzureDatabricks", + "typeProperties": { + "domain": "https://adb-9876543210123456.7.azuredatabricks.net", + "accessToken": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "ls_key_vault", + "type": "LinkedServiceReference" + }, + "secretName": "databricks-access-token-dev" + }, + "authentication": "MSI", + "workspaceResourceId": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-data-platform-dev/providers/Microsoft.Databricks/workspaces/contoso-databricks-dev", + "newClusterVersion": "14.3.x-scala2.12", + "newClusterNumOfWorker": "4", + "newClusterNodeType": "Standard_DS4_v2", + "newClusterDriverNodeType": "Standard_DS5_v2", + "newClusterSparkConf": { + "spark.databricks.delta.preview.enabled": "true", + "spark.databricks.cluster.profile": "singleNode", + "spark.speculation": "true", + "spark.sql.adaptive.enabled": "true" + }, + "newClusterSparkEnvVars": { + "PYSPARK_PYTHON": "/databricks/python3/bin/python3", + "ENV": "dev" + }, + "newClusterCustomTags": { + "Team": "DataEngineering", + "CostCenter": "DE-001", + "Project": "contoso-etl" + }, + "newClusterInitScripts": [ + "dbfs:/init-scripts/install_deps.sh" + ], + "newClusterLogDestination": "dbfs:/cluster-logs", + "newClusterEnableElasticDisk": true + }, + "annotations": [ + "databricks", + "new-cluster", + "job-cluster" + ], + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + } + } +} diff --git a/tests/resources/json/linked_services/ls_key_vault.json b/tests/resources/json/linked_services/ls_key_vault.json new file mode 100644 index 0000000..87ad758 --- /dev/null +++ b/tests/resources/json/linked_services/ls_key_vault.json @@ -0,0 +1,14 @@ +{ + "name": "ls_key_vault", + "properties": { + "type": "AzureKeyVault", + "typeProperties": { + "baseUrl": "https://contoso-data-kv.vault.azure.net/" + }, + "annotations": [ + "key-vault", + "secrets", + "security" + ] + } +} diff --git a/tests/resources/json/linked_services/ls_postgresql.json b/tests/resources/json/linked_services/ls_postgresql.json new file mode 100644 index 0000000..544622f --- /dev/null +++ b/tests/resources/json/linked_services/ls_postgresql.json @@ -0,0 +1,26 @@ +{ + "name": "ls_postgresql", + "properties": { + "type": "AzurePostgreSql", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "ls_key_vault", + "type": "LinkedServiceReference" + }, + "secretName": "postgresql-connection-string" + }, + "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCn0=" + }, + "annotations": [ + "postgresql", + "database", + "source" + ], + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_all_activity_types.json b/tests/resources/json/pipelines/pipeline_all_activity_types.json new file mode 100644 index 0000000..6219f3e --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_all_activity_types.json @@ -0,0 +1,261 @@ +{ + "name": "pipeline_all_activity_types", + "properties": { + "activities": [ + { + "name": "Wait for Upstream", + "type": "Wait", + "dependsOn": [], + "typeProperties": { + "waitTimeInSeconds": 30 + } + }, + { + "name": "Lookup Config", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Wait for Upstream", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT * FROM dbo.config" + }, + "firstRowOnly": true + } + }, + { + "name": "Copy CSV Data", + "type": "Copy", + "dependsOn": [ + { + "activity": "Lookup Config", + "dependencyConditions": ["Succeeded"] + } + ], + "policy": { + "timeout": "0.06:00:00", + "retry": 2, + "retryIntervalInSeconds": 30 + }, + "typeProperties": { + "source": { + "type": "BlobSource" + }, + "sink": { + "type": "DeltaSink" + } + }, + "inputs": [ + { + "referenceName": "ds_blob_source", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "ds_delta_target", + "type": "DatasetReference" + } + ] + }, + { + "name": "Run Notebook", + "type": "DatabricksNotebook", + "dependsOn": [ + { + "activity": "Copy CSV Data", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "notebookPath": "/Shared/ETL/transform", + "baseParameters": { + "env": "dev" + } + }, + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + } + }, + { + "name": "Run Spark Jar", + "type": "DatabricksSparkJar", + "dependsOn": [ + { + "activity": "Run Notebook", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "mainClassName": "com.example.MainJob", + "parameters": ["--input", "/mnt/data"], + "libraries": [ + {"jar": "dbfs:/libs/my-job.jar"} + ] + }, + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + } + }, + { + "name": "Run Spark Python", + "type": "DatabricksSparkPython", + "dependsOn": [ + { + "activity": "Run Spark Jar", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "pythonFile": "dbfs:/scripts/etl.py", + "parameters": ["--mode", "batch"] + }, + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + } + }, + { + "name": "Run Databricks Job", + "type": "DatabricksJob", + "dependsOn": [ + { + "activity": "Run Spark Python", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "jobName": "nightly-aggregation", + "jobId": "12345" + } + }, + { + "name": "Call Web API", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Run Databricks Job", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "url": "https://api.example.com/status", + "method": "GET" + } + }, + { + "name": "Delete Staging Files", + "type": "Delete", + "dependsOn": [ + { + "activity": "Call Web API", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "recursive": true + }, + "inputs": [ + { + "referenceName": "ds_staging_folder", + "type": "DatasetReference" + } + ] + }, + { + "name": "Set Status Variable", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Delete Staging Files", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "variableName": "status", + "value": "completed" + } + }, + { + "name": "Append Log Entry", + "type": "AppendVariable", + "dependsOn": [ + { + "activity": "Set Status Variable", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "variableName": "logEntries", + "value": "Pipeline completed successfully" + } + }, + { + "name": "Filter Active Items", + "type": "Filter", + "dependsOn": [ + { + "activity": "Append Log Entry", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "items": { + "type": "Expression", + "value": "@variables('logEntries')" + }, + "condition": { + "type": "Expression", + "value": "@not(empty(item()))" + } + } + }, + { + "name": "Execute Child Pipeline", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Filter Active Items", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "pipeline": { + "referenceName": "child_pipeline_cleanup", + "type": "PipelineReference" + }, + "parameters": { + "targetDate": "2024-01-15" + }, + "waitOnCompletion": true + } + } + ], + "parameters": { + "env": { + "type": "String", + "defaultValue": "dev" + } + }, + "variables": { + "status": { + "type": "String", + "defaultValue": "" + }, + "logEntries": { + "type": "Array", + "defaultValue": [] + } + }, + "annotations": ["integration-test"], + "folder": { + "name": "Tests" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_all_dependency_conditions.json b/tests/resources/json/pipelines/pipeline_all_dependency_conditions.json new file mode 100644 index 0000000..732b1d4 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_all_dependency_conditions.json @@ -0,0 +1,243 @@ +{ + "name": "pipeline_all_dependency_conditions", + "properties": { + "activities": [ + { + "name": "Primary Data Load", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.02:00:00", + "retry": 2, + "retryIntervalInSeconds": 60, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT * FROM dbo.primary_data WHERE load_date = '@{pipeline().parameters.loadDate}'" + }, + "sink": { + "type": "DeltaSink", + "writeBatchSize": 25000, + "importSettings": { + "type": "DeltaImportCommand" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_primary", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "ds_delta_primary", + "type": "DatasetReference" + } + ] + }, + { + "name": "On Success - Run Transform", + "type": "DatabricksNotebook", + "dependsOn": [ + { + "activity": "Primary Data Load", + "dependencyConditions": ["Succeeded"] + } + ], + "policy": { + "timeout": "0.01:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookPath": "/Shared/ETL/transform_primary", + "baseParameters": { + "loadDate": "@pipeline().parameters.loadDate" + } + }, + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + } + }, + { + "name": "On Failure - Send Alert", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Primary Data Load", + "dependencyConditions": ["Failed"] + } + ], + "policy": { + "timeout": "0.00:05:00", + "retry": 3, + "retryIntervalInSeconds": 5, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "https://example.com/webhook/notify", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "text": "@concat('ALERT: Primary Data Load FAILED. Pipeline: ', pipeline().Pipeline, ', Run: ', pipeline().RunId, ', Error: ', activity('Primary Data Load').Error.message)" + } + } + }, + { + "name": "On Failure - Log Error Details", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Primary Data Load", + "dependencyConditions": ["Failed"] + } + ], + "typeProperties": { + "variableName": "errorDetails", + "value": "@concat('Error in Primary Data Load: ', activity('Primary Data Load').Error.message, ' at ', utcNow())" + } + }, + { + "name": "On Completion - Audit Log", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Primary Data Load", + "dependencyConditions": ["Completed"] + } + ], + "policy": { + "timeout": "0.00:05:00", + "retry": 2, + "retryIntervalInSeconds": 10, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "https://api.contoso.com/v1/audit/log", + "method": "POST", + "headers": { + "Content-Type": "application/json", + "Authorization": "@concat('Bearer ', pipeline().parameters.auditToken)" + }, + "body": { + "pipelineName": "@pipeline().Pipeline", + "runId": "@pipeline().RunId", + "activityName": "Primary Data Load", + "status": "@activity('Primary Data Load').Status", + "timestamp": "@utcNow()" + } + } + }, + { + "name": "On Skipped - Mark Skipped", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "On Success - Run Transform", + "dependencyConditions": ["Skipped"] + } + ], + "typeProperties": { + "variableName": "transformStatus", + "value": "skipped_due_to_upstream_failure" + } + }, + { + "name": "Final Status Update", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "On Success - Run Transform", + "dependencyConditions": ["Succeeded"] + }, + { + "activity": "On Completion - Audit Log", + "dependencyConditions": ["Completed"] + } + ], + "typeProperties": { + "variableName": "finalStatus", + "value": "pipeline_complete" + } + }, + { + "name": "Multi-Condition Dependent Step", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "On Failure - Send Alert", + "dependencyConditions": ["Succeeded"] + }, + { + "activity": "On Failure - Log Error Details", + "dependencyConditions": ["Succeeded"] + } + ], + "policy": { + "timeout": "0.00:05:00", + "retry": 1, + "retryIntervalInSeconds": 10, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "https://api.contoso.com/v1/incidents/create", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "title": "@concat('Data Load Failure: ', pipeline().Pipeline)", + "description": "@variables('errorDetails')", + "severity": "P2", + "team": "data-engineering" + } + } + } + ], + "parameters": { + "loadDate": { + "type": "String", + "defaultValue": "@utcNow('yyyy-MM-dd')" + }, + "auditToken": { + "type": "String", + "defaultValue": "" + } + }, + "variables": { + "errorDetails": { + "type": "String", + "defaultValue": "" + }, + "transformStatus": { + "type": "String", + "defaultValue": "" + }, + "finalStatus": { + "type": "String", + "defaultValue": "pending" + } + }, + "annotations": [ + "dependency-conditions", + "error-handling", + "all-conditions" + ], + "folder": { + "name": "ErrorHandling" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_append_variable_loop.json b/tests/resources/json/pipelines/pipeline_append_variable_loop.json new file mode 100644 index 0000000..91f9619 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_append_variable_loop.json @@ -0,0 +1,136 @@ +{ + "name": "pipeline_append_variable_loop", + "properties": { + "activities": [ + { + "name": "Lookup Source Files", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "0.00:10:00", + "retry": 1, + "retryIntervalInSeconds": 10, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT file_name, file_path, file_size_mb, expected_row_count FROM dbo.file_manifest WHERE process_date = '@{pipeline().parameters.processDate}'", + "queryTimeout": "00:05:00" + }, + "dataset": { + "referenceName": "ds_azure_sql_config", + "type": "DatasetReference" + }, + "firstRowOnly": false + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_config", + "type": "DatasetReference" + } + ] + }, + { + "name": "ForEach File To Process", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Lookup Source Files", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "items": { + "value": "@activity('Lookup Source Files').output.value", + "type": "Expression" + }, + "isSequential": true, + "activities": [ + { + "name": "Append Processed File Name", + "type": "AppendVariable", + "dependsOn": [], + "typeProperties": { + "variableName": "processedFiles", + "value": "@item().file_name" + } + }, + { + "name": "Append File Size", + "type": "AppendVariable", + "dependsOn": [ + { + "activity": "Append Processed File Name", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "variableName": "fileSizes", + "value": "@string(item().file_size_mb)" + } + } + ] + } + }, + { + "name": "Log Processed Files", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "ForEach File To Process", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.00:05:00", + "retry": 1, + "retryIntervalInSeconds": 10, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "https://example.com/webhook/notify", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "text": "@concat('Processed files: ', join(variables('processedFiles'), ', '), '. Total files: ', string(length(variables('processedFiles'))))" + } + } + } + ], + "parameters": { + "processDate": { + "type": "String", + "defaultValue": "@utcNow('yyyy-MM-dd')" + } + }, + "variables": { + "processedFiles": { + "type": "Array", + "defaultValue": [] + }, + "fileSizes": { + "type": "Array", + "defaultValue": [] + } + }, + "annotations": [ + "append-variable", + "accumulator", + "file-processing" + ], + "folder": { + "name": "ETL/FileProcessing" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_complex_etl.json b/tests/resources/json/pipelines/pipeline_complex_etl.json new file mode 100644 index 0000000..698b1e3 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_complex_etl.json @@ -0,0 +1,324 @@ +{ + "name": "pipeline_complex_etl", + "properties": { + "activities": [ + { + "name": "Lookup ETL Config", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "0.00:10:00", + "retry": 2, + "retryIntervalInSeconds": 15, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT source_schema, source_table, target_table, watermark_column, last_watermark, incremental_flag, partition_column FROM dbo.etl_control WHERE pipeline_group = '@{pipeline().parameters.pipelineGroup}' AND is_active = 1 ORDER BY load_sequence", + "queryTimeout": "00:05:00" + }, + "dataset": { + "referenceName": "ds_azure_sql_config", + "type": "DatasetReference" + }, + "firstRowOnly": false + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_config", + "type": "DatasetReference" + } + ] + }, + { + "name": "ForEach Table In Config", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Lookup ETL Config", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "items": { + "value": "@activity('Lookup ETL Config').output.value", + "type": "Expression" + }, + "isSequential": false, + "batchCount": 8, + "activities": [ + { + "name": "Copy Source to Staging", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.02:00:00", + "retry": 3, + "retryIntervalInSeconds": 60, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "@if(equals(item().incremental_flag, 1), concat('SELECT * FROM ', item().source_schema, '.', item().source_table, ' WHERE ', item().watermark_column, ' > ''', item().last_watermark, ''''), concat('SELECT * FROM ', item().source_schema, '.', item().source_table))", + "queryTimeout": "01:00:00", + "partitionOption": "None" + }, + "sink": { + "type": "DeltaSink", + "writeBatchSize": 50000, + "importSettings": { + "type": "DeltaImportCommand" + }, + "tableActionOption": "Append" + }, + "enableStaging": true, + "stagingSettings": { + "linkedServiceName": { + "referenceName": "ls_azure_blob", + "type": "LinkedServiceReference" + }, + "path": "@concat('staging/', item().source_table)" + }, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + }, + "parallelCopies": 4, + "dataIntegrationUnits": 8 + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_generic", + "type": "DatasetReference", + "parameters": { + "tableName": "@item().source_table", + "schemaName": "@item().source_schema" + } + } + ], + "outputs": [ + { + "referenceName": "ds_delta_staging", + "type": "DatasetReference", + "parameters": { + "tableName": "@concat('stg_', item().source_table)" + } + } + ] + } + ] + } + }, + { + "name": "Run Transform Notebook", + "type": "DatabricksNotebook", + "dependsOn": [ + { + "activity": "ForEach Table In Config", + "dependencyConditions": ["Succeeded"] + } + ], + "policy": { + "timeout": "0.04:00:00", + "retry": 1, + "retryIntervalInSeconds": 120, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookPath": "/Repos/data-engineering/notebooks/transform_and_merge", + "baseParameters": { + "pipelineGroup": "@pipeline().parameters.pipelineGroup", + "processDate": "@pipeline().parameters.processDate", + "environment": "@pipeline().parameters.environment", + "enableMerge": "@string(pipeline().parameters.enableMerge)" + } + }, + "linkedServiceName": { + "referenceName": "ls_databricks_new_cluster", + "type": "LinkedServiceReference" + } + }, + { + "name": "Run Data Quality Checks", + "type": "DatabricksNotebook", + "dependsOn": [ + { + "activity": "Run Transform Notebook", + "dependencyConditions": ["Succeeded"] + } + ], + "policy": { + "timeout": "0.01:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookPath": "/Repos/data-engineering/notebooks/quality_gate", + "baseParameters": { + "pipelineGroup": "@pipeline().parameters.pipelineGroup", + "qualityThreshold": "0.98", + "failOnBreach": "true" + } + }, + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + } + }, + { + "name": "Delete Staging Data", + "type": "Delete", + "dependsOn": [ + { + "activity": "Run Data Quality Checks", + "dependencyConditions": ["Completed"] + } + ], + "policy": { + "timeout": "0.01:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "dataset": { + "referenceName": "ds_adls_staging_folder", + "type": "DatasetReference", + "parameters": { + "folderPath": "@concat('staging/', pipeline().parameters.pipelineGroup)" + } + }, + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true + }, + "recursive": true, + "enableLogging": true, + "logStorageSettings": { + "linkedServiceName": { + "referenceName": "ls_adls_gen2", + "type": "LinkedServiceReference" + }, + "path": "logs/cleanup" + } + }, + "inputs": [ + { + "referenceName": "ds_adls_staging_folder", + "type": "DatasetReference", + "parameters": { + "folderPath": "@concat('staging/', pipeline().parameters.pipelineGroup)" + } + } + ] + }, + { + "name": "Update Watermarks", + "type": "DatabricksNotebook", + "dependsOn": [ + { + "activity": "Run Data Quality Checks", + "dependencyConditions": ["Succeeded"] + } + ], + "policy": { + "timeout": "0.00:30:00", + "retry": 2, + "retryIntervalInSeconds": 15, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookPath": "/Repos/data-engineering/notebooks/update_watermarks", + "baseParameters": { + "pipelineGroup": "@pipeline().parameters.pipelineGroup", + "processDate": "@pipeline().parameters.processDate" + } + }, + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + } + }, + { + "name": "Send Completion Notification", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Update Watermarks", + "dependencyConditions": ["Succeeded"] + }, + { + "activity": "Delete Staging Data", + "dependencyConditions": ["Completed"] + } + ], + "policy": { + "timeout": "0.00:05:00", + "retry": 1, + "retryIntervalInSeconds": 10, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "https://example.com/webhook/notify", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "text": "@concat('ETL Pipeline [', pipeline().parameters.pipelineGroup, '] completed successfully for ', pipeline().parameters.processDate, '. Run ID: ', pipeline().RunId)" + } + } + } + ], + "parameters": { + "pipelineGroup": { + "type": "String", + "defaultValue": "sales" + }, + "processDate": { + "type": "String", + "defaultValue": "@utcNow('yyyy-MM-dd')" + }, + "environment": { + "type": "String", + "defaultValue": "prod" + }, + "enableMerge": { + "type": "Bool", + "defaultValue": true + } + }, + "variables": { + "etlStatus": { + "type": "String", + "defaultValue": "InProgress" + }, + "processedTableCount": { + "type": "String", + "defaultValue": "0" + } + }, + "annotations": [ + "complex-etl", + "multi-stage", + "production" + ], + "folder": { + "name": "ETL/Production" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_complex_orchestration.json b/tests/resources/json/pipelines/pipeline_complex_orchestration.json new file mode 100644 index 0000000..e85117f --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_complex_orchestration.json @@ -0,0 +1,336 @@ +{ + "name": "pipeline_complex_orchestration", + "properties": { + "activities": [ + { + "name": "Set Run Mode", + "type": "SetVariable", + "dependsOn": [], + "typeProperties": { + "variableName": "runMode", + "value": "@pipeline().parameters.mode" + } + }, + { + "name": "Route By Mode", + "type": "Switch", + "dependsOn": [ + { + "activity": "Set Run Mode", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "on": { + "value": "@variables('runMode')", + "type": "Expression" + }, + "cases": [ + { + "value": "full", + "activities": [ + { + "name": "Check Source Readiness", + "type": "IfCondition", + "dependsOn": [], + "typeProperties": { + "expression": { + "value": "@equals(pipeline().parameters.sourceReady, true)", + "type": "Expression" + }, + "ifTrueActivities": [ + { + "name": "Run Full Ingestion Pipeline", + "type": "ExecutePipeline", + "dependsOn": [], + "typeProperties": { + "pipeline": { + "referenceName": "pipeline_copy_sql_to_delta", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "watermarkDate": "1900-01-01", + "partitionUpperBound": "@pipeline().parameters.maxPartition" + } + } + }, + { + "name": "Run Full Transform Pipeline", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Run Full Ingestion Pipeline", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "pipeline": { + "referenceName": "pipeline_notebook_with_params", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "env": "@pipeline().parameters.environment", + "processDate": "@pipeline().parameters.processDate", + "sourceTable": "raw.all_data", + "targetTable": "curated.all_data" + } + } + } + ], + "ifFalseActivities": [ + { + "name": "Wait For Source", + "type": "Wait", + "dependsOn": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Notify Source Not Ready", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Wait For Source", + "dependencyConditions": ["Succeeded"] + } + ], + "policy": { + "timeout": "0.00:05:00", + "retry": 1, + "retryIntervalInSeconds": 10 + }, + "typeProperties": { + "url": "https://example.com/webhook/notify", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "text": "@concat('Source system not ready for full load. Pipeline: ', pipeline().Pipeline, ', Run ID: ', pipeline().RunId)" + } + } + } + ] + } + } + ] + }, + { + "value": "incremental", + "activities": [ + { + "name": "Run Incremental Ingestion", + "type": "ExecutePipeline", + "dependsOn": [], + "typeProperties": { + "pipeline": { + "referenceName": "pipeline_copy_sql_to_delta", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "watermarkDate": "@pipeline().parameters.lastWatermark", + "partitionUpperBound": "@pipeline().parameters.maxPartition" + } + } + }, + { + "name": "Wait Between Steps", + "type": "Wait", + "dependsOn": [ + { + "activity": "Run Incremental Ingestion", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "waitTimeInSeconds": 30 + } + }, + { + "name": "Run Incremental Transform", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Wait Between Steps", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "pipeline": { + "referenceName": "pipeline_notebook_with_params", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "env": "@pipeline().parameters.environment", + "processDate": "@pipeline().parameters.processDate", + "sourceTable": "raw.incremental_data", + "targetTable": "curated.incremental_data" + } + } + } + ] + }, + { + "value": "reprocess", + "activities": [ + { + "name": "Set Reprocess Date Range", + "type": "SetVariable", + "dependsOn": [], + "typeProperties": { + "variableName": "dateRange", + "value": "@concat(pipeline().parameters.reprocessStart, '|', pipeline().parameters.reprocessEnd)" + } + }, + { + "name": "Run Reprocess Pipeline", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Set Reprocess Date Range", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "pipeline": { + "referenceName": "pipeline_foreach_with_copy", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "watermarkDate": "@pipeline().parameters.reprocessStart" + } + } + } + ] + } + ], + "defaultActivities": [ + { + "name": "Log Unknown Mode", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "0.00:05:00", + "retry": 0, + "retryIntervalInSeconds": 30 + }, + "typeProperties": { + "url": "https://example.com/webhook/notify", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "text": "@concat('Unknown pipeline mode: ', variables('runMode'))" + } + } + } + ] + } + }, + { + "name": "Set Completion Status", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Route By Mode", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "variableName": "completionStatus", + "value": "completed" + } + }, + { + "name": "Send Final Notification", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Set Completion Status", + "dependencyConditions": ["Completed"] + } + ], + "policy": { + "timeout": "0.00:05:00", + "retry": 2, + "retryIntervalInSeconds": 10, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "https://example.com/webhook/notify", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "text": "@concat('Orchestration pipeline completed. Mode: ', variables('runMode'), ', Status: ', variables('completionStatus'), ', Run: ', pipeline().RunId)" + } + } + } + ], + "parameters": { + "mode": { + "type": "String", + "defaultValue": "incremental" + }, + "environment": { + "type": "String", + "defaultValue": "prod" + }, + "processDate": { + "type": "String", + "defaultValue": "@utcNow('yyyy-MM-dd')" + }, + "lastWatermark": { + "type": "String", + "defaultValue": "2024-01-01" + }, + "maxPartition": { + "type": "Int", + "defaultValue": 1000000 + }, + "sourceReady": { + "type": "Bool", + "defaultValue": true + }, + "reprocessStart": { + "type": "String", + "defaultValue": "2024-01-01" + }, + "reprocessEnd": { + "type": "String", + "defaultValue": "2024-01-31" + } + }, + "variables": { + "runMode": { + "type": "String", + "defaultValue": "" + }, + "completionStatus": { + "type": "String", + "defaultValue": "pending" + }, + "dateRange": { + "type": "String", + "defaultValue": "" + } + }, + "annotations": [ + "orchestration", + "complex", + "multi-mode" + ], + "folder": { + "name": "Orchestration/Master" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_copy_csv_to_delta.json b/tests/resources/json/pipelines/pipeline_copy_csv_to_delta.json new file mode 100644 index 0000000..f8f3b31 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_copy_csv_to_delta.json @@ -0,0 +1,116 @@ +{ + "name": "pipeline_copy_csv_to_delta", + "properties": { + "activities": [ + { + "name": "Copy CSV to Delta Lake", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 2, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true, + "wildcardFolderPath": "raw/customers", + "wildcardFileName": "*.csv", + "enablePartitionDiscovery": false + }, + "formatSettings": { + "type": "DelimitedTextReadSettings", + "skipLineCount": 0, + "compressionProperties": null + } + }, + "sink": { + "type": "DeltaSink", + "writeBatchSize": 10000, + "writeBatchTimeout": "00:30:00", + "importSettings": { + "type": "DeltaImportCommand" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { "name": "customer_id", "type": "String" }, + "sink": { "name": "customer_id", "type": "String" } + }, + { + "source": { "name": "first_name", "type": "String" }, + "sink": { "name": "first_name", "type": "String" } + }, + { + "source": { "name": "last_name", "type": "String" }, + "sink": { "name": "last_name", "type": "String" } + }, + { + "source": { "name": "email", "type": "String" }, + "sink": { "name": "email", "type": "String" } + }, + { + "source": { "name": "signup_date", "type": "DateTime" }, + "sink": { "name": "signup_date", "type": "Timestamp" } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + }, + "parallelCopies": 4, + "dataIntegrationUnits": 8 + }, + "inputs": [ + { + "referenceName": "ds_csv_adls_customers", + "type": "DatasetReference", + "parameters": { + "folderPath": "@pipeline().parameters.sourceFolderPath" + } + } + ], + "outputs": [ + { + "referenceName": "ds_delta_customers", + "type": "DatasetReference" + } + ] + } + ], + "parameters": { + "sourceFolderPath": { + "type": "String", + "defaultValue": "raw/customers/2024" + }, + "triggerDate": { + "type": "String", + "defaultValue": "" + } + }, + "variables": { + "rowsCopied": { + "type": "String", + "defaultValue": "0" + } + }, + "annotations": [ + "etl", + "customers", + "csv-to-delta" + ], + "folder": { + "name": "ETL/Ingestion" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_copy_parquet_to_delta.json b/tests/resources/json/pipelines/pipeline_copy_parquet_to_delta.json new file mode 100644 index 0000000..83f6258 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_copy_parquet_to_delta.json @@ -0,0 +1,93 @@ +{ + "name": "pipeline_copy_parquet_to_delta", + "properties": { + "activities": [ + { + "name": "Copy Parquet to Delta Partitioned", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.06:00:00", + "retry": 1, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "ParquetSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true, + "wildcardFolderPath": "@concat('events/', pipeline().parameters.eventDate)", + "wildcardFileName": "part-*.parquet", + "enablePartitionDiscovery": true, + "partitionRootPath": "events", + "modifiedDatetimeStart": "@pipeline().parameters.windowStart", + "modifiedDatetimeEnd": "@pipeline().parameters.windowEnd" + } + }, + "sink": { + "type": "DeltaSink", + "writeBatchSize": 100000, + "importSettings": { + "type": "DeltaImportCommand" + }, + "tableActionOption": "Append" + }, + "enableStaging": false, + "parallelCopies": 16, + "dataIntegrationUnits": 32, + "preserveRules": [ + "*.parquet" + ], + "preserve": [ + "Attributes" + ] + }, + "inputs": [ + { + "referenceName": "ds_parquet_adls_events", + "type": "DatasetReference", + "parameters": { + "folderPath": "@pipeline().parameters.sourcePath" + } + } + ], + "outputs": [ + { + "referenceName": "ds_delta_events", + "type": "DatasetReference" + } + ] + } + ], + "parameters": { + "sourcePath": { + "type": "String", + "defaultValue": "events/2024/01" + }, + "eventDate": { + "type": "String", + "defaultValue": "2024-01-15" + }, + "windowStart": { + "type": "String", + "defaultValue": "2024-01-15T00:00:00Z" + }, + "windowEnd": { + "type": "String", + "defaultValue": "2024-01-16T00:00:00Z" + } + }, + "variables": {}, + "annotations": [ + "etl", + "events", + "parquet-to-delta" + ], + "folder": { + "name": "ETL/Ingestion" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_copy_sql_to_delta.json b/tests/resources/json/pipelines/pipeline_copy_sql_to_delta.json new file mode 100644 index 0000000..295e0d0 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_copy_sql_to_delta.json @@ -0,0 +1,114 @@ +{ + "name": "pipeline_copy_sql_to_delta", + "properties": { + "activities": [ + { + "name": "Copy SQL Server to Delta", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "1.00:00:00", + "retry": 3, + "retryIntervalInSeconds": 60, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT o.order_id, o.customer_id, o.order_date, o.total_amount, o.status, c.region FROM dbo.orders o INNER JOIN dbo.customers c ON o.customer_id = c.customer_id WHERE o.order_date >= '@{pipeline().parameters.watermarkDate}'", + "queryTimeout": "00:30:00", + "isolationLevel": "ReadCommitted", + "partitionOption": "DynamicRange", + "partitionSettings": { + "partitionColumnName": "order_id", + "partitionUpperBound": "@{pipeline().parameters.partitionUpperBound}", + "partitionLowerBound": "1" + } + }, + "sink": { + "type": "DeltaSink", + "writeBatchSize": 50000, + "importSettings": { + "type": "DeltaImportCommand", + "dateFormat": "yyyy-MM-dd", + "timestampFormat": "yyyy-MM-dd'T'HH:mm:ss.SSSZ" + }, + "tableActionOption": "Append" + }, + "enableStaging": true, + "stagingSettings": { + "linkedServiceName": { + "referenceName": "ls_staging_blob", + "type": "LinkedServiceReference" + }, + "path": "staging/orders" + }, + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { "name": "order_id", "type": "Int32" }, + "sink": { "name": "order_id", "type": "Int64" } + }, + { + "source": { "name": "customer_id", "type": "Int32" }, + "sink": { "name": "customer_id", "type": "Int64" } + }, + { + "source": { "name": "order_date", "type": "DateTime" }, + "sink": { "name": "order_date", "type": "Timestamp" } + }, + { + "source": { "name": "total_amount", "type": "Decimal" }, + "sink": { "name": "total_amount", "type": "Double" } + }, + { + "source": { "name": "status", "type": "String" }, + "sink": { "name": "status", "type": "String" } + }, + { + "source": { "name": "region", "type": "String" }, + "sink": { "name": "region", "type": "String" } + } + ] + }, + "parallelCopies": 8, + "dataIntegrationUnits": 16 + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_orders", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "ds_delta_orders", + "type": "DatasetReference" + } + ] + } + ], + "parameters": { + "watermarkDate": { + "type": "String", + "defaultValue": "2024-01-01" + }, + "partitionUpperBound": { + "type": "Int", + "defaultValue": 1000000 + } + }, + "variables": {}, + "annotations": [ + "etl", + "orders", + "sql-to-delta", + "incremental" + ], + "folder": { + "name": "ETL/Ingestion" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_delete_recursive.json b/tests/resources/json/pipelines/pipeline_delete_recursive.json new file mode 100644 index 0000000..006ccf3 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_delete_recursive.json @@ -0,0 +1,114 @@ +{ + "name": "pipeline_delete_recursive", + "properties": { + "activities": [ + { + "name": "Delete Staging Files", + "type": "Delete", + "dependsOn": [], + "policy": { + "timeout": "0.01:00:00", + "retry": 1, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "dataset": { + "referenceName": "ds_adls_staging_folder", + "type": "DatasetReference", + "parameters": { + "folderPath": "@pipeline().parameters.stagingPath" + } + }, + "enableLogging": true, + "logStorageSettings": { + "linkedServiceName": { + "referenceName": "ls_adls_gen2", + "type": "LinkedServiceReference" + }, + "path": "logs/delete-activity" + }, + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true, + "wildcardFolderPath": "@pipeline().parameters.stagingPath", + "wildcardFileName": "*" + }, + "recursive": true, + "maxConcurrentConnections": 4 + }, + "inputs": [ + { + "referenceName": "ds_adls_staging_folder", + "type": "DatasetReference", + "parameters": { + "folderPath": "@pipeline().parameters.stagingPath" + } + } + ] + }, + { + "name": "Delete Archive Files Older Than 30 Days", + "type": "Delete", + "dependsOn": [ + { + "activity": "Delete Staging Files", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.02:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "dataset": { + "referenceName": "ds_adls_archive_folder", + "type": "DatasetReference" + }, + "enableLogging": true, + "logStorageSettings": { + "linkedServiceName": { + "referenceName": "ls_adls_gen2", + "type": "LinkedServiceReference" + }, + "path": "logs/delete-activity" + }, + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true, + "modifiedDatetimeEnd": "@adddays(utcNow(), -30)", + "wildcardFileName": "*.parquet" + }, + "recursive": true + }, + "inputs": [ + { + "referenceName": "ds_adls_archive_folder", + "type": "DatasetReference" + } + ] + } + ], + "parameters": { + "stagingPath": { + "type": "String", + "defaultValue": "staging/daily" + } + }, + "variables": {}, + "annotations": [ + "cleanup", + "delete", + "maintenance" + ], + "folder": { + "name": "Maintenance/Cleanup" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_execute_pipeline_nested.json b/tests/resources/json/pipelines/pipeline_execute_pipeline_nested.json new file mode 100644 index 0000000..0aa9a96 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_execute_pipeline_nested.json @@ -0,0 +1,93 @@ +{ + "name": "pipeline_execute_pipeline_nested", + "properties": { + "activities": [ + { + "name": "Run Ingestion Pipeline", + "type": "ExecutePipeline", + "dependsOn": [], + "typeProperties": { + "pipeline": { + "referenceName": "pipeline_copy_sql_to_delta", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "watermarkDate": "@pipeline().parameters.startDate", + "partitionUpperBound": "@pipeline().parameters.maxPartition" + } + } + }, + { + "name": "Run Transform Pipeline", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Run Ingestion Pipeline", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "pipeline": { + "referenceName": "pipeline_notebook_with_params", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "env": "@pipeline().parameters.environment", + "processDate": "@pipeline().parameters.startDate", + "sourceTable": "raw.orders", + "targetTable": "curated.orders" + } + } + }, + { + "name": "Run Cleanup Pipeline", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Run Transform Pipeline", + "dependencyConditions": [ + "Completed" + ] + } + ], + "typeProperties": { + "pipeline": { + "referenceName": "pipeline_delete_recursive", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "stagingPath": "@concat('staging/', pipeline().parameters.startDate)" + } + } + } + ], + "parameters": { + "environment": { + "type": "String", + "defaultValue": "prod" + }, + "startDate": { + "type": "String", + "defaultValue": "@utcNow('yyyy-MM-dd')" + }, + "maxPartition": { + "type": "Int", + "defaultValue": 500000 + } + }, + "variables": {}, + "annotations": [ + "orchestration", + "nested", + "execute-pipeline" + ], + "folder": { + "name": "Orchestration" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_filter_array.json b/tests/resources/json/pipelines/pipeline_filter_array.json new file mode 100644 index 0000000..5e0208f --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_filter_array.json @@ -0,0 +1,115 @@ +{ + "name": "pipeline_filter_array", + "properties": { + "activities": [ + { + "name": "Lookup All Tables", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "0.00:10:00", + "retry": 1, + "retryIntervalInSeconds": 10, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT table_name, table_schema, row_count, last_updated, is_priority FROM dbo.table_inventory", + "queryTimeout": "00:05:00" + }, + "dataset": { + "referenceName": "ds_azure_sql_config", + "type": "DatasetReference" + }, + "firstRowOnly": false + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_config", + "type": "DatasetReference" + } + ] + }, + { + "name": "Filter Priority Tables", + "type": "Filter", + "dependsOn": [ + { + "activity": "Lookup All Tables", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "items": { + "value": "@activity('Lookup All Tables').output.value", + "type": "Expression" + }, + "condition": { + "value": "@and(equals(item().is_priority, true), greater(item().row_count, 0))", + "type": "Expression" + } + } + }, + { + "name": "ForEach Priority Table", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Filter Priority Tables", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "items": { + "value": "@activity('Filter Priority Tables').output.Value", + "type": "Expression" + }, + "isSequential": false, + "batchCount": 5, + "activities": [ + { + "name": "Run Priority Table ETL", + "type": "DatabricksNotebook", + "dependsOn": [], + "policy": { + "timeout": "0.01:00:00", + "retry": 1, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookPath": "/Shared/ETL/process_priority_table", + "baseParameters": { + "tableName": "@item().table_name", + "schemaName": "@item().table_schema", + "rowCount": "@string(item().row_count)" + } + }, + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + } + } + ] + } + } + ], + "parameters": {}, + "variables": {}, + "annotations": [ + "filter", + "priority", + "dynamic" + ], + "folder": { + "name": "ETL/Dynamic" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_foreach_switch.json b/tests/resources/json/pipelines/pipeline_foreach_switch.json new file mode 100644 index 0000000..e1cc47b --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_foreach_switch.json @@ -0,0 +1,129 @@ +{ + "name": "pipeline_foreach_switch", + "properties": { + "activities": [ + { + "name": "Get Table List", + "type": "Lookup", + "dependsOn": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT table_name, load_type FROM dbo.config_tables WHERE is_active = 1" + }, + "firstRowOnly": false + } + }, + { + "name": "Process Each Table", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Table List", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "items": { + "type": "Expression", + "value": "@activity('Get Table List').output.value" + }, + "isSequential": false, + "batchCount": 10, + "activities": [ + { + "name": "Route By Load Type", + "type": "Switch", + "dependsOn": [], + "typeProperties": { + "on": { + "type": "Expression", + "value": "@item().load_type" + }, + "cases": [ + { + "value": "full", + "activities": [ + { + "name": "Full Load Copy", + "type": "Copy", + "dependsOn": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT * FROM @{item().table_name}" + }, + "sink": { + "type": "DeltaSink" + } + } + } + ] + }, + { + "value": "incremental", + "activities": [ + { + "name": "Incremental Load Copy", + "type": "Copy", + "dependsOn": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT * FROM @{item().table_name} WHERE updated_at > '@{item().last_watermark}'" + }, + "sink": { + "type": "DeltaSink" + } + } + } + ] + } + ], + "defaultActivities": [ + { + "name": "Log Unknown Load Type", + "type": "WebActivity", + "dependsOn": [], + "typeProperties": { + "url": "https://api.internal.com/log", + "method": "POST", + "body": { + "message": "Unknown load type" + } + } + } + ] + } + } + ] + } + }, + { + "name": "Set Completion Status", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Process Each Table", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "variableName": "completionStatus", + "value": "completed" + } + } + ], + "parameters": {}, + "variables": { + "completionStatus": { + "type": "String", + "defaultValue": "pending" + } + }, + "annotations": ["etl", "dynamic-tables"], + "folder": { + "name": "ETL/Dynamic" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_foreach_with_copy.json b/tests/resources/json/pipelines/pipeline_foreach_with_copy.json new file mode 100644 index 0000000..a5a3ee7 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_foreach_with_copy.json @@ -0,0 +1,93 @@ +{ + "name": "pipeline_foreach_with_copy", + "properties": { + "activities": [ + { + "name": "ForEach Table Copy", + "type": "ForEach", + "dependsOn": [], + "typeProperties": { + "items": { + "value": "@pipeline().parameters.tableList", + "type": "Expression" + }, + "isSequential": false, + "batchCount": 10, + "activities": [ + { + "name": "Copy Table Data", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.01:00:00", + "retry": 2, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "@concat('SELECT * FROM ', item().schemaName, '.', item().tableName, ' WHERE modified_date >= ''', pipeline().parameters.watermarkDate, '''')", + "queryTimeout": "00:10:00" + }, + "sink": { + "type": "DeltaSink", + "writeBatchSize": 10000, + "importSettings": { + "type": "DeltaImportCommand" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_generic", + "type": "DatasetReference", + "parameters": { + "tableName": "@item().tableName", + "schemaName": "@item().schemaName" + } + } + ], + "outputs": [ + { + "referenceName": "ds_delta_generic", + "type": "DatasetReference", + "parameters": { + "tableName": "@item().tableName" + } + } + ] + } + ] + } + } + ], + "parameters": { + "tableList": { + "type": "Array", + "defaultValue": [ + { "schemaName": "dbo", "tableName": "customers" }, + { "schemaName": "dbo", "tableName": "orders" }, + { "schemaName": "dbo", "tableName": "products" }, + { "schemaName": "dbo", "tableName": "order_items" }, + { "schemaName": "sales", "tableName": "regions" } + ] + }, + "watermarkDate": { + "type": "String", + "defaultValue": "2024-01-01" + } + }, + "variables": {}, + "annotations": [ + "etl", + "foreach", + "multi-table" + ], + "folder": { + "name": "ETL/Ingestion" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_if_condition_branching.json b/tests/resources/json/pipelines/pipeline_if_condition_branching.json new file mode 100644 index 0000000..c3da24c --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_if_condition_branching.json @@ -0,0 +1,164 @@ +{ + "name": "pipeline_if_condition_branching", + "properties": { + "activities": [ + { + "name": "Lookup Row Count", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "0.00:10:00", + "retry": 1, + "retryIntervalInSeconds": 10, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT COUNT(*) as row_count FROM dbo.staging_data WHERE process_date = '@{pipeline().parameters.processDate}'", + "queryTimeout": "00:05:00" + }, + "dataset": { + "referenceName": "ds_azure_sql_staging", + "type": "DatasetReference" + }, + "firstRowOnly": true + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_staging", + "type": "DatasetReference" + } + ] + }, + { + "name": "Check Data Exists", + "type": "IfCondition", + "dependsOn": [ + { + "activity": "Lookup Row Count", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "expression": { + "value": "@greater(int(activity('Lookup Row Count').output.firstRow.row_count), 0)", + "type": "Expression" + }, + "ifTrueActivities": [ + { + "name": "Copy Staging to Curated", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.02:00:00", + "retry": 2, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT * FROM dbo.staging_data WHERE process_date = '@{pipeline().parameters.processDate}'" + }, + "sink": { + "type": "DeltaSink", + "writeBatchSize": 25000, + "importSettings": { + "type": "DeltaImportCommand" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_staging", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "ds_delta_curated", + "type": "DatasetReference" + } + ] + }, + { + "name": "Run Quality Check Notebook", + "type": "DatabricksNotebook", + "dependsOn": [ + { + "activity": "Copy Staging to Curated", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.01:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookPath": "/Shared/DQ/quality_checks", + "baseParameters": { + "tableName": "curated.staging_data", + "processDate": "@pipeline().parameters.processDate" + } + }, + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + } + } + ], + "ifFalseActivities": [ + { + "name": "Log No Data Found", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "0.00:05:00", + "retry": 1, + "retryIntervalInSeconds": 10, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "https://example.com/webhook/notify", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "text": "@concat('No data found for process date: ', pipeline().parameters.processDate, ' in pipeline: ', pipeline().Pipeline)" + } + } + } + ] + } + } + ], + "parameters": { + "processDate": { + "type": "String", + "defaultValue": "@utcNow('yyyy-MM-dd')" + } + }, + "variables": {}, + "annotations": [ + "conditional", + "branching", + "etl" + ], + "folder": { + "name": "ETL/Conditional" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_lookup_and_foreach.json b/tests/resources/json/pipelines/pipeline_lookup_and_foreach.json new file mode 100644 index 0000000..ab587b0 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_lookup_and_foreach.json @@ -0,0 +1,114 @@ +{ + "name": "pipeline_lookup_and_foreach", + "properties": { + "activities": [ + { + "name": "Lookup Config Tables", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "0.00:10:00", + "retry": 2, + "retryIntervalInSeconds": 15, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT table_schema, table_name, watermark_column, watermark_value, is_active FROM dbo.etl_config WHERE is_active = 1 ORDER BY load_priority", + "queryTimeout": "00:05:00" + }, + "dataset": { + "referenceName": "ds_azure_sql_config", + "type": "DatasetReference" + }, + "firstRowOnly": false + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_config", + "type": "DatasetReference" + } + ] + }, + { + "name": "ForEach Config Table", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Lookup Config Tables", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "items": { + "value": "@activity('Lookup Config Tables').output.value", + "type": "Expression" + }, + "isSequential": true, + "activities": [ + { + "name": "Copy Config Driven Table", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.01:00:00", + "retry": 2, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "@concat('SELECT * FROM ', item().table_schema, '.', item().table_name, ' WHERE ', item().watermark_column, ' > ''', item().watermark_value, '''')", + "queryTimeout": "00:30:00" + }, + "sink": { + "type": "DeltaSink", + "writeBatchSize": 20000, + "importSettings": { + "type": "DeltaImportCommand" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_generic", + "type": "DatasetReference", + "parameters": { + "tableName": "@item().table_name", + "schemaName": "@item().table_schema" + } + } + ], + "outputs": [ + { + "referenceName": "ds_delta_generic", + "type": "DatasetReference", + "parameters": { + "tableName": "@item().table_name" + } + } + ] + } + ] + } + } + ], + "parameters": {}, + "variables": {}, + "annotations": [ + "etl", + "config-driven", + "lookup-foreach" + ], + "folder": { + "name": "ETL/ConfigDriven" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_mixed_agentic.json b/tests/resources/json/pipelines/pipeline_mixed_agentic.json new file mode 100644 index 0000000..25a304c --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_mixed_agentic.json @@ -0,0 +1,108 @@ +{ + "name": "pipeline_mixed_agentic", + "properties": { + "activities": [ + { + "name": "Copy Source Data", + "type": "Copy", + "dependsOn": [], + "typeProperties": { + "source": { + "type": "BlobSource" + }, + "sink": { + "type": "DeltaSink" + } + } + }, + { + "name": "Run Data Flow", + "type": "ExecuteDataFlow", + "dependsOn": [ + { + "activity": "Copy Source Data", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "dataFlow": { + "referenceName": "df_transform_customers", + "type": "DataFlowReference" + }, + "compute": { + "coreCount": 8, + "computeType": "General" + } + } + }, + { + "name": "Run Stored Proc", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Run Data Flow", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "storedProcedureName": "dbo.sp_process_data", + "storedProcedureParameters": { + "date": { + "type": "DateTime", + "value": "@utcnow()" + } + } + } + }, + { + "name": "Get File Metadata", + "type": "GetMetadata", + "dependsOn": [ + { + "activity": "Run Stored Proc", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "fieldList": ["itemName", "itemType", "lastModified", "size"], + "dataset": { + "referenceName": "ds_source_files", + "type": "DatasetReference" + } + } + }, + { + "name": "Validate Output", + "type": "Validation", + "dependsOn": [ + { + "activity": "Get File Metadata", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": { + "timeout": "7.00:00:00", + "sleep": 10, + "minimumSize": 1024 + } + }, + { + "name": "FancyUnknownType", + "type": "SomeFutureActivity", + "dependsOn": [ + { + "activity": "Validate Output", + "dependencyConditions": ["Succeeded"] + } + ], + "typeProperties": {} + } + ], + "parameters": {}, + "variables": {}, + "annotations": ["mixed", "agentic"], + "folder": { + "name": "ETL/Mixed" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_mixed_deterministic_agentic.json b/tests/resources/json/pipelines/pipeline_mixed_deterministic_agentic.json new file mode 100644 index 0000000..1896fd6 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_mixed_deterministic_agentic.json @@ -0,0 +1,210 @@ +{ + "name": "pipeline_mixed_deterministic_agentic", + "properties": { + "activities": [ + { + "name": "Lookup Source Config", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "0.00:10:00", + "retry": 1, + "retryIntervalInSeconds": 15, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT source_name, connection_string, table_list FROM dbo.integration_config WHERE is_active = 1", + "queryTimeout": "00:05:00" + }, + "dataset": { + "referenceName": "ds_azure_sql_config", + "type": "DatasetReference" + }, + "firstRowOnly": true + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_config", + "type": "DatasetReference" + } + ] + }, + { + "name": "Copy Raw Data", + "type": "Copy", + "dependsOn": [ + { + "activity": "Lookup Source Config", + "dependencyConditions": ["Succeeded"] + } + ], + "policy": { + "timeout": "0.04:00:00", + "retry": 2, + "retryIntervalInSeconds": 60, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "@concat('SELECT * FROM ', activity('Lookup Source Config').output.firstRow.table_list)" + }, + "sink": { + "type": "DeltaSink", + "writeBatchSize": 50000, + "importSettings": { + "type": "DeltaImportCommand" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_generic", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "ds_delta_raw", + "type": "DatasetReference" + } + ] + }, + { + "name": "Execute Data Flow Transform", + "type": "ExecuteDataFlow", + "dependsOn": [ + { + "activity": "Copy Raw Data", + "dependencyConditions": ["Succeeded"] + } + ], + "policy": { + "timeout": "0.02:00:00", + "retry": 1, + "retryIntervalInSeconds": 120, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "dataFlow": { + "referenceName": "df_transform_and_cleanse", + "type": "DataFlowReference", + "parameters": { + "sourceSchema": "@activity('Lookup Source Config').output.firstRow.source_name", + "targetSchema": "curated" + } + }, + "compute": { + "coreCount": 16, + "computeType": "MemoryOptimized" + }, + "staging": { + "linkedService": { + "referenceName": "ls_azure_blob", + "type": "LinkedServiceReference" + }, + "folderPath": "staging/dataflow" + }, + "traceLevel": "Fine", + "runConcurrently": true, + "continueOnError": false + } + }, + { + "name": "Run Quality Notebook", + "type": "DatabricksNotebook", + "dependsOn": [ + { + "activity": "Execute Data Flow Transform", + "dependencyConditions": ["Succeeded"] + } + ], + "policy": { + "timeout": "0.01:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookPath": "/Repos/data-engineering/notebooks/quality_validation", + "baseParameters": { + "sourceTable": "curated.transformed_data", + "thresholdPct": "99.5" + } + }, + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + } + }, + { + "name": "Set Pipeline Result", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Run Quality Notebook", + "dependencyConditions": ["Completed"] + } + ], + "typeProperties": { + "variableName": "pipelineResult", + "value": "@if(equals(activity('Run Quality Notebook').Status, 'Succeeded'), 'PASS', 'FAIL')" + } + }, + { + "name": "Notify Completion", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Set Pipeline Result", + "dependencyConditions": ["Succeeded"] + } + ], + "policy": { + "timeout": "0.00:05:00", + "retry": 1, + "retryIntervalInSeconds": 10, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "https://example.com/webhook/notify", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "text": "@concat('Mixed pipeline completed. Result: ', variables('pipelineResult'), ', Run: ', pipeline().RunId)" + } + } + } + ], + "parameters": { + "environment": { + "type": "String", + "defaultValue": "prod" + } + }, + "variables": { + "pipelineResult": { + "type": "String", + "defaultValue": "UNKNOWN" + } + }, + "annotations": [ + "mixed", + "deterministic-agentic", + "dataflow" + ], + "folder": { + "name": "ETL/Mixed" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_notebook_basic.json b/tests/resources/json/pipelines/pipeline_notebook_basic.json new file mode 100644 index 0000000..b92a6d1 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_notebook_basic.json @@ -0,0 +1,35 @@ +{ + "name": "pipeline_notebook_basic", + "properties": { + "activities": [ + { + "name": "Run Transform Notebook", + "type": "DatabricksNotebook", + "dependsOn": [], + "policy": { + "timeout": "0.02:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookPath": "/Shared/ETL/transform_customers" + }, + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + } + } + ], + "parameters": {}, + "variables": {}, + "annotations": [ + "notebook", + "transform" + ], + "folder": { + "name": "Transform" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_notebook_with_params.json b/tests/resources/json/pipelines/pipeline_notebook_with_params.json new file mode 100644 index 0000000..131d7e7 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_notebook_with_params.json @@ -0,0 +1,68 @@ +{ + "name": "pipeline_notebook_with_params", + "properties": { + "activities": [ + { + "name": "Run Parameterized Notebook", + "type": "DatabricksNotebook", + "dependsOn": [], + "policy": { + "timeout": "0.04:00:00", + "retry": 1, + "retryIntervalInSeconds": 60, + "secureOutput": false, + "secureInput": true + }, + "typeProperties": { + "notebookPath": "/Repos/data-engineering/notebooks/process_transactions", + "baseParameters": { + "environment": "@pipeline().parameters.env", + "processDate": "@pipeline().parameters.processDate", + "catalogName": "@concat(pipeline().parameters.env, '_catalog')", + "schemaName": "transactions", + "batchSize": "50000", + "enableCDC": "true", + "sourceTable": "@pipeline().parameters.sourceTable", + "targetTable": "@pipeline().parameters.targetTable" + } + }, + "linkedServiceName": { + "referenceName": "ls_databricks_new_cluster", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "env": { + "type": "String", + "defaultValue": "dev" + }, + "processDate": { + "type": "String", + "defaultValue": "@utcNow('yyyy-MM-dd')" + }, + "sourceTable": { + "type": "String", + "defaultValue": "raw.transactions" + }, + "targetTable": { + "type": "String", + "defaultValue": "curated.transactions" + } + }, + "variables": { + "notebookOutput": { + "type": "String", + "defaultValue": "" + } + }, + "annotations": [ + "notebook", + "parameterized", + "transactions" + ], + "folder": { + "name": "Transform/Transactions" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_set_variable_chain.json b/tests/resources/json/pipelines/pipeline_set_variable_chain.json new file mode 100644 index 0000000..88301b9 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_set_variable_chain.json @@ -0,0 +1,151 @@ +{ + "name": "pipeline_set_variable_chain", + "properties": { + "activities": [ + { + "name": "Set Environment", + "type": "SetVariable", + "dependsOn": [], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "currentEnv", + "value": "@pipeline().parameters.environment" + } + }, + { + "name": "Set Catalog Name", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set Environment", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "catalogName", + "value": "@concat(variables('currentEnv'), '_catalog')" + } + }, + { + "name": "Set Schema Name", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set Catalog Name", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "schemaName", + "value": "@concat(variables('catalogName'), '.', pipeline().parameters.schemaPrefix, '_data')" + } + }, + { + "name": "Set Full Table Path", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set Schema Name", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "fullTablePath", + "value": "@concat(variables('schemaName'), '.', pipeline().parameters.tableName)" + } + }, + { + "name": "Run Notebook With Resolved Vars", + "type": "DatabricksNotebook", + "dependsOn": [ + { + "activity": "Set Full Table Path", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.02:00:00", + "retry": 1, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookPath": "/Shared/ETL/generic_table_processor", + "baseParameters": { + "fullTablePath": "@variables('fullTablePath')", + "catalogName": "@variables('catalogName')", + "schemaName": "@variables('schemaName')" + } + }, + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "environment": { + "type": "String", + "defaultValue": "dev" + }, + "schemaPrefix": { + "type": "String", + "defaultValue": "sales" + }, + "tableName": { + "type": "String", + "defaultValue": "orders" + } + }, + "variables": { + "currentEnv": { + "type": "String", + "defaultValue": "" + }, + "catalogName": { + "type": "String", + "defaultValue": "" + }, + "schemaName": { + "type": "String", + "defaultValue": "" + }, + "fullTablePath": { + "type": "String", + "defaultValue": "" + } + }, + "annotations": [ + "variables", + "chain", + "dynamic-path" + ], + "folder": { + "name": "Utilities" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_spark_jar_job.json b/tests/resources/json/pipelines/pipeline_spark_jar_job.json new file mode 100644 index 0000000..8d11936 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_spark_jar_job.json @@ -0,0 +1,74 @@ +{ + "name": "pipeline_spark_jar_job", + "properties": { + "activities": [ + { + "name": "Run Spark JAR ETL", + "type": "DatabricksSparkJar", + "dependsOn": [], + "policy": { + "timeout": "0.04:00:00", + "retry": 1, + "retryIntervalInSeconds": 120, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "mainClassName": "com.contoso.etl.DataLakeProcessor", + "parameters": [ + "--input-path", + "@pipeline().parameters.inputPath", + "--output-path", + "@pipeline().parameters.outputPath", + "--process-date", + "@pipeline().parameters.processDate", + "--partition-count", + "200", + "--mode", + "overwrite" + ], + "libraries": [ + { + "jar": "dbfs:/jars/contoso-etl-1.2.3.jar" + }, + { + "jar": "dbfs:/jars/contoso-common-utils-0.9.1.jar" + }, + { + "maven": { + "coordinates": "com.databricks:spark-xml_2.12:0.14.0" + } + } + ] + }, + "linkedServiceName": { + "referenceName": "ls_databricks_new_cluster", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "inputPath": { + "type": "String", + "defaultValue": "abfss://raw@contosodatalake.dfs.core.windows.net/events" + }, + "outputPath": { + "type": "String", + "defaultValue": "abfss://curated@contosodatalake.dfs.core.windows.net/events" + }, + "processDate": { + "type": "String", + "defaultValue": "@utcNow('yyyy-MM-dd')" + } + }, + "variables": {}, + "annotations": [ + "spark-jar", + "etl", + "java" + ], + "folder": { + "name": "Spark/JAR" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_spark_python_job.json b/tests/resources/json/pipelines/pipeline_spark_python_job.json new file mode 100644 index 0000000..0cc9c2c --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_spark_python_job.json @@ -0,0 +1,98 @@ +{ + "name": "pipeline_spark_python_job", + "properties": { + "activities": [ + { + "name": "Run PySpark Data Quality", + "type": "DatabricksSparkPython", + "dependsOn": [], + "policy": { + "timeout": "0.02:00:00", + "retry": 2, + "retryIntervalInSeconds": 60, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "pythonFile": "dbfs:/scripts/data_quality_checks.py", + "parameters": [ + "@pipeline().parameters.catalogName", + "@pipeline().parameters.schemaName", + "@pipeline().parameters.tableName", + "--threshold", + "0.95", + "--output-format", + "json", + "--alert-on-failure", + "true" + ] + }, + "linkedServiceName": { + "referenceName": "ls_databricks_new_cluster", + "type": "LinkedServiceReference" + } + }, + { + "name": "Run PySpark Aggregation", + "type": "DatabricksSparkPython", + "dependsOn": [ + { + "activity": "Run PySpark Data Quality", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.03:00:00", + "retry": 1, + "retryIntervalInSeconds": 120, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "pythonFile": "dbfs:/scripts/aggregate_daily_metrics.py", + "parameters": [ + "--source-table", + "@concat(pipeline().parameters.catalogName, '.', pipeline().parameters.schemaName, '.', pipeline().parameters.tableName)", + "--target-table", + "@concat(pipeline().parameters.catalogName, '.metrics.daily_', pipeline().parameters.tableName)", + "--process-date", + "@pipeline().parameters.processDate" + ] + }, + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "catalogName": { + "type": "String", + "defaultValue": "prod_catalog" + }, + "schemaName": { + "type": "String", + "defaultValue": "sales" + }, + "tableName": { + "type": "String", + "defaultValue": "transactions" + }, + "processDate": { + "type": "String", + "defaultValue": "@utcNow('yyyy-MM-dd')" + } + }, + "variables": {}, + "annotations": [ + "spark-python", + "data-quality", + "aggregation" + ], + "folder": { + "name": "Spark/Python" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_switch_multi_case.json b/tests/resources/json/pipelines/pipeline_switch_multi_case.json new file mode 100644 index 0000000..ae3e74e --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_switch_multi_case.json @@ -0,0 +1,243 @@ +{ + "name": "pipeline_switch_multi_case", + "properties": { + "activities": [ + { + "name": "Set Data Source Type", + "type": "SetVariable", + "dependsOn": [], + "typeProperties": { + "variableName": "sourceType", + "value": "@pipeline().parameters.dataSourceType" + } + }, + { + "name": "Route By Source Type", + "type": "Switch", + "dependsOn": [ + { + "activity": "Set Data Source Type", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "on": { + "value": "@variables('sourceType')", + "type": "Expression" + }, + "cases": [ + { + "value": "SQL", + "activities": [ + { + "name": "Copy From SQL", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.02:00:00", + "retry": 2, + "retryIntervalInSeconds": 30 + }, + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "@pipeline().parameters.sourceQuery" + }, + "sink": { + "type": "DeltaSink", + "writeBatchSize": 10000, + "importSettings": { + "type": "DeltaImportCommand" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "ds_azure_sql_generic", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "ds_delta_generic", + "type": "DatasetReference" + } + ] + } + ] + }, + { + "value": "CSV", + "activities": [ + { + "name": "Copy From CSV", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.01:00:00", + "retry": 1, + "retryIntervalInSeconds": 30 + }, + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DeltaSink", + "importSettings": { + "type": "DeltaImportCommand" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "ds_csv_adls_generic", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "ds_delta_generic", + "type": "DatasetReference" + } + ] + } + ] + }, + { + "value": "Parquet", + "activities": [ + { + "name": "Copy From Parquet", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.01:00:00", + "retry": 1, + "retryIntervalInSeconds": 30 + }, + "typeProperties": { + "source": { + "type": "ParquetSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true + } + }, + "sink": { + "type": "DeltaSink", + "importSettings": { + "type": "DeltaImportCommand" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "ds_parquet_adls_generic", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "ds_delta_generic", + "type": "DatasetReference" + } + ] + } + ] + }, + { + "value": "API", + "activities": [ + { + "name": "Call REST API", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "0.00:30:00", + "retry": 3, + "retryIntervalInSeconds": 60 + }, + "typeProperties": { + "url": "@pipeline().parameters.apiEndpoint", + "method": "GET", + "headers": { + "Authorization": "@concat('Bearer ', pipeline().parameters.apiToken)", + "Accept": "application/json" + } + } + } + ] + } + ], + "defaultActivities": [ + { + "name": "Log Unknown Source Type", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "0.00:05:00", + "retry": 0, + "retryIntervalInSeconds": 30 + }, + "typeProperties": { + "url": "https://example.com/webhook/notify", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "text": "@concat('Unknown data source type: ', variables('sourceType'), ' in pipeline run: ', pipeline().RunId)" + } + } + } + ] + } + } + ], + "parameters": { + "dataSourceType": { + "type": "String", + "defaultValue": "SQL" + }, + "sourceQuery": { + "type": "String", + "defaultValue": "SELECT * FROM dbo.source_table" + }, + "apiEndpoint": { + "type": "String", + "defaultValue": "" + }, + "apiToken": { + "type": "String", + "defaultValue": "" + } + }, + "variables": { + "sourceType": { + "type": "String", + "defaultValue": "" + } + }, + "annotations": [ + "routing", + "switch", + "multi-source" + ], + "folder": { + "name": "ETL/MultiSource" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_wait_between_steps.json b/tests/resources/json/pipelines/pipeline_wait_between_steps.json new file mode 100644 index 0000000..4a7bb7c --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_wait_between_steps.json @@ -0,0 +1,138 @@ +{ + "name": "pipeline_wait_between_steps", + "properties": { + "activities": [ + { + "name": "Trigger External System", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "0.00:10:00", + "retry": 2, + "retryIntervalInSeconds": 15, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "https://api.contoso.com/v1/jobs/trigger", + "method": "POST", + "headers": { + "Content-Type": "application/json", + "Authorization": "@concat('Bearer ', pipeline().parameters.apiToken)" + }, + "body": { + "jobName": "data-refresh", + "parameters": { + "date": "@pipeline().parameters.processDate" + } + } + } + }, + { + "name": "Wait For External Processing", + "type": "Wait", + "dependsOn": [ + { + "activity": "Trigger External System", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "waitTimeInSeconds": 300 + } + }, + { + "name": "Check External System Status", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Wait For External Processing", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.00:05:00", + "retry": 3, + "retryIntervalInSeconds": 10, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "@concat('https://api.contoso.com/v1/jobs/', activity('Trigger External System').output.jobId, '/status')", + "method": "GET", + "headers": { + "Authorization": "@concat('Bearer ', pipeline().parameters.apiToken)" + } + } + }, + { + "name": "Wait Before Retry", + "type": "Wait", + "dependsOn": [ + { + "activity": "Check External System Status", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "waitTimeInSeconds": 60 + } + }, + { + "name": "Run Post Processing Notebook", + "type": "DatabricksNotebook", + "dependsOn": [ + { + "activity": "Wait Before Retry", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.01:00:00", + "retry": 1, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookPath": "/Shared/PostProcessing/reconcile_external_data", + "baseParameters": { + "jobId": "@activity('Trigger External System').output.jobId", + "processDate": "@pipeline().parameters.processDate" + } + }, + "linkedServiceName": { + "referenceName": "ls_databricks_existing_cluster", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "processDate": { + "type": "String", + "defaultValue": "@utcNow('yyyy-MM-dd')" + }, + "apiToken": { + "type": "String", + "defaultValue": "" + } + }, + "variables": {}, + "annotations": [ + "wait", + "external-system", + "polling" + ], + "folder": { + "name": "Integration/External" + } + } +} diff --git a/tests/resources/json/pipelines/pipeline_web_activity_auth.json b/tests/resources/json/pipelines/pipeline_web_activity_auth.json new file mode 100644 index 0000000..a019f67 --- /dev/null +++ b/tests/resources/json/pipelines/pipeline_web_activity_auth.json @@ -0,0 +1,119 @@ +{ + "name": "pipeline_web_activity_auth", + "properties": { + "activities": [ + { + "name": "Get OAuth Token", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "0.00:05:00", + "retry": 3, + "retryIntervalInSeconds": 10, + "secureOutput": true, + "secureInput": true + }, + "typeProperties": { + "url": "https://login.microsoftonline.com/@{pipeline().parameters.tenantId}/oauth2/v2.0/token", + "method": "POST", + "headers": { + "Content-Type": "application/x-www-form-urlencoded" + }, + "body": "grant_type=client_credentials&client_id=@{pipeline().parameters.clientId}&client_secret=@{pipeline().parameters.clientSecret}&scope=https://management.azure.com/.default", + "authentication": { + "type": "MSI", + "resource": "https://management.azure.com/" + }, + "disableCertValidation": false, + "httpRequestTimeout": "00:01:00" + } + }, + { + "name": "Call External API", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Get OAuth Token", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.00:10:00", + "retry": 2, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "@concat(pipeline().parameters.apiBaseUrl, '/api/v2/data/export')", + "method": "GET", + "headers": { + "Authorization": "@concat('Bearer ', activity('Get OAuth Token').output.access_token)", + "Accept": "application/json", + "X-Correlation-Id": "@pipeline().RunId" + }, + "disableCertValidation": false, + "httpRequestTimeout": "00:05:00" + } + }, + { + "name": "Post Status Webhook", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Call External API", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "timeout": "0.00:02:00", + "retry": 1, + "retryIntervalInSeconds": 5, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "url": "https://example.com/webhook/notify", + "method": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": { + "text": "@concat('Pipeline ', pipeline().Pipeline, ' completed. Run ID: ', pipeline().RunId, '. Status: ', if(equals(activity('Call External API').Status, 'Succeeded'), 'SUCCESS', 'FAILURE'))" + } + } + } + ], + "parameters": { + "tenantId": { + "type": "String", + "defaultValue": "00000000-0000-0000-0000-000000000000" + }, + "clientId": { + "type": "String", + "defaultValue": "" + }, + "clientSecret": { + "type": "String", + "defaultValue": "" + }, + "apiBaseUrl": { + "type": "String", + "defaultValue": "https://api.contoso.com" + } + }, + "variables": {}, + "annotations": [ + "web-activity", + "authentication", + "api-call" + ], + "folder": { + "name": "Integration/API" + } + } +} diff --git a/tests/resources/json/pipelines/pl_test_appendvariable_coverage.json b/tests/resources/json/pipelines/pl_test_appendvariable_coverage.json new file mode 100644 index 0000000..69c322b --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_appendvariable_coverage.json @@ -0,0 +1,82 @@ +{ + "etag": "09027fe9-0000-0100-0000-69d82e910000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_appendvariable_coverage", + "name": "pl_test_appendvariable_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "name": "AppendLiteral", + "type": "AppendVariable", + "typeProperties": { + "value": "customers", + "variableName": "tableList" + } + }, + { + "dependsOn": [ + { + "activity": "AppendLiteral", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "AppendExpression", + "type": "AppendVariable", + "typeProperties": { + "value": { + "type": "Expression", + "value": "@concat('dbo.', pipeline().parameters.TableName)" + }, + "variableName": "tableList" + } + }, + { + "dependsOn": [ + { + "activity": "AppendExpression", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "AppendTimestamp", + "type": "AppendVariable", + "typeProperties": { + "value": { + "type": "Expression", + "value": "@concat('Processed at: ', utcNow())" + }, + "variableName": "auditLog" + } + } + ], + "annotations": [ + "test", + "appendvariable", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:56:17Z", + "parameters": { + "TableName": { + "defaultValue": "orders", + "type": "String" + } + }, + "variables": { + "auditLog": { + "defaultValue": [], + "type": "Array" + }, + "tableList": { + "defaultValue": [], + "type": "Array" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_copy_coverage.json b/tests/resources/json/pipelines/pl_test_copy_coverage.json new file mode 100644 index 0000000..38fe9f0 --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_copy_coverage.json @@ -0,0 +1,196 @@ +{ + "etag": "0902dae1-0000-0100-0000-69d82e340000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_copy_coverage", + "name": "pl_test_copy_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "inputs": [ + { + "referenceName": "ds_csv_customers", + "type": "DatasetReference" + } + ], + "name": "CopyBlobToSql", + "outputs": [ + { + "referenceName": "ds_sql_customers", + "type": "DatasetReference" + } + ], + "policy": { + "retry": 3, + "retryIntervalInSeconds": 30, + "secureInput": false, + "secureOutput": false, + "timeout": "1.00:00:00" + }, + "type": "Copy", + "typeProperties": { + "dataIntegrationUnits": 16, + "enableStaging": true, + "logSettings": { + "copyActivityLogSettings": { + "enableReliableLogging": true, + "logLevel": "Warning" + }, + "enableCopyActivityLog": true, + "logLocationSettings": { + "linkedServiceName": { + "referenceName": "ls_blob_storage", + "type": "LinkedServiceReference" + }, + "path": "logs/copy" + } + }, + "parallelCopies": 8, + "sink": { + "disableMetricsCollection": false, + "sqlWriterUseTableLock": false, + "tableOption": "autoCreate", + "type": "AzureSqlSink", + "writeBehavior": "insert" + }, + "source": { + "formatSettings": { + "type": "DelimitedTextReadSettings" + }, + "storeSettings": { + "enablePartitionDiscovery": false, + "recursive": true, + "type": "AzureBlobFSReadSettings", + "wildcardFileName": "*.csv", + "wildcardFolderPath": "data/*" + }, + "type": "DelimitedTextSource" + }, + "stagingSettings": { + "linkedServiceName": { + "referenceName": "ls_blob_storage", + "type": "LinkedServiceReference" + }, + "path": "staging/copy" + }, + "translator": { + "mappings": [ + { + "sink": { + "name": "customer_id", + "type": "Int64" + }, + "source": { + "name": "id", + "type": "Int32" + } + }, + { + "sink": { + "name": "customer_name", + "type": "String" + }, + "source": { + "name": "name", + "type": "String" + } + }, + { + "sink": { + "name": "email_address", + "type": "String" + }, + "source": { + "name": "email", + "type": "String" + } + } + ], + "type": "TabularTranslator" + }, + "validateDataConsistency": true + } + }, + { + "dependsOn": [ + { + "activity": "CopyBlobToSql", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "inputs": [ + { + "referenceName": "ds_sql_orders", + "type": "DatasetReference" + } + ], + "name": "CopySqlToParquet", + "outputs": [ + { + "referenceName": "ds_parquet_orders", + "type": "DatasetReference" + } + ], + "policy": { + "retry": 2, + "retryIntervalInSeconds": 60, + "timeout": "0.12:00:00" + }, + "type": "Copy", + "typeProperties": { + "dataIntegrationUnits": 8, + "enableStaging": false, + "parallelCopies": 4, + "sink": { + "formatSettings": { + "fileNamePrefix": "orders_", + "maxRowsPerFile": 100000, + "type": "ParquetWriteSettings" + }, + "storeSettings": { + "copyBehavior": "PreserveHierarchy", + "maxConcurrentConnections": 10, + "type": "AzureBlobFSWriteSettings" + }, + "type": "ParquetSink" + }, + "source": { + "partitionOption": "DynamicRange", + "partitionSettings": { + "partitionColumnName": "customer_id", + "partitionLowerBound": "1", + "partitionUpperBound": "1000000" + }, + "queryTimeout": "02:00:00", + "sqlReaderQuery": { + "type": "Expression", + "value": "@concat('SELECT * FROM customers WHERE created_date >= ', pipeline().parameters.StartDate, ' AND status = ', pipeline().parameters.FilterStatus, '')" + }, + "type": "AzureSqlSource" + } + } + } + ], + "annotations": [ + "test", + "copy", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:54:44Z", + "parameters": { + "FilterStatus": { + "defaultValue": "active", + "type": "String" + }, + "StartDate": { + "defaultValue": "2024-01-01", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_delete_coverage.json b/tests/resources/json/pipelines/pl_test_delete_coverage.json new file mode 100644 index 0000000..8b46b63 --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_delete_coverage.json @@ -0,0 +1,51 @@ +{ + "etag": "0902e7ed-0000-0100-0000-69d82ec80000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_delete_coverage", + "name": "pl_test_delete_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "name": "DeleteStagingFiles", + "policy": { + "retry": 2, + "retryIntervalInSeconds": 30, + "timeout": "0.01:00:00" + }, + "type": "Delete", + "typeProperties": { + "dataset": { + "referenceName": "ds_blob_staging", + "type": "DatasetReference" + }, + "enableLogging": true, + "logStorageSettings": { + "linkedServiceName": { + "referenceName": "ls_blob_storage", + "type": "LinkedServiceReference" + }, + "path": "logs/delete" + }, + "maxConcurrentConnections": 10, + "recursive": true, + "storeSettings": { + "recursive": true, + "type": "AzureBlobFSReadSettings", + "wildcardFileName": "*.tmp", + "wildcardFolderPath": "staging/*" + } + } + } + ], + "annotations": [ + "test", + "delete", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:57:12Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_executepipeline_coverage.json b/tests/resources/json/pipelines/pl_test_executepipeline_coverage.json new file mode 100644 index 0000000..31b4496 --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_executepipeline_coverage.json @@ -0,0 +1,75 @@ +{ + "etag": "090256ee-0000-0100-0000-69d82ed00000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_executepipeline_coverage", + "name": "pl_test_executepipeline_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "name": "RunChildSync", + "type": "ExecutePipeline", + "typeProperties": { + "parameters": { + "InputPath": { + "type": "Expression", + "value": "@concat('/mnt/data/', pipeline().parameters.DataSource)" + }, + "Mode": { + "type": "Expression", + "value": "@pipeline().parameters.ProcessMode" + }, + "OutputPath": "/mnt/output/processed" + }, + "pipeline": { + "referenceName": "pl_notebook_basic", + "type": "PipelineReference" + }, + "waitOnCompletion": true + } + }, + { + "dependsOn": [ + { + "activity": "RunChildSync", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "RunChildAsync", + "type": "ExecutePipeline", + "typeProperties": { + "parameters": { + "Catalog": "main", + "NotebookPath": "/etl/async_handler" + }, + "pipeline": { + "referenceName": "pl_notebook_with_params", + "type": "PipelineReference" + }, + "waitOnCompletion": false + } + } + ], + "annotations": [ + "test", + "executepipeline", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:57:20Z", + "parameters": { + "DataSource": { + "defaultValue": "customers", + "type": "String" + }, + "ProcessMode": { + "defaultValue": "incremental", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_filter_coverage.json b/tests/resources/json/pipelines/pl_test_filter_coverage.json new file mode 100644 index 0000000..ab43d6a --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_filter_coverage.json @@ -0,0 +1,107 @@ +{ + "etag": "09024aef-0000-0100-0000-69d82edf0000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_filter_coverage", + "name": "pl_test_filter_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "name": "LookupAllTables", + "type": "Lookup", + "typeProperties": { + "dataset": { + "referenceName": "ds_sql_customers", + "type": "DatasetReference" + }, + "firstRowOnly": false, + "source": { + "sqlReaderQuery": "SELECT table_name, row_count, last_modified FROM sys.table_info", + "type": "AzureSqlSource" + } + } + }, + { + "dependsOn": [ + { + "activity": "LookupAllTables", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "FilterLargeTables", + "type": "Filter", + "typeProperties": { + "condition": { + "type": "Expression", + "value": "@greater(item().row_count, 10000)" + }, + "items": { + "type": "Expression", + "value": "@activity('LookupAllTables').output.value" + } + } + }, + { + "dependsOn": [ + { + "activity": "LookupAllTables", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "FilterByName", + "type": "Filter", + "typeProperties": { + "condition": { + "type": "Expression", + "value": "@contains(item().table_name, pipeline().parameters.NameFilter)" + }, + "items": { + "type": "Expression", + "value": "@activity('LookupAllTables').output.value" + } + } + }, + { + "dependsOn": [ + { + "activity": "LookupAllTables", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "FilterEquals", + "type": "Filter", + "typeProperties": { + "condition": { + "type": "Expression", + "value": "@equals(item().status, 'active')" + }, + "items": { + "type": "Expression", + "value": "@activity('LookupAllTables').output.value" + } + } + } + ], + "annotations": [ + "test", + "filter", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:57:35Z", + "parameters": { + "NameFilter": { + "defaultValue": "customer", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_foreach_coverage.json b/tests/resources/json/pipelines/pl_test_foreach_coverage.json new file mode 100644 index 0000000..dcadeb4 --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_foreach_coverage.json @@ -0,0 +1,87 @@ +{ + "etag": "090242e7-0000-0100-0000-69d82e720000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_foreach_coverage", + "name": "pl_test_foreach_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "name": "ForEachTable", + "type": "ForEach", + "typeProperties": { + "activities": [ + { + "dependsOn": [], + "inputs": [ + { + "parameters": { + "TableName": { + "type": "Expression", + "value": "@item().tableName" + } + }, + "referenceName": "ds_sql_orders", + "type": "DatasetReference" + } + ], + "name": "CopyTableData", + "outputs": [ + { + "referenceName": "ds_parquet_orders", + "type": "DatasetReference" + } + ], + "policy": { + "retry": 2, + "retryIntervalInSeconds": 30, + "timeout": "0.04:00:00" + }, + "type": "Copy", + "typeProperties": { + "enableStaging": false, + "sink": { + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "type": "ParquetSink" + }, + "source": { + "sqlReaderQuery": { + "type": "Expression", + "value": "@concat('SELECT * FROM ', item().schema, '.', item().tableName, ' WHERE modified_date >= ''', pipeline().parameters.StartDate, '''')" + }, + "type": "AzureSqlSource" + } + } + } + ], + "batchCount": 5, + "isSequential": true, + "items": { + "type": "Expression", + "value": "@pipeline().parameters.TableList" + } + } + } + ], + "annotations": [ + "test", + "foreach", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:55:46Z", + "parameters": { + "StartDate": { + "defaultValue": "2024-01-01", + "type": "String" + }, + "TableList": { + "type": "Array" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_ifcondition_coverage.json b/tests/resources/json/pipelines/pl_test_ifcondition_coverage.json new file mode 100644 index 0000000..fb8427c --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_ifcondition_coverage.json @@ -0,0 +1,205 @@ +{ + "etag": "09028ce8-0000-0100-0000-69d82e820000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_ifcondition_coverage", + "name": "pl_test_ifcondition_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "name": "CheckEquality", + "type": "IfCondition", + "typeProperties": { + "expression": { + "type": "Expression", + "value": "@equals(pipeline().parameters.Mode, 'full')" + }, + "ifFalseActivities": [ + { + "dependsOn": [], + "inputs": [ + { + "referenceName": "ds_sql_customers", + "type": "DatasetReference" + } + ], + "name": "IncrementalCopy", + "outputs": [ + { + "referenceName": "ds_parquet_orders", + "type": "DatasetReference" + } + ], + "policy": { + "retry": 2, + "timeout": "0.02:00:00" + }, + "type": "Copy", + "typeProperties": { + "enableStaging": false, + "sink": { + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "type": "ParquetSink" + }, + "source": { + "sqlReaderQuery": { + "type": "Expression", + "value": "@concat('SELECT * FROM dbo.customers WHERE modified > ''', pipeline().parameters.WatermarkDate, '''')" + }, + "type": "AzureSqlSource" + } + } + }, + { + "dependsOn": [ + { + "activity": "IncrementalCopy", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "LogIncremental", + "policy": { + "timeout": "0.00:10:00" + }, + "type": "WebActivity", + "typeProperties": { + "body": "{\"event\": \"incremental_complete\"}", + "method": "POST", + "url": "https://api.example.com/log" + } + } + ], + "ifTrueActivities": [ + { + "dependsOn": [], + "inputs": [ + { + "referenceName": "ds_sql_customers", + "type": "DatasetReference" + } + ], + "name": "FullLoadCopy", + "outputs": [ + { + "referenceName": "ds_parquet_orders", + "type": "DatasetReference" + } + ], + "policy": { + "retry": 1, + "timeout": "0.06:00:00" + }, + "type": "Copy", + "typeProperties": { + "enableStaging": false, + "sink": { + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "type": "ParquetSink" + }, + "source": { + "sqlReaderQuery": "SELECT * FROM dbo.customers", + "type": "AzureSqlSource" + } + } + }, + { + "dependsOn": [ + { + "activity": "FullLoadCopy", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "LogFullLoad", + "policy": { + "timeout": "0.00:10:00" + }, + "type": "WebActivity", + "typeProperties": { + "body": "{\"event\": \"full_load_complete\"}", + "method": "POST", + "url": "https://api.example.com/log" + } + } + ] + } + }, + { + "dependsOn": [ + { + "activity": "CheckEquality", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "CheckGreaterThan", + "type": "IfCondition", + "typeProperties": { + "expression": { + "type": "Expression", + "value": "@greater(int(pipeline().parameters.RecordCount), 1000)" + }, + "ifFalseActivities": [ + { + "dependsOn": [], + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + }, + "name": "SmallDatasetNotebook", + "type": "DatabricksNotebook", + "typeProperties": { + "notebookPath": "/etl/small_dataset_handler" + } + } + ], + "ifTrueActivities": [ + { + "dependsOn": [], + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + }, + "name": "LargeDatasetNotebook", + "type": "DatabricksNotebook", + "typeProperties": { + "notebookPath": "/etl/large_dataset_handler" + } + } + ] + } + } + ], + "annotations": [ + "test", + "ifcondition", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:56:02Z", + "parameters": { + "Mode": { + "defaultValue": "incremental", + "type": "String" + }, + "RecordCount": { + "defaultValue": "500", + "type": "String" + }, + "WatermarkDate": { + "defaultValue": "2024-01-01", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_lookup_coverage.json b/tests/resources/json/pipelines/pl_test_lookup_coverage.json new file mode 100644 index 0000000..85a09cf --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_lookup_coverage.json @@ -0,0 +1,107 @@ +{ + "etag": "090242eb-0000-0100-0000-69d82ea50000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_lookup_coverage", + "name": "pl_test_lookup_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "name": "LookupWithQuery", + "policy": { + "retry": 2, + "retryIntervalInSeconds": 30, + "timeout": "0.00:30:00" + }, + "type": "Lookup", + "typeProperties": { + "dataset": { + "parameters": { + "SchemaName": { + "type": "Expression", + "value": "@pipeline().parameters.SchemaName" + } + }, + "referenceName": "ds_sql_customers", + "type": "DatasetReference" + }, + "firstRowOnly": true, + "source": { + "queryTimeout": "00:10:00", + "sqlReaderQuery": { + "type": "Expression", + "value": "@concat('SELECT MAX(modified_date) as watermark FROM ', pipeline().parameters.TableName)" + }, + "type": "AzureSqlSource" + } + } + }, + { + "dependsOn": [ + { + "activity": "LookupWithQuery", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "LookupStoredProc", + "policy": { + "retry": 1, + "timeout": "0.00:15:00" + }, + "type": "Lookup", + "typeProperties": { + "dataset": { + "referenceName": "ds_sql_customers", + "type": "DatasetReference" + }, + "firstRowOnly": false, + "source": { + "sqlReaderStoredProcedureName": "sp_GetTableConfig", + "storedProcedureParameters": { + "environment": { + "type": "String", + "value": { + "type": "Expression", + "value": "@pipeline().parameters.Env" + } + }, + "tableName": { + "type": "String", + "value": { + "type": "Expression", + "value": "@pipeline().parameters.TableName" + } + } + }, + "type": "AzureSqlSource" + } + } + } + ], + "annotations": [ + "test", + "lookup", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:56:37Z", + "parameters": { + "Env": { + "defaultValue": "dev", + "type": "String" + }, + "SchemaName": { + "defaultValue": "dbo", + "type": "String" + }, + "TableName": { + "defaultValue": "dbo.customers", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_notebook_coverage.json b/tests/resources/json/pipelines/pl_test_notebook_coverage.json new file mode 100644 index 0000000..999427f --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_notebook_coverage.json @@ -0,0 +1,103 @@ +{ + "etag": "0902bae2-0000-0100-0000-69d82e3e0000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_notebook_coverage", + "name": "pl_test_notebook_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + }, + "name": "RunNotebookWithAllParams", + "policy": { + "retry": 2, + "retryIntervalInSeconds": 120, + "secureInput": false, + "secureOutput": true, + "timeout": "2.00:00:00" + }, + "type": "DatabricksNotebook", + "typeProperties": { + "baseParameters": { + "batch_size": "10000", + "catalog": { + "type": "Expression", + "value": "@pipeline().parameters.Catalog" + }, + "mode": { + "type": "Expression", + "value": "@if(equals(pipeline().parameters.IsFullLoad, 'true'), 'overwrite', 'append')" + }, + "run_date": { + "type": "Expression", + "value": "@utcNow('yyyy-MM-dd')" + }, + "schema": { + "type": "Expression", + "value": "@pipeline().parameters.Schema" + }, + "table_name": "customers" + }, + "libraries": [ + { + "jar": "dbfs:/jars/my-udf-1.0.jar" + }, + { + "pypi": { + "package": "great-expectations==0.18.0" + } + }, + { + "maven": { + "coordinates": "com.databricks:spark-xml_2.12:0.16.0" + } + }, + { + "pypi": { + "package": "delta-spark==3.0.0" + } + } + ], + "notebookPath": { + "type": "Expression", + "value": "@concat('/Repos/', pipeline().parameters.Environment, '/etl/notebooks/', pipeline().parameters.NotebookName)" + } + } + } + ], + "annotations": [ + "test", + "notebook", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:54:54Z", + "parameters": { + "Catalog": { + "defaultValue": "main", + "type": "String" + }, + "Environment": { + "defaultValue": "dev", + "type": "String" + }, + "IsFullLoad": { + "defaultValue": "false", + "type": "String" + }, + "NotebookName": { + "defaultValue": "process_customers", + "type": "String" + }, + "Schema": { + "defaultValue": "default", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_setvariable_coverage.json b/tests/resources/json/pipelines/pl_test_setvariable_coverage.json new file mode 100644 index 0000000..8132e67 --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_setvariable_coverage.json @@ -0,0 +1,132 @@ +{ + "etag": "090236e9-0000-0100-0000-69d82e8b0000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_setvariable_coverage", + "name": "pl_test_setvariable_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "name": "SetLiteralVar", + "type": "SetVariable", + "typeProperties": { + "value": "started", + "variableName": "processStatus" + } + }, + { + "dependsOn": [ + { + "activity": "SetLiteralVar", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "SetUtcNowVar", + "type": "SetVariable", + "typeProperties": { + "value": { + "type": "Expression", + "value": "@utcNow()" + }, + "variableName": "runTimestamp" + } + }, + { + "dependsOn": [ + { + "activity": "SetUtcNowVar", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "SetParamVar", + "type": "SetVariable", + "typeProperties": { + "value": { + "type": "Expression", + "value": "@pipeline().parameters.Env" + }, + "variableName": "environment" + } + }, + { + "dependsOn": [ + { + "activity": "SetParamVar", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "SetConcatVar", + "type": "SetVariable", + "typeProperties": { + "value": { + "type": "Expression", + "value": "@concat('/mnt/output/', variables('environment'), '/', formatDateTime(utcNow(), 'yyyy/MM/dd'))" + }, + "variableName": "outputPath" + } + }, + { + "dependsOn": [ + { + "activity": "SetConcatVar", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "SetBoolVar", + "type": "SetVariable", + "typeProperties": { + "value": { + "type": "Expression", + "value": "@equals(pipeline().parameters.Env, 'prod')" + }, + "variableName": "isProduction" + } + } + ], + "annotations": [ + "test", + "setvariable", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:56:11Z", + "parameters": { + "Env": { + "defaultValue": "dev", + "type": "String" + } + }, + "variables": { + "environment": { + "defaultValue": "", + "type": "String" + }, + "isProduction": { + "defaultValue": "false", + "type": "String" + }, + "outputPath": { + "defaultValue": "", + "type": "String" + }, + "processStatus": { + "defaultValue": "", + "type": "String" + }, + "runTimestamp": { + "defaultValue": "", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_sparkjar_coverage.json b/tests/resources/json/pipelines/pl_test_sparkjar_coverage.json new file mode 100644 index 0000000..aaaad0f --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_sparkjar_coverage.json @@ -0,0 +1,71 @@ +{ + "etag": "090239e3-0000-0100-0000-69d82e450000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_sparkjar_coverage", + "name": "pl_test_sparkjar_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + }, + "name": "RunSparkJarJob", + "policy": { + "retry": 1, + "retryIntervalInSeconds": 300, + "timeout": "3.00:00:00" + }, + "type": "DatabricksSparkJar", + "typeProperties": { + "libraries": [ + { + "jar": "dbfs:/jars/etl-core-2.0.jar" + }, + { + "jar": "dbfs:/jars/etl-utils-1.5.jar" + }, + { + "maven": { + "coordinates": "org.apache.spark:spark-avro_2.12:3.5.0" + } + } + ], + "mainClassName": { + "type": "Expression", + "value": "@concat('com.company.etl.', pipeline().parameters.MainClass)" + }, + "parameters": [ + "--input", + "/mnt/raw/data.parquet", + "--output", + "/mnt/processed/output", + "--date", + { + "type": "Expression", + "value": "@formatDateTime(utcNow(), 'yyyy-MM-dd')" + }, + "--mode", + "overwrite" + ] + } + } + ], + "annotations": [ + "test", + "sparkjar", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:55:01Z", + "parameters": { + "MainClass": { + "defaultValue": "CustomerProcessor", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_sparkpython_coverage.json b/tests/resources/json/pipelines/pl_test_sparkpython_coverage.json new file mode 100644 index 0000000..f7e9091 --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_sparkpython_coverage.json @@ -0,0 +1,78 @@ +{ + "etag": "090277e4-0000-0100-0000-69d82e500000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_sparkpython_coverage", + "name": "pl_test_sparkpython_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + }, + "name": "RunSparkPythonScript", + "policy": { + "retry": 2, + "retryIntervalInSeconds": 60, + "timeout": "1.12:00:00" + }, + "type": "DatabricksSparkPython", + "typeProperties": { + "libraries": [ + { + "pypi": { + "package": "pandas==2.0.0" + } + }, + { + "pypi": { + "package": "numpy==1.24.0" + } + }, + { + "jar": "dbfs:/jars/custom-udf-1.0.jar" + } + ], + "parameters": [ + "--config", + { + "type": "Expression", + "value": "@concat('/mnt/config/', pipeline().parameters.Environment, '.json')" + }, + "--date", + { + "type": "Expression", + "value": "@formatDateTime(utcNow(), 'yyyy-MM-dd')" + }, + "--verbose", + "true" + ], + "pythonFile": { + "type": "Expression", + "value": "@concat('dbfs:/scripts/', pipeline().parameters.ScriptName, '.py')" + } + } + } + ], + "annotations": [ + "test", + "sparkpython", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:55:12Z", + "parameters": { + "Environment": { + "defaultValue": "dev", + "type": "String" + }, + "ScriptName": { + "defaultValue": "data_processor", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_switch_coverage.json b/tests/resources/json/pipelines/pl_test_switch_coverage.json new file mode 100644 index 0000000..429ab0d --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_switch_coverage.json @@ -0,0 +1,126 @@ +{ + "etag": "0902cfea-0000-0100-0000-69d82e9d0000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_switch_coverage", + "name": "pl_test_switch_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "name": "RouteByEnvironment", + "type": "Switch", + "typeProperties": { + "cases": [ + { + "activities": [ + { + "dependsOn": [], + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + }, + "name": "DevNotebook", + "type": "DatabricksNotebook", + "typeProperties": { + "notebookPath": "/etl/dev_handler" + } + } + ], + "value": "dev" + }, + { + "activities": [ + { + "dependsOn": [], + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + }, + "name": "StagingNotebook", + "type": "DatabricksNotebook", + "typeProperties": { + "notebookPath": "/etl/staging_handler" + } + } + ], + "value": "staging" + }, + { + "activities": [ + { + "dependsOn": [], + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + }, + "name": "ProdNotebook", + "type": "DatabricksNotebook", + "typeProperties": { + "notebookPath": "/etl/prod_handler" + } + }, + { + "dependsOn": [ + { + "activity": "ProdNotebook", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "ProdAuditLog", + "type": "WebActivity", + "typeProperties": { + "body": "{\"env\": \"prod\"}", + "method": "POST", + "url": "https://audit.example.com/log" + } + } + ], + "value": "prod" + } + ], + "defaultActivities": [ + { + "dependsOn": [], + "linkedServiceName": { + "referenceName": "ls_databricks", + "type": "LinkedServiceReference" + }, + "name": "UnknownEnvNotebook", + "type": "DatabricksNotebook", + "typeProperties": { + "notebookPath": "/etl/unknown_handler" + } + } + ], + "on": { + "type": "Expression", + "value": "@variables('targetEnv')" + } + } + } + ], + "annotations": [ + "test", + "switch", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:56:29Z", + "parameters": { + "Environment": { + "defaultValue": "dev", + "type": "String" + } + }, + "variables": { + "targetEnv": { + "defaultValue": "dev", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_wait_coverage.json b/tests/resources/json/pipelines/pl_test_wait_coverage.json new file mode 100644 index 0000000..58d3834 --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_wait_coverage.json @@ -0,0 +1,51 @@ +{ + "etag": "090240ef-0000-0100-0000-69d82edf0000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_wait_coverage", + "name": "pl_test_wait_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "name": "WaitLiteral", + "type": "Wait", + "typeProperties": { + "waitTimeInSeconds": 60 + } + }, + { + "dependsOn": [ + { + "activity": "WaitLiteral", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "WaitExpression", + "type": "Wait", + "typeProperties": { + "waitTimeInSeconds": { + "type": "Expression", + "value": "@int(pipeline().parameters.WaitSeconds)" + } + } + } + ], + "annotations": [ + "test", + "wait", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:57:35Z", + "parameters": { + "WaitSeconds": { + "defaultValue": "30", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/pipelines/pl_test_webactivity_coverage.json b/tests/resources/json/pipelines/pl_test_webactivity_coverage.json new file mode 100644 index 0000000..342abc3 --- /dev/null +++ b/tests/resources/json/pipelines/pl_test_webactivity_coverage.json @@ -0,0 +1,131 @@ +{ + "etag": "09026bed-0000-0100-0000-69d82ec30000", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/flowx-rg/providers/Microsoft.DataFactory/factories/flowx-adf/pipelines/pl_test_webactivity_coverage", + "name": "pl_test_webactivity_coverage", + "properties": { + "activities": [ + { + "dependsOn": [], + "name": "GetApiData", + "policy": { + "retry": 3, + "retryIntervalInSeconds": 10, + "timeout": "0.00:30:00" + }, + "type": "WebActivity", + "typeProperties": { + "authentication": { + "password": { + "type": "SecureString", + "value": "**********" + }, + "type": "Basic", + "username": "api_user" + }, + "disableCertValidation": false, + "headers": { + "Accept": "application/json", + "X-Environment": { + "type": "Expression", + "value": "@pipeline().parameters.Env" + }, + "X-Request-Id": { + "type": "Expression", + "value": "@pipeline().RunId" + } + }, + "httpRequestTimeout": "00:05:00", + "method": "GET", + "url": { + "type": "Expression", + "value": "@concat('https://api.example.com/v1/', pipeline().parameters.Endpoint, '?date=', formatDateTime(utcNow(), 'yyyy-MM-dd'))" + } + } + }, + { + "dependsOn": [ + { + "activity": "GetApiData", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "PostResults", + "policy": { + "retry": 2, + "timeout": "0.00:10:00" + }, + "type": "WebActivity", + "typeProperties": { + "body": { + "type": "Expression", + "value": "@json(concat('{\"runId\": \"', pipeline().RunId, '\", \"status\": \"complete\", \"timestamp\": \"', utcNow(), '\"}'))" + }, + "headers": { + "Authorization": { + "type": "Expression", + "value": "@concat('Bearer ', pipeline().parameters.ApiToken)" + }, + "Content-Type": "application/json" + }, + "method": "POST", + "url": "https://api.example.com/v1/results" + } + }, + { + "dependsOn": [ + { + "activity": "PostResults", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "name": "UpdateStatus", + "policy": { + "timeout": "0.00:05:00" + }, + "type": "WebActivity", + "typeProperties": { + "body": { + "completedAt": "2024-01-01T00:00:00Z", + "status": "completed" + }, + "headers": { + "Content-Type": "application/json" + }, + "method": "PUT", + "url": { + "type": "Expression", + "value": "@concat('https://api.example.com/v1/status/', pipeline().RunId)" + } + } + } + ], + "annotations": [ + "test", + "webactivity", + "coverage" + ], + "folder": { + "name": "test_coverage" + }, + "lastPublishTime": "2026-04-09T22:57:07Z", + "parameters": { + "ApiToken": { + "defaultValue": "", + "type": "String" + }, + "Endpoint": { + "defaultValue": "data", + "type": "String" + }, + "Env": { + "defaultValue": "dev", + "type": "String" + } + } + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/tests/resources/json/triggers/tr_daily_schedule.json b/tests/resources/json/triggers/tr_daily_schedule.json new file mode 100644 index 0000000..536e450 --- /dev/null +++ b/tests/resources/json/triggers/tr_daily_schedule.json @@ -0,0 +1,22 @@ +{ + "name": "tr_daily_schedule", + "properties": { + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "startTime": "2024-01-01T06:00:00Z", + "timeZone": "UTC" + } + }, + "pipelines": [ + { + "pipelineReference": { + "referenceName": "pipeline_copy_csv_to_delta", + "type": "PipelineReference" + } + } + ] + } +} diff --git a/tests/resources/json/triggers/trigger_blob_event.json b/tests/resources/json/triggers/trigger_blob_event.json new file mode 100644 index 0000000..0bd369b --- /dev/null +++ b/tests/resources/json/triggers/trigger_blob_event.json @@ -0,0 +1,33 @@ +{ + "name": "tr_blob_arrival_event", + "properties": { + "type": "BlobEventsTrigger", + "typeProperties": { + "blobPathBeginsWith": "/raw/inbound/", + "blobPathEndsWith": ".csv", + "ignoreEmptyBlobs": true, + "events": [ + "Microsoft.Storage.BlobCreated" + ], + "scope": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-data-platform/providers/Microsoft.Storage/storageAccounts/contosodatalake" + }, + "pipelines": [ + { + "pipelineReference": { + "referenceName": "pipeline_copy_csv_to_delta", + "type": "PipelineReference" + }, + "parameters": { + "sourceFolderPath": "@triggerBody().folderPath", + "triggerDate": "@trigger().startTime" + } + } + ], + "runtimeState": "Started", + "annotations": [ + "blob-event", + "file-arrival", + "event-driven" + ] + } +} diff --git a/tests/resources/json/triggers/trigger_schedule.json b/tests/resources/json/triggers/trigger_schedule.json new file mode 100644 index 0000000..21296bc --- /dev/null +++ b/tests/resources/json/triggers/trigger_schedule.json @@ -0,0 +1,58 @@ +{ + "name": "tr_daily_etl_schedule", + "properties": { + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "startTime": "2024-01-01T06:00:00Z", + "endTime": "2025-12-31T23:59:59Z", + "timeZone": "Central Standard Time", + "schedule": { + "hours": [6], + "minutes": [0], + "weekDays": [ + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday" + ] + } + } + }, + "pipelines": [ + { + "pipelineReference": { + "referenceName": "pipeline_complex_etl", + "type": "PipelineReference" + }, + "parameters": { + "pipelineGroup": "sales", + "processDate": "@trigger().scheduledTime", + "environment": "prod", + "enableMerge": true + } + }, + { + "pipelineReference": { + "referenceName": "pipeline_complex_etl", + "type": "PipelineReference" + }, + "parameters": { + "pipelineGroup": "finance", + "processDate": "@trigger().scheduledTime", + "environment": "prod", + "enableMerge": true + } + } + ], + "runtimeState": "Started", + "annotations": [ + "schedule", + "daily", + "production" + ] + } +} diff --git a/tests/resources/json/triggers/trigger_tumbling_window.json b/tests/resources/json/triggers/trigger_tumbling_window.json new file mode 100644 index 0000000..7f88d30 --- /dev/null +++ b/tests/resources/json/triggers/trigger_tumbling_window.json @@ -0,0 +1,66 @@ +{ + "name": "tr_hourly_tumbling_window", + "properties": { + "type": "TumblingWindowTrigger", + "typeProperties": { + "frequency": "Hour", + "interval": 1, + "startTime": "2024-01-01T00:00:00Z", + "endTime": "2025-12-31T23:59:59Z", + "delay": "00:15:00", + "maxConcurrency": 5, + "retryPolicy": { + "count": 3, + "intervalInSeconds": 300 + }, + "dependsOn": [ + { + "type": "TumblingWindowTriggerDependencyReference", + "referenceTrigger": { + "referenceName": "tr_upstream_data_ready", + "type": "TriggerReference" + }, + "offset": "-01:00:00", + "size": "01:00:00" + }, + { + "type": "SelfDependencyTumblingWindowTriggerReference", + "offset": "-01:00:00", + "size": "01:00:00" + } + ] + }, + "pipeline": { + "pipelineReference": { + "referenceName": "pipeline_copy_parquet_to_delta", + "type": "PipelineReference" + }, + "parameters": { + "windowStart": "@trigger().outputs.windowStartTime", + "windowEnd": "@trigger().outputs.windowEndTime", + "sourcePath": "@concat('events/', formatDateTime(trigger().outputs.windowStartTime, 'yyyy/MM/dd/HH'))", + "eventDate": "@formatDateTime(trigger().outputs.windowStartTime, 'yyyy-MM-dd')" + } + }, + "pipelines": [ + { + "pipelineReference": { + "referenceName": "pipeline_copy_parquet_to_delta", + "type": "PipelineReference" + }, + "parameters": { + "windowStart": "@trigger().outputs.windowStartTime", + "windowEnd": "@trigger().outputs.windowEndTime", + "sourcePath": "@concat('events/', formatDateTime(trigger().outputs.windowStartTime, 'yyyy/MM/dd/HH'))", + "eventDate": "@formatDateTime(trigger().outputs.windowStartTime, 'yyyy-MM-dd')" + } + } + ], + "runtimeState": "Started", + "annotations": [ + "tumbling-window", + "hourly", + "incremental" + ] + } +} diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 0000000..da0f62e --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1 @@ +# Unit test configuration — shared fixtures are in tests/conftest.py. diff --git a/tests/unit/test_adf_loader.py b/tests/unit/test_adf_loader.py new file mode 100644 index 0000000..678c24a --- /dev/null +++ b/tests/unit/test_adf_loader.py @@ -0,0 +1,304 @@ +"""Unit tests for the ADF parser (adf_loader.py).""" + +from __future__ import annotations + +from flowx.models.adf_ast import ( + AdfDefinitions, + TranslationStrategy, +) +from flowx.parser.adf_loader import ( + AGENTIC_TYPES, + DETERMINISTIC_TYPES, + _normalize_arm, + _parse_pipeline_json, + build_inventory, + classify_activity, + load_adf_definitions, +) + +# --------------------------------------------------------------------------- +# load_adf_definitions +# --------------------------------------------------------------------------- + + +class TestLoadDefinitions: + def test_load_definitions_from_fixtures(self, fixtures_dir): + """All fixture pipelines load without error and the pipeline count matches.""" + defs = load_adf_definitions(fixtures_dir) + assert isinstance(defs, AdfDefinitions) + pipeline_names = {p.name for p in defs.pipelines} + # We should have at least the core fixtures + assert len(defs.pipelines) >= 8 + assert "pipeline_copy_csv_to_delta" in pipeline_names + assert "pipeline_notebook_basic" in pipeline_names + assert "pipeline_complex_etl" in pipeline_names + assert "pipeline_foreach_switch" in pipeline_names + assert "pipeline_all_activity_types" in pipeline_names + assert "pipeline_mixed_agentic" in pipeline_names + + def test_load_definitions_loads_datasets(self, fixtures_dir): + defs = load_adf_definitions(fixtures_dir) + assert "ds_csv_adls_customers" in defs.datasets + assert "ds_delta_customers" in defs.datasets + + def test_load_definitions_loads_linked_services(self, fixtures_dir): + defs = load_adf_definitions(fixtures_dir) + assert "ls_databricks_existing_cluster" in defs.linked_services + assert "ls_databricks_new_cluster" in defs.linked_services + + def test_load_definitions_loads_triggers(self, fixtures_dir): + defs = load_adf_definitions(fixtures_dir) + assert len(defs.triggers) >= 1 + trigger_names = {t.name for t in defs.triggers} + assert "tr_daily_schedule" in trigger_names + + +# --------------------------------------------------------------------------- +# classify_activity +# --------------------------------------------------------------------------- + + +class TestClassifyActivity: + def test_classify_deterministic_types(self): + """All 16 deterministic types are classified correctly.""" + expected = { + "Copy", + "DatabricksNotebook", + "DatabricksSparkJar", + "DatabricksSparkPython", + "ForEach", + "IfCondition", + "SetVariable", + "Switch", + "Lookup", + "WebActivity", + "Delete", + "ExecutePipeline", + "DatabricksJob", + "Wait", + "Filter", + "AppendVariable", + } + assert DETERMINISTIC_TYPES == expected + + for atype in expected: + strategy, skill = classify_activity(atype) + assert strategy is TranslationStrategy.DETERMINISTIC, f"{atype} should be DETERMINISTIC" + assert skill is None, f"{atype} should have no agentic skill" + + def test_classify_agentic_types(self): + """All agentic types are classified with correct skill names.""" + for atype, expected_skill in AGENTIC_TYPES.items(): + strategy, skill = classify_activity(atype) + assert strategy is TranslationStrategy.AGENTIC, f"{atype} should be AGENTIC" + assert skill == expected_skill, f"{atype} skill should be {expected_skill}" + + def test_classify_unknown_types(self): + """Unknown activity types are classified as UNSUPPORTED.""" + for unknown_type in ("Bogus", "SomeFutureActivity", "MagicTransform", ""): + strategy, skill = classify_activity(unknown_type) + assert strategy is TranslationStrategy.UNSUPPORTED + assert skill is None + + +# --------------------------------------------------------------------------- +# build_inventory +# --------------------------------------------------------------------------- + + +class TestBuildInventory: + def test_build_inventory_counts(self, adf_definitions): + """Inventory has correct total and per-strategy counts.""" + inv = build_inventory(adf_definitions) + total = inv.deterministic_count + inv.agentic_count + inv.unsupported_count + assert total == len(inv.items) + assert inv.pipeline_count == len(adf_definitions.pipelines) + # There should be at least some deterministic items + assert inv.deterministic_count > 0 + + def test_build_inventory_has_pipeline_names(self, adf_definitions): + """Every inventory item references a valid pipeline name.""" + inv = build_inventory(adf_definitions) + pipeline_names = {p.name for p in adf_definitions.pipelines} + for item in inv.items: + assert item.pipeline_name in pipeline_names + + def test_build_inventory_agentic_items_have_skills(self, adf_definitions): + """Agentic inventory items have a non-None skill.""" + inv = build_inventory(adf_definitions) + for item in inv.items: + if item.strategy is TranslationStrategy.AGENTIC: + assert item.agentic_skill is not None + + def test_build_inventory_deterministic_items_no_skill(self, adf_definitions): + """Deterministic inventory items have no agentic skill.""" + inv = build_inventory(adf_definitions) + for item in inv.items: + if item.strategy is TranslationStrategy.DETERMINISTIC: + assert item.agentic_skill is None + + +# --------------------------------------------------------------------------- +# Pipeline parsing details +# --------------------------------------------------------------------------- + + +class TestParsePipeline: + def test_parse_pipeline_with_parameters(self): + """Parameters are extracted correctly from pipeline JSON.""" + data = { + "name": "test_pipeline", + "properties": { + "activities": [], + "parameters": { + "env": {"type": "String", "defaultValue": "dev"}, + "runDate": {"type": "String"}, + }, + }, + } + pipeline = _parse_pipeline_json(data) + assert pipeline.name == "test_pipeline" + assert pipeline.parameters is not None + assert "env" in pipeline.parameters + assert pipeline.parameters["env"].type == "String" + assert pipeline.parameters["env"].default_value == "dev" + assert "runDate" in pipeline.parameters + assert pipeline.parameters["runDate"].default_value is None + + def test_parse_activity_with_dependencies(self): + """depends_on is parsed correctly from activity JSON.""" + data = { + "name": "dep_pipeline", + "properties": { + "activities": [ + {"name": "A", "type": "Wait", "typeProperties": {"waitTimeInSeconds": 1}}, + { + "name": "B", + "type": "Wait", + "dependsOn": [ + {"activity": "A", "dependencyConditions": ["Succeeded"]}, + ], + "typeProperties": {"waitTimeInSeconds": 1}, + }, + { + "name": "C", + "type": "Wait", + "dependsOn": [ + {"activity": "A", "dependencyConditions": ["Failed"]}, + {"activity": "B", "dependencyConditions": ["Completed"]}, + ], + "typeProperties": {"waitTimeInSeconds": 1}, + }, + ], + }, + } + pipeline = _parse_pipeline_json(data) + activities_by_name = {a.name: a for a in pipeline.activities} + + assert activities_by_name["A"].depends_on is None or len(activities_by_name["A"].depends_on) == 0 + assert len(activities_by_name["B"].depends_on) == 1 + assert activities_by_name["B"].depends_on[0].activity == "A" + assert activities_by_name["B"].depends_on[0].dependency_conditions == ["Succeeded"] + + assert len(activities_by_name["C"].depends_on) == 2 + assert activities_by_name["C"].depends_on[0].dependency_conditions == ["Failed"] + assert activities_by_name["C"].depends_on[1].dependency_conditions == ["Completed"] + + def test_parse_activity_with_policy(self): + """Timeout and retry are parsed from the activity policy.""" + data = { + "name": "policy_pipeline", + "properties": { + "activities": [ + { + "name": "Act1", + "type": "Copy", + "policy": { + "timeout": "0.12:00:00", + "retry": 3, + "retryIntervalInSeconds": 60, + "secureInput": True, + "secureOutput": False, + }, + "typeProperties": {"source": {}, "sink": {}}, + } + ], + }, + } + pipeline = _parse_pipeline_json(data) + act = pipeline.activities[0] + assert act.policy is not None + assert act.policy.timeout == "0.12:00:00" + assert act.policy.retry == 3 + assert act.policy.retry_interval_in_seconds == 60 + assert act.policy.secure_input is True + assert act.policy.secure_output is False + + def test_parse_pipeline_with_variables(self): + """Variables are extracted correctly.""" + data = { + "name": "var_pipeline", + "properties": { + "activities": [], + "variables": { + "status": {"type": "String", "defaultValue": "pending"}, + "counter": {"type": "Int", "defaultValue": 0}, + }, + }, + } + pipeline = _parse_pipeline_json(data) + assert pipeline.variables is not None + assert "status" in pipeline.variables + assert pipeline.variables["status"].default_value == "pending" + assert pipeline.variables["counter"].default_value == 0 + + def test_parse_pipeline_annotations_and_folder(self): + """Annotations and folder are parsed correctly.""" + data = { + "name": "annotated", + "properties": { + "activities": [], + "annotations": ["etl", "daily"], + "folder": {"name": "ETL/Ingestion"}, + }, + } + pipeline = _parse_pipeline_json(data) + assert pipeline.annotations == ["etl", "daily"] + assert pipeline.folder == "ETL/Ingestion" + + +# --------------------------------------------------------------------------- +# ARM template normalization +# --------------------------------------------------------------------------- + + +class TestNormalizeArm: + def test_normalize_arm_template(self): + """ARM template wrapper is unwrapped to the inner pipeline.""" + arm_data = { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "resources": [ + { + "type": "Microsoft.DataFactory/factories/pipelines", + "name": "[concat(parameters('factoryName'), '/MyPipeline')]", + "properties": { + "activities": [ + { + "name": "DoStuff", + "type": "Copy", + "typeProperties": {"source": {}, "sink": {}}, + } + ], + }, + } + ], + } + result = _normalize_arm(arm_data) + assert result["name"] == "MyPipeline" + assert "activities" in result["properties"] + + def test_normalize_arm_passthrough(self): + """Non-ARM data is returned unchanged.""" + data = {"name": "simple", "properties": {"activities": []}} + result = _normalize_arm(data) + assert result is data diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py new file mode 100644 index 0000000..605dd89 --- /dev/null +++ b/tests/unit/test_bundler.py @@ -0,0 +1,312 @@ +"""Unit tests for the DAB bundle writer.""" + +from __future__ import annotations + +import yaml + +from flowx.bundler.dab_writer import write_bundle +from flowx.models.dab import SecretInstruction, SetupTask +from flowx.models.ir import ( + CopyActivity, + NotebookActivity, + Pipeline, + WaitActivity, +) +from flowx.preparer.workflow_preparer import PreparedWorkflow, prepare_workflow + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _simple_workflow(name: str = "test_workflow") -> PreparedWorkflow: + """Build a minimal PreparedWorkflow with a couple of tasks.""" + pipeline = Pipeline( + name=name, + tasks=[ + NotebookActivity( + name="Run NB", + task_key="run_nb", + notebook_path="/Shared/ETL/transform", + base_parameters={"env": "dev"}, + ), + WaitActivity( + name="Pause", + task_key="pause", + wait_time_seconds=10, + ), + CopyActivity( + name="Copy Data", + task_key="copy_data", + source_type="BlobSource", + sink_type="DeltaSink", + ), + ], + ) + return prepare_workflow(pipeline) + + +def _workflow_with_secrets(name: str = "secret_workflow") -> PreparedWorkflow: + """Build a workflow that includes secret instructions and setup tasks.""" + pipeline = Pipeline( + name=name, + tasks=[ + CopyActivity( + name="Copy SQL", + task_key="copy_sql", + source_type="AzureSqlSource", + sink_type="DeltaSink", + ), + ], + ) + wf = prepare_workflow(pipeline) + # Ensure the copy preparer added secrets + return wf + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestWriteBundle: + def test_databricks_yml_exists(self, tmp_path): + """write_bundle creates a databricks.yml file.""" + wf = _simple_workflow() + write_bundle(wf, tmp_path) + assert (tmp_path / "databricks.yml").exists() + + def test_databricks_yml_structure(self, tmp_path): + """databricks.yml has the expected top-level keys.""" + wf = _simple_workflow("my_pipeline") + write_bundle(wf, tmp_path, catalog="my_catalog", schema="my_schema") + content = yaml.safe_load((tmp_path / "databricks.yml").read_text()) + assert "bundle" in content + assert content["bundle"]["name"] == "my_pipeline" + assert "variables" in content + assert content["variables"]["catalog"]["default"] == "my_catalog" + assert content["variables"]["schema"]["default"] == "my_schema" + assert "include" in content + assert "resources/*.yml" in content["include"] + assert "targets" in content + assert "dev" in content["targets"] + assert "prod" in content["targets"] + + def test_job_resource_yml_exists(self, tmp_path): + """A job resource YAML is created under resources/.""" + wf = _simple_workflow("my_job") + write_bundle(wf, tmp_path) + resource_files = list((tmp_path / "resources").glob("*.yml")) + assert len(resource_files) >= 1 + + def test_job_resource_yml_structure(self, tmp_path): + """Job resource YAML has correct resources.jobs structure.""" + wf = _simple_workflow("my_job") + write_bundle(wf, tmp_path) + resource_file = list((tmp_path / "resources").glob("*.yml"))[0] + content = yaml.safe_load(resource_file.read_text()) + assert "resources" in content + assert "jobs" in content["resources"] + # There should be exactly one job + jobs = content["resources"]["jobs"] + assert len(jobs) == 1 + job_key = list(jobs.keys())[0] + job = jobs[job_key] + assert "name" in job + assert "tasks" in job + assert len(job["tasks"]) == 3 # NB + Wait + Copy + + def test_job_resource_task_keys_unique(self, tmp_path): + """All task keys within the job resource are unique.""" + wf = _simple_workflow() + write_bundle(wf, tmp_path) + resource_file = list((tmp_path / "resources").glob("*.yml"))[0] + content = yaml.safe_load(resource_file.read_text()) + job = list(content["resources"]["jobs"].values())[0] + task_keys = [t["task_key"] for t in job["tasks"]] + assert len(task_keys) == len(set(task_keys)) + + def test_notebooks_written(self, tmp_path): + """Generated notebooks are written to src/notebooks/.""" + wf = _simple_workflow() + write_bundle(wf, tmp_path) + notebooks_dir = tmp_path / "src" / "notebooks" + assert notebooks_dir.exists() + notebook_files = list(notebooks_dir.glob("*.py")) + assert len(notebook_files) >= 1 + + def test_notebook_content_not_empty(self, tmp_path): + """Every generated notebook has non-empty content.""" + wf = _simple_workflow() + write_bundle(wf, tmp_path) + for nb_file in (tmp_path / "src" / "notebooks").glob("*.py"): + content = nb_file.read_text() + assert len(content) > 0, f"Notebook {nb_file.name} is empty" + + def test_setup_notebooks_for_secrets(self, tmp_path): + """Setup notebooks are created when secrets are present.""" + wf = _workflow_with_secrets() + write_bundle(wf, tmp_path) + setup_dir = tmp_path / "src" / "setup" + if wf.secrets: + assert setup_dir.exists() + setup_files = list(setup_dir.glob("*.py")) + assert len(setup_files) >= 1 + # Check content mentions secret scope + secrets_nb = setup_dir / "create_secrets.py" + if secrets_nb.exists(): + content = secrets_nb.read_text() + assert "createScope" in content + + def test_write_bundle_returns_created_files(self, tmp_path): + """write_bundle returns a list of all created file paths.""" + wf = _simple_workflow() + created = write_bundle(wf, tmp_path) + assert isinstance(created, list) + assert len(created) > 0 + for path in created: + assert path.exists(), f"Created file {path} does not exist" + + def test_bundle_name_override(self, tmp_path): + """Custom bundle name overrides the workflow name.""" + wf = _simple_workflow("original_name") + write_bundle(wf, tmp_path, bundle_name="custom_bundle") + content = yaml.safe_load((tmp_path / "databricks.yml").read_text()) + assert content["bundle"]["name"] == "custom_bundle" + + def test_yaml_is_parseable(self, tmp_path): + """All generated YAML files are valid YAML.""" + wf = _simple_workflow() + write_bundle(wf, tmp_path) + for yml_file in tmp_path.rglob("*.yml"): + content = yaml.safe_load(yml_file.read_text()) + assert content is not None, f"YAML file {yml_file} parsed as None" + + def test_module_state_does_not_leak_across_calls(self, tmp_path): + """Successive write_bundle calls don't accumulate warnings or cross-bundle vars. + + Regression: ``_bundle_warnings`` and ``_cross_bundle_variables`` used to + be cleared only by ``main()`` (the CLI), so library callers iterating + over multiple workflows leaked state from one bundle into the next. + """ + from flowx.bundler.dab_writer import _bundle_warnings, _cross_bundle_variables + + first_dir = tmp_path / "first" + second_dir = tmp_path / "second" + first_dir.mkdir() + second_dir.mkdir() + + _bundle_warnings.append("- **stale_task**: stale warning from a prior bundle") + _cross_bundle_variables["stale_var"] = "stale-pipeline" + + write_bundle(_simple_workflow("first"), first_dir) + write_bundle(_simple_workflow("second"), second_dir) + + first_yml = yaml.safe_load((first_dir / "databricks.yml").read_text()) + second_yml = yaml.safe_load((second_dir / "databricks.yml").read_text()) + assert "stale_var" not in (first_yml.get("variables") or {}) + assert "stale_var" not in (second_yml.get("variables") or {}) + # No WARNINGS.md should appear when the workflow itself produces no warnings. + assert not (first_dir / "WARNINGS.md").exists() + assert not (second_dir / "WARNINGS.md").exists() + + def test_load_report_handles_aggregated_translations_format(self, tmp_path): + """``_load_report`` accepts the multi-pipeline aggregated report. + + Regression: an earlier collapse refactor left this branch calling a + deleted ``_placeholder_notebook`` helper, so any user passing the + documented ``translation_report.json`` aggregated format would have + hit ``NameError`` the first time a notebook task was emitted. + """ + import json + + from flowx.bundler.dab_writer import _load_report + + report = { + "translations": [ + { + "pipeline": "agg_pipeline", + "status": "translated", + "ir": { + "type": "WaitActivity", + "name": "Pause", + "task_key": "pause", + "wait_time_seconds": 5, + }, + }, + { + "pipeline": "agg_pipeline", + "status": "translated", + "ir": { + "type": "NotebookActivity", + "name": "Run NB", + "task_key": "run_nb", + "notebook_path": "/Shared/etl/run", + }, + }, + { + "pipeline": "agg_pipeline", + "status": "skipped", + "ir": {}, + }, + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + + workflows = _load_report(report_path) + assert len(workflows) == 1 + assert workflows[0].name == "agg_pipeline" + task_keys = {task["task_key"] for task in workflows[0].tasks} + assert task_keys == {"pause", "run_nb"} + + +class TestSetupGenerator: + def test_secrets_setup_notebook_content(self): + from flowx.bundler.setup_generator import generate_setup_tasks + + secrets = [ + SecretInstruction(scope="my-scope", key="jdbc-url", value_source="JDBC URL for source"), + SecretInstruction(scope="my-scope", key="jdbc-password", value_source="JDBC password for source"), + ] + notebooks = generate_setup_tasks(secrets=secrets, setup_tasks=[], catalog="main", schema="default") + assert len(notebooks) == 1 + nb = notebooks[0] + assert nb.relative_path == "setup/create_secrets.py" + assert "createScope" in nb.content + assert "my-scope" in nb.content + assert "jdbc-url" in nb.content + assert "jdbc-password" in nb.content + + def test_volume_setup_notebook(self): + from flowx.bundler.setup_generator import generate_setup_tasks + + setup_tasks = [ + SetupTask(type="volume", config={"volume_name": "raw_data", "volume_type": "MANAGED"}), + ] + notebooks = generate_setup_tasks(secrets=[], setup_tasks=setup_tasks, catalog="prod", schema="ingest") + assert len(notebooks) == 1 + nb = notebooks[0] + assert nb.relative_path == "setup/create_volumes.py" + assert "prod.ingest.raw_data" in nb.content + + def test_connection_setup_notebook(self): + from flowx.bundler.setup_generator import generate_setup_tasks + + setup_tasks = [ + SetupTask( + type="connection", + config={"connection_name": "sql_conn", "connection_type": "SQLSERVER", "host": "sql.example.com"}, + ), + ] + notebooks = generate_setup_tasks(secrets=[], setup_tasks=setup_tasks, catalog="main", schema="default") + assert len(notebooks) == 1 + nb = notebooks[0] + assert nb.relative_path == "setup/create_connections.py" + assert "sql_conn" in nb.content + + def test_no_setup_when_empty(self): + from flowx.bundler.setup_generator import generate_setup_tasks + + notebooks = generate_setup_tasks(secrets=[], setup_tasks=[], catalog="main", schema="default") + assert len(notebooks) == 0 diff --git a/tests/unit/test_code_generator.py b/tests/unit/test_code_generator.py new file mode 100644 index 0000000..84a07d0 --- /dev/null +++ b/tests/unit/test_code_generator.py @@ -0,0 +1,507 @@ +"""Unit tests for code_generator.py notebook generators. + +Verifies each generator produces valid Python that passes ast.parse(). +""" + +from __future__ import annotations + +import ast +from typing import Any + +import pytest + +from flowx.models.ir import ( + AppendVariableActivity, + CopyActivity, + DeleteActivity, + FilterActivity, + LookupActivity, + SetVariableActivity, + WaitActivity, + WebActivity, +) +from flowx.preparer.code_generator import ( + generate_append_variable_notebook, + generate_copy_notebook, + generate_delete_notebook, + generate_filter_notebook, + generate_lookup_notebook, + generate_set_variable_notebook, + generate_wait_notebook, + generate_web_activity_notebook, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_base(name: str = "test", task_key: str = "test") -> dict[str, Any]: + return { + "name": name, + "task_key": task_key, + "description": None, + "timeout_seconds": None, + "max_retries": None, + "min_retry_interval_millis": None, + "depends_on": None, + "cluster": None, + } + + +def _assert_valid_python(content: str, label: str = "notebook"): + """Strip Databricks magic comments and assert the remaining code parses.""" + lines = [] + for line in content.split("\n"): + stripped = line.lstrip() + if stripped.startswith("# MAGIC") or stripped.startswith("# COMMAND"): + continue + if stripped == "# Databricks notebook source": + continue + lines.append(line) + python_code = "\n".join(lines) + try: + ast.parse(python_code) + except SyntaxError as exc: + pytest.fail(f"{label} has invalid Python syntax: {exc}\n---\n{python_code}") + + +# --------------------------------------------------------------------------- +# Copy notebook generator +# --------------------------------------------------------------------------- + + +class TestGenerateCopyNotebook: + def test_file_source_auto_loader(self): + """File-based source generates Auto Loader (cloudFiles) notebook.""" + activity = CopyActivity( + **_make_base("CopyBlob", "copy_blob"), + source_type="DelimitedTextSource", + sink_type="DeltaSink", + source_properties={"path": "/mnt/raw/data.csv"}, + sink_properties={"table": "bronze.raw_data"}, + ) + content = generate_copy_notebook(activity) + _assert_valid_python(content, "copy_blob (file source)") + assert "cloudFiles" in content + assert "csv" in content.lower() or "cloudFiles.format" in content + + def test_sql_source_jdbc(self): + """SQL-based source generates JDBC read notebook.""" + activity = CopyActivity( + **_make_base("CopySql", "copy_sql"), + source_type="AzureSqlSource", + sink_type="DeltaSink", + source_properties={"sqlReaderQuery": "SELECT * FROM orders"}, + sink_properties={"table": "bronze.orders"}, + ) + content = generate_copy_notebook(activity, scope="copy_sql") + _assert_valid_python(content, "copy_sql (jdbc source)") + assert "jdbc" in content + assert "dbutils.secrets.get" in content + + def test_rest_source(self): + """REST-based source generates HTTP fetch notebook.""" + activity = CopyActivity( + **_make_base("CopyRest", "copy_rest"), + source_type="RestSource", + sink_type="DeltaSink", + source_properties={"url": "https://api.example.com/data"}, + sink_properties={"table": "bronze.api_data"}, + ) + content = generate_copy_notebook(activity) + _assert_valid_python(content, "copy_rest (rest source)") + assert "requests" in content + + def test_expression_query_jdbc(self): + """JDBC source with an ADF expression query generates parameterized notebook.""" + activity = CopyActivity( + **_make_base("CopyExpr", "copy_expr"), + source_type="AzureSqlSource", + sink_type="DeltaSink", + source_properties={ + "sqlReaderQuery": { + "type": "Expression", + "value": "@concat('SELECT * FROM ', item().schema_name, '.', item().table_name)", + }, + }, + sink_properties={"table": "bronze.dynamic"}, + ) + content = generate_copy_notebook(activity, scope="copy_expr") + _assert_valid_python(content, "copy_expr (expression query)") + assert "item" in content + assert "json" in content + + def test_generic_fallback(self): + """Unknown source type generates generic Spark read/write.""" + activity = CopyActivity( + **_make_base("CopyGeneric", "copy_generic"), + source_type="SomeUnknownSource", + sink_type="DeltaSink", + ) + content = generate_copy_notebook(activity) + _assert_valid_python(content, "copy_generic (unknown source)") + assert "spark.read" in content + + def test_csv_file_format_inferred(self): + """DelimitedTextSource infers CSV file format.""" + activity = CopyActivity( + **_make_base("CopyCsv", "copy_csv"), + source_type="DelimitedTextSource", + sink_type="DeltaSink", + source_properties={}, + ) + content = generate_copy_notebook(activity) + assert '"csv"' in content + + +# --------------------------------------------------------------------------- +# Lookup notebook generator +# --------------------------------------------------------------------------- + + +class TestGenerateLookupNotebook: + def test_db_source_with_task_values(self): + """DB source lookup generates JDBC notebook with task value output.""" + activity = LookupActivity( + **_make_base("LookupConfig", "lookup_config"), + source_type="AzureSqlSource", + first_row_only=True, + source_query="SELECT TOP 1 * FROM config", + ) + content = generate_lookup_notebook(activity, scope="lookup_config") + _assert_valid_python(content, "lookup_config (DB source)") + assert "jdbc" in content + assert "dbutils.secrets.get" in content + assert "dbutils.jobs.taskValues.set" in content + + def test_non_db_source_spark_sql(self): + """Non-DB source lookup uses Spark SQL.""" + activity = LookupActivity( + **_make_base("LookupSpark", "lookup_spark"), + source_type="ParquetSource", + first_row_only=False, + source_query="SELECT * FROM table_list", + ) + content = generate_lookup_notebook(activity) + _assert_valid_python(content, "lookup_spark (Spark SQL)") + assert "spark.sql" in content + assert "jdbc" not in content + + def test_query_with_triple_quotes_does_not_break_notebook(self): + """Regression: a query containing ``\\"\\"\\"`` used to break the embedded string.""" + activity = LookupActivity( + **_make_base("LookupQQ", "lookup_qq"), + source_type="AzureSqlSource", + first_row_only=True, + source_query='SELECT \'""">\' AS q FROM dual', + ) + content = generate_lookup_notebook(activity, scope="s") + _assert_valid_python(content, "lookup_qq (triple-quote in query)") + + def test_dynamic_query_with_widget_passthrough(self): + """Pre-translated queries (containing dbutils.widgets.get) are spliced as code.""" + activity = LookupActivity( + **_make_base("LookupDyn", "lookup_dyn"), + source_type="AzureSqlSource", + first_row_only=True, + source_query="\"SELECT * FROM \" + dbutils.widgets.get('table')", + ) + content = generate_lookup_notebook(activity, scope="s") + _assert_valid_python(content, "lookup_dyn") + assert "dbutils.widgets.get('table')" in content + + def test_unparseable_dynamic_query_falls_back_to_literal(self): + """A pre-translated string that doesn't parse is embedded as a string literal.""" + activity = LookupActivity( + **_make_base("LookupBad", "lookup_bad"), + source_type="AzureSqlSource", + first_row_only=True, + source_query="dbutils.widgets.get('x' BROKEN", + ) + content = generate_lookup_notebook(activity, scope="s") + _assert_valid_python(content, "lookup_bad") + # The broken expression is embedded as a quoted string literal + # ("dbutils.widgets.get(..."), not as executable Python. + assert 'query = "dbutils.widgets.get(' in content + + +# --------------------------------------------------------------------------- +# Web activity notebook generator +# --------------------------------------------------------------------------- + + +class TestGenerateWebActivityNotebook: + def test_static_headers(self): + """Web activity with plain string headers.""" + activity = WebActivity( + **_make_base("GetApi", "get_api"), + url="https://api.example.com", + method="GET", + headers={"Accept": "application/json", "X-Custom": "value"}, + ) + content = generate_web_activity_notebook(activity) + _assert_valid_python(content, "get_api (static headers)") + assert "requests" in content + assert "application/json" in content + + def test_expression_headers(self): + """Web activity with expression dict headers generates resolved code.""" + activity = WebActivity( + **_make_base("PostApi", "post_api"), + url="https://api.example.com/submit", + method="POST", + headers={ + "Content-Type": "application/json", + "Authorization": { + "type": "Expression", + "value": "@concat('Bearer ', pipeline().parameters.Token)", + }, + }, + body={"key": "value"}, + ) + content = generate_web_activity_notebook(activity) + _assert_valid_python(content, "post_api (expression headers)") + assert "requests" in content + assert "headers" in content + + def test_auth_block_service_principal(self): + """ServicePrincipal auth generates secret-based Bearer token.""" + activity = WebActivity( + **_make_base("AuthApi", "auth_api"), + url="https://api.example.com", + method="GET", + authentication={"type": "ServicePrincipal", "resource": "https://api.example.com"}, + ) + content = generate_web_activity_notebook(activity, scope="auth_api") + _assert_valid_python(content, "auth_api (ServicePrincipal)") + assert "auth-credential" in content + assert "Bearer" in content + + def test_auth_block_basic(self): + """Basic auth generates username/password secret retrieval.""" + activity = WebActivity( + **_make_base("BasicApi", "basic_api"), + url="https://api.example.com", + method="GET", + authentication={"type": "Basic"}, + ) + content = generate_web_activity_notebook(activity, scope="basic_api") + _assert_valid_python(content, "basic_api (Basic auth)") + assert "auth-username" in content + assert "base64" in content + + def test_post_with_body(self): + """POST method includes body block.""" + activity = WebActivity( + **_make_base("PostData", "post_data"), + url="https://api.example.com", + method="POST", + body={"payload": "data"}, + ) + content = generate_web_activity_notebook(activity) + _assert_valid_python(content, "post_data (POST)") + assert "body" in content + assert "data" in content + + +# --------------------------------------------------------------------------- +# Delete notebook generator +# --------------------------------------------------------------------------- + + +class TestGenerateDeleteNotebook: + def test_delete_generates_valid_notebook(self): + activity = DeleteActivity( + **_make_base("DeleteFiles", "delete_files"), + dataset_name="ds_staging", + folder_path="/mnt/staging/old", + recursive=True, + ) + content = generate_delete_notebook(activity) + _assert_valid_python(content, "delete_files") + assert "dbutils.fs.rm" in content + assert "ds_staging" in content + + def test_delete_no_folder_path(self): + activity = DeleteActivity( + **_make_base("DeleteDs", "delete_ds"), + dataset_name="ds_cleanup", + recursive=False, + ) + content = generate_delete_notebook(activity) + _assert_valid_python(content, "delete_ds (no folder)") + + +# --------------------------------------------------------------------------- +# Set variable notebook generator +# --------------------------------------------------------------------------- + + +class TestGenerateSetVariableNotebook: + def test_literal_value(self): + """Literal value is read from widget parameter.""" + activity = SetVariableActivity( + **_make_base("SetLiteral", "set_literal"), + variable_name="status", + variable_value="completed", + value_kind="literal", + ) + content = generate_set_variable_notebook(activity) + _assert_valid_python(content, "set_literal") + assert "dbutils.widgets.get" in content + assert "dbutils.jobs.taskValues.set" in content + + def test_dab_ref_value(self): + """DAB ref value is read from widget parameter.""" + activity = SetVariableActivity( + **_make_base("SetEnv", "set_env"), + variable_name="env", + variable_value="{{job.parameters.environment}}", + value_kind="dab_ref", + ) + content = generate_set_variable_notebook(activity) + _assert_valid_python(content, "set_env (dab_ref)") + assert 'dbutils.widgets.get("value")' in content + + def test_notebook_code_value(self): + """Notebook code value embeds Python directly, no widget parameter.""" + activity = SetVariableActivity( + **_make_base("SetDate", "set_date"), + variable_name="runDate", + variable_value="datetime.now(timezone.utc).strftime('%Y-%m-%d')", + value_kind="notebook_code", + notebook_code="datetime.now(timezone.utc).strftime('%Y-%m-%d')", + notebook_imports=["from datetime import datetime, timezone"], + ) + content = generate_set_variable_notebook(activity) + _assert_valid_python(content, "set_date (notebook_code)") + assert "strftime" in content + assert "from datetime import" in content + # Should NOT reference widgets.get("value") for code values + assert 'dbutils.widgets.get("value")' not in content + + +# --------------------------------------------------------------------------- +# Wait notebook generator +# --------------------------------------------------------------------------- + + +class TestGenerateWaitNotebook: + def test_wait_generates_valid_notebook(self): + activity = WaitActivity( + **_make_base("Wait60", "wait_60"), + wait_time_seconds=60, + ) + content = generate_wait_notebook(activity) + _assert_valid_python(content, "wait_60") + assert "time.sleep" in content + assert "60" in content + + +# --------------------------------------------------------------------------- +# Filter notebook generator +# --------------------------------------------------------------------------- + + +class TestGenerateFilterNotebook: + def test_filter_generates_valid_notebook(self): + activity = FilterActivity( + **_make_base("FilterItems", "filter_items"), + items_expression="@variables('myList')", + condition_expression="@not(empty(item()))", + ) + content = generate_filter_notebook(activity) + _assert_valid_python(content, "filter_items") + assert "dbutils.jobs.taskValues.set" in content + assert "filter" in content.lower() or "filtered" in content + + def test_resolved_condition_emits_inline_comprehension(self): + """When the translator pre-resolves the condition, no eval() call appears.""" + activity = FilterActivity( + **_make_base("FilterActive", "filter_active"), + items_expression="{{tasks.Lookup.values.result}}", + condition_expression="@equals(item().status, 'active')", + condition_code="(item.get('status') == 'active')", + ) + content = generate_filter_notebook(activity) + _assert_valid_python(content, "filter_active") + assert _contains_eval_call(content) is False, "resolved condition path must not contain an eval() call" + assert "filtered = [item for item in items if (item.get('status') == 'active')]" in content + assert "json.loads(items_expression)" in content + assert "NotImplementedError" not in content + + def test_unresolved_condition_falls_back_to_placeholder(self): + """When condition_code is None, the notebook is a TODO placeholder (no eval call).""" + activity = FilterActivity( + **_make_base("FilterMystery", "filter_mystery"), + items_expression="@variables('myList')", + condition_expression="@some_function_we_dont_translate(item())", + condition_code=None, + ) + content = generate_filter_notebook(activity) + _assert_valid_python(content, "filter_mystery") + assert _contains_eval_call(content) is False, "placeholder path must not contain an eval() call" + assert "NotImplementedError" in content + assert "Implement filter condition" in content + + def test_no_eval_in_either_branch_with_quotes_in_expressions(self): + """ADF expressions containing single quotes don't break the notebook syntax.""" + for resolved_code in (None, "(item.get('region') == 'us-west-1')"): + activity = FilterActivity( + **_make_base("FilterQ", "filter_q"), + items_expression="@activity('GetList').output.value", + condition_expression="@equals(item().region, 'us-west-1')", + condition_code=resolved_code, + ) + content = generate_filter_notebook(activity) + _assert_valid_python(content, "filter_q") + assert _contains_eval_call(content) is False + + +def _contains_eval_call(source: str) -> bool: + """Returns True if the parsed Python source contains a real ``eval(...)`` call.""" + import ast + + tree = ast.parse(source) + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "eval": + return True + return False + + +# --------------------------------------------------------------------------- +# Append variable notebook generator +# --------------------------------------------------------------------------- + + +class TestGenerateAppendVariableNotebook: + def test_literal_append(self): + """Literal value generates widget-based append.""" + activity = AppendVariableActivity( + **_make_base("AppendLog", "append_log"), + variable_name="logEntries", + append_value="step1 done", + value_kind="literal", + ) + content = generate_append_variable_notebook(activity) + _assert_valid_python(content, "append_log (literal)") + assert "dbutils.jobs.taskValues.set" in content + assert "append" in content + + def test_notebook_code_append(self): + """Notebook code generates embedded Python.""" + activity = AppendVariableActivity( + **_make_base("AppendTS", "append_ts"), + variable_name="timestamps", + append_value="datetime.now(timezone.utc).isoformat()", + value_kind="notebook_code", + notebook_code="datetime.now(timezone.utc).isoformat()", + notebook_imports=["from datetime import datetime, timezone"], + ) + content = generate_append_variable_notebook(activity) + _assert_valid_python(content, "append_ts (notebook_code)") + assert "isoformat" in content + assert "from datetime import" in content + # Should NOT reference widgets.get("value") for code values + assert 'dbutils.widgets.get("value")' not in content diff --git a/tests/unit/test_expression_parser.py b/tests/unit/test_expression_parser.py new file mode 100644 index 0000000..b88bfdf --- /dev/null +++ b/tests/unit/test_expression_parser.py @@ -0,0 +1,756 @@ +"""Unit tests for the unified resolve_expression() function.""" + +from __future__ import annotations + +from types import MappingProxyType + +from flowx.models.ir import TranslationContext +from flowx.parser.expression_parser import ( + parse_expression, + parse_expression_for_dab, + resolve_expression, +) + + +def _context(**variable_mappings: str) -> TranslationContext: + """Build a context with optional variable -> task_key mappings.""" + variable_cache = MappingProxyType(variable_mappings) if variable_mappings else MappingProxyType({}) + return TranslationContext(variable_cache=variable_cache) + + +class TestLiterals: + def test_plain_string(self): + result = resolve_expression("hello", _context()) + assert result is not None + assert result.kind == "literal" + assert result.value == "hello" + + def test_integer(self): + result = resolve_expression(42, _context()) + assert result is not None + assert result.kind == "literal" + assert result.value == "42" + + def test_float(self): + result = resolve_expression(3.14, _context()) + assert result is not None + assert result.kind == "literal" + assert result.value == "3.14" + + def test_boolean(self): + result = resolve_expression(True, _context()) + assert result is not None + assert result.kind == "literal" + assert result.value == "True" + + def test_expression_dict_wrapping(self): + result = resolve_expression({"type": "Expression", "value": "hello"}, _context()) + assert result is not None + assert result.kind == "literal" + assert result.value == "hello" + + +class TestPipelineProperties: + def test_run_id(self): + result = resolve_expression("@pipeline().RunId", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.run_id}}" + + def test_pipeline_name(self): + result = resolve_expression("@pipeline().Pipeline", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.name}}" + + def test_trigger_time(self): + result = resolve_expression("@pipeline().TriggerTime", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.start_time.iso_datetime}}" + + def test_group_id(self): + result = resolve_expression("@pipeline().GroupId", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.run_id}}" + + +class TestPipelineParameters: + def test_parameter(self): + result = resolve_expression("@pipeline().parameters.environment", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.parameters.environment}}" + + def test_parameter_expression_dict(self): + result = resolve_expression( + {"type": "Expression", "value": "@pipeline().parameters.date"}, + _context(), + ) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.parameters.date}}" + + +class TestActivityOutput: + def test_firstrow_column(self): + result = resolve_expression("@activity('Lookup').output.firstRow.cnt", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{tasks.Lookup.values.cnt}}" + + def test_output_value(self): + result = resolve_expression("@activity('GetList').output.value", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{tasks.GetList.values.result}}" + + def test_output_no_path(self): + result = resolve_expression("@activity('Task').output", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{tasks.Task.values.result}}" + + def test_sanitizes_task_key(self): + result = resolve_expression("@activity('Lookup Row Count').output.firstRow.row_count", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert "Lookup_Row_Count" in result.value + + +class TestVariables: + def test_variable_with_context(self): + result = resolve_expression("@variables('outputPath')", _context(outputPath="SetOutputPath")) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{tasks.SetOutputPath.values.outputPath}}" + + def test_variable_with_explicit_task_keys(self): + result = resolve_expression( + "@variables('runDate')", + _context(), + variable_task_keys={"runDate": "SetRunDate"}, + ) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{tasks.SetRunDate.values.runDate}}" + + def test_variable_fallback_to_name(self): + result = resolve_expression("@variables('unknown')", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{tasks.unknown.values.unknown}}" + + +class TestItem: + def test_item(self): + result = resolve_expression("@item()", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{input}}" + + +class TestUtcNow: + def test_utcnow_no_format(self): + # ``@utcNow()`` resolves to Python ``datetime.now(...).isoformat()`` + # rather than the DAB ref ``{{job.start_time.iso_datetime}}`` so + # compositions like ``@formatDateTime(utcNow(), '...')`` chain + # correctly. DAB does not evaluate ADF expressions, so wrapping a + # DAB ref in another ADF function would emit broken YAML. + result = resolve_expression("@utcNow()", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert result.value == "datetime.now(timezone.utc).isoformat()" + + def test_utcnow_with_format(self): + result = resolve_expression("@utcNow('yyyy-MM-dd')", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "strftime" in result.value + assert "%Y-%m-%d" in result.value + + def test_utcnow_expression_dict(self): + result = resolve_expression( + {"type": "Expression", "value": "@utcNow('yyyy-MM-dd')"}, + _context(), + ) + assert result is not None + assert result.kind == "notebook_code" + + +class TestConcat: + def test_concat_literals(self): + result = resolve_expression("@concat('hello', ' ', 'world')", _context()) + assert result is not None + assert result.kind == "notebook_code" + # Should produce a Python concatenation + assert "+" in result.value + + def test_concat_with_variable(self): + result = resolve_expression( + "@concat('output/', variables('runDate'), '/processed')", + _context(runDate="SetRunDate"), + ) + assert result is not None + assert result.kind == "notebook_code" + assert "runDate" in result.value + + def test_concat_with_utcnow(self): + result = resolve_expression("@concat('date_', utcNow('yyyy-MM-dd'))", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "strftime" in result.value + + def test_concat_with_pipeline_param(self): + result = resolve_expression( + "@concat(variables('catalogName'), '.', pipeline().parameters.schemaPrefix, '_data')", + _context(catalogName="SetCatalogName"), + ) + assert result is not None + assert result.kind == "notebook_code" + + +class TestUnsupported: + def test_non_dict_non_scalar(self): + result = resolve_expression({"type": "Other"}, _context()) + assert result is None + + def test_agentic_data_uri(self): + result = resolve_expression("@dataUri('hello')", _context()) + assert result is None + + def test_agentic_xml(self): + result = resolve_expression("@xml('')", _context()) + assert result is None + + def test_agentic_xpath(self): + result = resolve_expression("@xpath(xml(''), '/')", _context()) + assert result is None + + def test_convert_from_utc_resolves_to_notebook_code(self): + result = resolve_expression("@convertFromUtc('2024-01-01T00:00:00Z', 'Pacific Standard Time')", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "ZoneInfo" in result.value + + def test_ticks_resolves_to_notebook_code(self): + result = resolve_expression("@ticks('2024-01-01T00:00:00Z')", _context()) + assert result is not None + assert result.kind == "notebook_code" + + +class TestStringFunctions: + def test_ends_with(self): + result = resolve_expression("@endsWith('hello world', 'world')", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "endswith" in result.value + + def test_ends_with_nested(self): + result = resolve_expression("@endsWith(toLower('HELLO'), 'hello')", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "endswith" in result.value + assert "lower" in result.value + + def test_guid_no_args(self): + result = resolve_expression("@guid()", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "uuid4" in result.value + + def test_guid_format_n(self): + result = resolve_expression("@guid('N')", _context()) + assert result is not None + assert "replace" in result.value + + def test_index_of(self): + result = resolve_expression("@indexOf('hello world', 'world')", _context()) + assert result is not None + assert "find" in result.value + + def test_last_index_of(self): + result = resolve_expression("@lastIndexOf('hello hello', 'hello')", _context()) + assert result is not None + assert "rfind" in result.value + + def test_replace(self): + result = resolve_expression("@replace('hello world', 'world', 'python')", _context()) + assert result is not None + assert "replace" in result.value + + def test_split(self): + result = resolve_expression("@split('a,b,c', ',')", _context()) + assert result is not None + assert "split" in result.value + + def test_starts_with(self): + result = resolve_expression("@startsWith('hello world', 'hello')", _context()) + assert result is not None + assert "startswith" in result.value + + def test_substring(self): + result = resolve_expression("@substring('hello', 0, 3)", _context()) + assert result is not None + assert result.kind == "notebook_code" + # Should produce a slice expression + assert "[" in result.value + + def test_to_lower(self): + result = resolve_expression("@toLower('HELLO')", _context()) + assert result is not None + assert "lower" in result.value + + def test_to_upper(self): + result = resolve_expression("@toUpper('hello')", _context()) + assert result is not None + assert "upper" in result.value + + def test_trim(self): + result = resolve_expression("@trim(' hello ')", _context()) + assert result is not None + assert "strip" in result.value + + +class TestCollectionFunctions: + def test_contains(self): + result = resolve_expression("@contains('hello world', 'hello')", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "in" in result.value + + def test_empty(self): + result = resolve_expression("@empty('')", _context()) + assert result is not None + assert "len" in result.value + + def test_first(self): + result = resolve_expression("@first(createArray(1, 2, 3))", _context()) + assert result is not None + assert "[0]" in result.value + + def test_join(self): + result = resolve_expression("@join(createArray('a', 'b', 'c'), ',')", _context()) + assert result is not None + assert "join" in result.value + + def test_last(self): + result = resolve_expression("@last(createArray(1, 2, 3))", _context()) + assert result is not None + assert "[-1]" in result.value + + def test_length(self): + result = resolve_expression("@length('hello')", _context()) + assert result is not None + assert "len" in result.value + + def test_skip(self): + result = resolve_expression("@skip(createArray(1, 2, 3), 1)", _context()) + assert result is not None + assert result.kind == "notebook_code" + + def test_take(self): + result = resolve_expression("@take(createArray(1, 2, 3), 2)", _context()) + assert result is not None + assert result.kind == "notebook_code" + + def test_intersection(self): + result = resolve_expression("@intersection(createArray(1, 2, 3), createArray(2, 3, 4))", _context()) + assert result is not None + assert "set" in result.value + + def test_union(self): + result = resolve_expression("@union(createArray(1, 2), createArray(3, 4))", _context()) + assert result is not None + assert "set" in result.value + + +class TestLogicalFunctions: + def test_and(self): + result = resolve_expression("@and(true, false)", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "and" in result.value + + def test_equals(self): + result = resolve_expression("@equals(1, 1)", _context()) + assert result is not None + assert "==" in result.value + + def test_greater(self): + result = resolve_expression("@greater(5, 3)", _context()) + assert result is not None + assert ">" in result.value + + def test_greater_or_equals(self): + result = resolve_expression("@greaterOrEquals(5, 5)", _context()) + assert result is not None + assert ">=" in result.value + + def test_if(self): + result = resolve_expression("@if(equals(1, 1), 'yes', 'no')", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "if" in result.value + assert "else" in result.value + + def test_less(self): + result = resolve_expression("@less(3, 5)", _context()) + assert result is not None + assert "<" in result.value + + def test_less_or_equals(self): + result = resolve_expression("@lessOrEquals(3, 3)", _context()) + assert result is not None + assert "<=" in result.value + + def test_not(self): + result = resolve_expression("@not(true)", _context()) + assert result is not None + assert "not" in result.value + + def test_or(self): + result = resolve_expression("@or(true, false)", _context()) + assert result is not None + assert "or" in result.value + + +class TestConversionFunctions: + def test_array(self): + result = resolve_expression("@array('hello')", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "[" in result.value + + def test_base64(self): + result = resolve_expression("@base64('hello')", _context()) + assert result is not None + assert "b64encode" in result.value + + def test_base64_to_string(self): + result = resolve_expression("@base64ToString('aGVsbG8=')", _context()) + assert result is not None + assert "b64decode" in result.value + assert "decode" in result.value + + def test_base64_to_binary(self): + result = resolve_expression("@base64ToBinary('aGVsbG8=')", _context()) + assert result is not None + assert "b64decode" in result.value + + def test_binary(self): + result = resolve_expression("@binary('hello')", _context()) + assert result is not None + assert "encode" in result.value + + def test_bool(self): + result = resolve_expression("@bool(1)", _context()) + assert result is not None + assert "bool" in result.value + + def test_coalesce(self): + result = resolve_expression("@coalesce(null, 'fallback')", _context()) + assert result is not None + assert "next" in result.value + assert "None" in result.value + + def test_create_array(self): + result = resolve_expression("@createArray(1, 2, 3)", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "[" in result.value + + def test_decode_base64_alias(self): + result = resolve_expression("@decodeBase64('aGVsbG8=')", _context()) + assert result is not None + assert "b64decode" in result.value + + def test_decode_uri_component(self): + result = resolve_expression("@decodeUriComponent('hello%20world')", _context()) + assert result is not None + assert "unquote" in result.value + + def test_encode_uri_component(self): + result = resolve_expression("@encodeUriComponent('hello world')", _context()) + assert result is not None + assert "quote" in result.value + + def test_float(self): + result = resolve_expression("@float('3.14')", _context()) + assert result is not None + assert "float" in result.value + + def test_int(self): + result = resolve_expression("@int('42')", _context()) + assert result is not None + assert "int" in result.value + + def test_json(self): + result = resolve_expression('@json(\'{"key": "value"}\')', _context()) + assert result is not None + assert "loads" in result.value + + def test_string(self): + result = resolve_expression("@string(42)", _context()) + assert result is not None + assert "str" in result.value + + def test_uri_component_alias(self): + result = resolve_expression("@uriComponent('hello world')", _context()) + assert result is not None + assert "quote" in result.value + + def test_uri_component_to_string_alias(self): + result = resolve_expression("@uriComponentToString('hello%20world')", _context()) + assert result is not None + assert "unquote" in result.value + + +class TestMathFunctions: + def test_add(self): + result = resolve_expression("@add(1, 2)", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "+" in result.value + + def test_div(self): + result = resolve_expression("@div(10, 3)", _context()) + assert result is not None + assert "//" in result.value + + def test_max(self): + result = resolve_expression("@max(1, 5, 3)", _context()) + assert result is not None + assert "max" in result.value + + def test_min(self): + result = resolve_expression("@min(1, 5, 3)", _context()) + assert result is not None + assert "min" in result.value + + def test_mod(self): + result = resolve_expression("@mod(7, 3)", _context()) + assert result is not None + assert "%" in result.value + + def test_mul(self): + result = resolve_expression("@mul(3, 4)", _context()) + assert result is not None + assert "*" in result.value + + def test_rand(self): + result = resolve_expression("@rand(1, 100)", _context()) + assert result is not None + assert "randint" in result.value + + def test_range(self): + result = resolve_expression("@range(0, 10)", _context()) + assert result is not None + assert "range" in result.value + + def test_sub(self): + result = resolve_expression("@sub(10, 3)", _context()) + assert result is not None + assert "-" in result.value + + +class TestDateTimeFunctions: + def test_add_days(self): + result = resolve_expression("@addDays('2024-01-01T00:00:00', 5)", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "timedelta" in result.value + assert "days" in result.value + + def test_add_days_with_format(self): + result = resolve_expression("@addDays('2024-01-01T00:00:00', 5, 'yyyy-MM-dd')", _context()) + assert result is not None + assert "strftime" in result.value + assert "%Y-%m-%d" in result.value + + def test_add_hours(self): + result = resolve_expression("@addHours('2024-01-01T00:00:00', 3)", _context()) + assert result is not None + assert "hours" in result.value + + def test_add_minutes(self): + result = resolve_expression("@addMinutes('2024-01-01T00:00:00', 30)", _context()) + assert result is not None + assert "minutes" in result.value + + def test_add_seconds(self): + result = resolve_expression("@addSeconds('2024-01-01T00:00:00', 90)", _context()) + assert result is not None + assert "seconds" in result.value + + def test_add_to_time(self): + result = resolve_expression("@addToTime('2024-01-01T00:00:00', 2, 'Hour')", _context()) + assert result is not None + assert "timedelta" in result.value + assert "hours" in result.value + + def test_day_of_month(self): + result = resolve_expression("@dayOfMonth('2024-01-15T00:00:00')", _context()) + assert result is not None + assert ".day" in result.value + + def test_day_of_week(self): + result = resolve_expression("@dayOfWeek('2024-01-15T00:00:00')", _context()) + assert result is not None + assert "isoweekday" in result.value + + def test_day_of_year(self): + result = resolve_expression("@dayOfYear('2024-01-15T00:00:00')", _context()) + assert result is not None + assert "tm_yday" in result.value + + def test_format_date_time(self): + result = resolve_expression("@formatDateTime('2024-01-01T00:00:00', 'yyyy-MM-dd')", _context()) + assert result is not None + assert "strftime" in result.value + assert "%Y-%m-%d" in result.value + + def test_format_date_time_no_format(self): + result = resolve_expression("@formatDateTime('2024-01-01T00:00:00')", _context()) + assert result is not None + assert "isoformat" in result.value + + def test_get_future_time(self): + result = resolve_expression("@getFutureTime(5, 'Day')", _context()) + assert result is not None + assert "timedelta" in result.value + assert "days" in result.value + assert "from datetime import datetime, timezone, timedelta" in result.imports + + def test_get_past_time(self): + result = resolve_expression("@getPastTime(3, 'Hour')", _context()) + assert result is not None + assert "timedelta" in result.value + assert "hours" in result.value + + def test_start_of_day(self): + result = resolve_expression("@startOfDay('2024-01-15T14:30:00')", _context()) + assert result is not None + assert "hour=0" in result.value + + def test_start_of_hour(self): + result = resolve_expression("@startOfHour('2024-01-15T14:30:00')", _context()) + assert result is not None + assert "minute=0" in result.value + + def test_start_of_month(self): + result = resolve_expression("@startOfMonth('2024-01-15T14:30:00')", _context()) + assert result is not None + assert "day=1" in result.value + + def test_subtract_from_time(self): + result = resolve_expression("@subtractFromTime('2024-01-15T00:00:00', 5, 'Day')", _context()) + assert result is not None + assert "timedelta" in result.value + assert " - " in result.value + + +class TestNestedFunctions: + def test_concat_with_toLower_and_toUpper(self): + result = resolve_expression("@concat(toLower('Hello'), toUpper('world'))", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "lower" in result.value + assert "upper" in result.value + + def test_if_with_equals(self): + result = resolve_expression("@if(equals(1, 1), 'yes', 'no')", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "==" in result.value + assert "if" in result.value + + def test_deeply_nested(self): + result = resolve_expression("@concat(toLower(trim(' HELLO ')), '_', toUpper('world'))", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "lower" in result.value + assert "strip" in result.value + assert "upper" in result.value + + def test_first_of_create_array(self): + result = resolve_expression("@first(createArray('a', 'b', 'c'))", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "[0]" in result.value + + def test_length_of_split(self): + result = resolve_expression("@length(split('a,b,c', ','))", _context()) + assert result is not None + assert "len" in result.value + assert "split" in result.value + + def test_nested_math(self): + result = resolve_expression("@add(mul(2, 3), sub(10, 4))", _context()) + assert result is not None + assert result.kind == "notebook_code" + + def test_replace_with_pipeline_param(self): + result = resolve_expression( + "@replace(pipeline().parameters.path, '/old/', '/new/')", + _context(), + ) + assert result is not None + assert result.kind == "notebook_code" + assert "replace" in result.value + assert "dbutils.widgets.get" in result.value + + +class TestFunctionsWithDabRefs: + def test_to_lower_with_pipeline_param(self): + result = resolve_expression("@toLower(pipeline().parameters.env)", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "lower" in result.value + assert "dbutils.widgets.get" in result.value + + def test_concat_with_variable_and_literal(self): + result = resolve_expression( + "@concat(variables('prefix'), '_suffix')", + _context(prefix="SetPrefix"), + ) + assert result is not None + assert result.kind == "notebook_code" + assert "dbutils.widgets.get" in result.value + + def test_equals_with_activity_output(self): + result = resolve_expression( + "@equals(activity('Check').output.firstRow.status, 'done')", + _context(), + ) + assert result is not None + assert result.kind == "notebook_code" + assert "==" in result.value + assert "dbutils.widgets.get" in result.value + + +class TestBackwardCompat: + def test_parse_expression_returns_value(self): + result = parse_expression("@pipeline().RunId", _context()) + assert result == "{{job.run_id}}" + + def test_parse_expression_returns_none_for_unsupported(self): + result = parse_expression("@dataUri('hello')", _context()) + assert result is None + + def test_parse_expression_for_dab_returns_ref(self): + result = parse_expression_for_dab("@pipeline().RunId") + assert result == "{{job.run_id}}" + + def test_parse_expression_for_dab_returns_none_for_utcnow(self): + # ``@utcNow()`` now resolves to notebook_code so it composes correctly + # with other ADF time functions. ``parse_expression_for_dab`` only + # returns dab_ref kinds, so utcNow now yields ``None`` (the caller + # routes through the notebook_code path instead). + result = parse_expression_for_dab("@utcNow()") + assert result is None + + def test_parse_expression_for_dab_returns_none_for_non_expression(self): + result = parse_expression_for_dab("plain_string") + assert result is None diff --git a/tests/unit/test_helpers.py b/tests/unit/test_helpers.py new file mode 100644 index 0000000..aea8086 --- /dev/null +++ b/tests/unit/test_helpers.py @@ -0,0 +1,230 @@ +"""Unit tests for the shared helpers used across the preparer and bundler.""" + +from __future__ import annotations + +from flowx.models.dab import DabNotebook +from flowx.models.ir import ( + Dependency, + SetVariableActivity, +) +from flowx.models.source_types import ( + FILE_SOURCE_TYPES, + JDBC_SOURCE_TYPES, + REST_SOURCE_TYPES, +) +from flowx.preparer.activity_preparers.helpers import ( + build_notebook_activity_task, + build_notebook_task_artifacts, + make_jdbc_secrets, + resolve_param_value, +) +from flowx.preparer.workflow_preparer import ( + PreparedActivity, + PreparedArtifacts, + merge_prepared_artifacts, +) + + +class TestSourceTypeTaxonomy: + def test_jdbc_and_rest_are_disjoint(self): + assert JDBC_SOURCE_TYPES.isdisjoint(REST_SOURCE_TYPES) + + def test_jdbc_and_file_are_disjoint(self): + assert JDBC_SOURCE_TYPES.isdisjoint(FILE_SOURCE_TYPES) + + def test_http_source_is_file_only_not_rest(self): + """ADF HttpSource downloads a single file; RestSource is for paginated APIs.""" + assert "HttpSource" in FILE_SOURCE_TYPES + assert "HttpSource" not in REST_SOURCE_TYPES + assert "RestSource" in REST_SOURCE_TYPES + + def test_azure_sql_database_source_is_jdbc(self): + """Regression: ``AzureSqlDatabaseSource`` (the v2 name) was missing once.""" + assert "AzureSqlDatabaseSource" in JDBC_SOURCE_TYPES + assert "AzureSqlSource" in JDBC_SOURCE_TYPES + + +class TestResolveParamValue: + def test_literal_passes_through(self): + assert resolve_param_value("plain") == "plain" + + def test_dab_ref_resolves(self): + assert resolve_param_value("@pipeline().RunId") == "{{job.run_id}}" + + def test_interpolated_resolves(self): + result = resolve_param_value("prefix-@{pipeline().parameters.env}-suffix") + assert result == "prefix-{{job.parameters.env}}-suffix" + + def test_notebook_code_returns_raw_for_manual_handling(self): + """Expressions that resolve to Python code return raw text (manual handling).""" + raw = "@formatDateTime(utcnow(), 'yyyy-MM-dd')" + result = resolve_param_value(raw) + assert result == raw + + +class TestBuildNotebookTaskArtifacts: + def test_returns_task_dict_and_one_notebook(self): + notebook_task, notebooks = build_notebook_task_artifacts( + notebook_relative_path="notebooks/foo.py", + notebook_content="# stub", + ) + assert notebook_task == {"notebook_path": "../src/notebooks/foo.py"} + assert len(notebooks) == 1 + assert notebooks[0].relative_path == "notebooks/foo.py" + assert notebooks[0].content == "# stub" + + def test_includes_base_parameters_when_set(self): + notebook_task, _ = build_notebook_task_artifacts( + notebook_relative_path="notebooks/foo.py", + notebook_content="# stub", + base_parameters={"env": "dev"}, + ) + assert notebook_task["base_parameters"] == {"env": "dev"} + + def test_omits_base_parameters_when_none(self): + notebook_task, _ = build_notebook_task_artifacts( + notebook_relative_path="notebooks/foo.py", + notebook_content="# stub", + base_parameters=None, + ) + assert "base_parameters" not in notebook_task + + +class TestBuildNotebookActivityTask: + def test_combines_common_fields_and_notebook_task(self): + activity = SetVariableActivity( + name="Set Var", + task_key="set_var", + depends_on=[Dependency(task_key="upstream")], + timeout_seconds=600, + variable_name="x", + variable_value="hello", + ) + task, notebooks = build_notebook_activity_task( + activity, + notebook_relative_path="notebooks/set_var.py", + notebook_content="# stub", + base_parameters={"variable_name": "x"}, + ) + assert task["task_key"] == "set_var" + assert task["depends_on"] == [{"task_key": "upstream"}] + assert task["timeout_seconds"] == 600 + assert task["notebook_task"] == { + "notebook_path": "../src/notebooks/set_var.py", + "base_parameters": {"variable_name": "x"}, + } + assert len(notebooks) == 1 + + +class TestMakeJdbcSecrets: + def test_emits_url_and_password_pair(self): + secrets = make_jdbc_secrets( + scope_name="my_pipeline", + source_type="AzureSqlSource", + activity_name="LookupConfig", + role="lookup", + ) + assert len(secrets) == 2 + assert {s.key for s in secrets} == {"jdbc-url", "jdbc-password"} + assert all(s.scope == "my_pipeline" for s in secrets) + assert "AzureSqlSource lookup in activity 'LookupConfig'" in secrets[0].value_source + + def test_role_changes_descriptions_only(self): + source_secrets = make_jdbc_secrets( + scope_name="s", source_type="SqlServerSource", activity_name="Copy", role="source" + ) + sink_secrets = make_jdbc_secrets( + scope_name="s", source_type="SqlServerSource", activity_name="Copy", role="sink" + ) + assert {s.key for s in source_secrets} == {s.key for s in sink_secrets} + assert "source in activity" in source_secrets[0].value_source + assert "sink in activity" in sink_secrets[0].value_source + + +class TestPreparedArtifacts: + def test_default_is_empty(self): + artifacts = PreparedArtifacts() + assert artifacts.notebooks == () + assert artifacts.secrets == () + assert artifacts.setup_tasks == () + assert artifacts.inner_workflows == () + + def test_is_immutable(self): + artifacts = PreparedArtifacts() + # frozen dataclass: cannot reassign fields + try: + artifacts.notebooks = (DabNotebook(relative_path="x", content=""),) # type: ignore[misc] + except Exception: + return + raise AssertionError("PreparedArtifacts must be frozen") + + +class TestMergePreparedArtifacts: + def test_does_not_mutate_input(self): + original = PreparedArtifacts() + prepared = PreparedActivity( + task={}, + notebooks=[DabNotebook(relative_path="a.py", content="# a")], + ) + merged = merge_prepared_artifacts(original, prepared) + + assert merged is not original + assert original.notebooks == () + assert len(merged.notebooks) == 1 + assert merged.notebooks[0].relative_path == "a.py" + + def test_extends_each_collection(self): + from flowx.models.dab import SecretInstruction, SetupTask + + prepared = PreparedActivity( + task={"task_key": "t"}, + notebooks=[DabNotebook(relative_path="a.py", content="# a")], + secrets=[SecretInstruction(scope="s", key="k", value_source="v")], + setup_tasks=[SetupTask(type="volume", config={})], + ) + artifacts = merge_prepared_artifacts(PreparedArtifacts(), prepared) + assert len(artifacts.notebooks) == 1 + assert len(artifacts.secrets) == 1 + assert len(artifacts.setup_tasks) == 1 + + def test_chains_correctly_across_multiple_activities(self): + artifacts = PreparedArtifacts() + for i in range(3): + prepared = PreparedActivity( + task={"task_key": f"t{i}"}, + notebooks=[DabNotebook(relative_path=f"{i}.py", content="")], + ) + artifacts = merge_prepared_artifacts(artifacts, prepared) + assert [nb.relative_path for nb in artifacts.notebooks] == ["0.py", "1.py", "2.py"] + + +class TestExpressionParserHandlerFactories: + """The ``addDays`` / ``addHours`` / ``addMinutes`` / ``addSeconds`` family + is generated from a single factory; same for ``getFutureTime`` / + ``getPastTime``. Sanity-check each handler still resolves end-to-end.""" + + def test_add_unit_handlers_emit_correct_timedelta_keyword(self): + from flowx.models.ir import TranslationContext + from flowx.parser.expression_parser import resolve_expression + + cases = [ + ("@addDays('2024-01-01', 7)", "timedelta(days=7)"), + ("@addHours('2024-01-01', 5)", "timedelta(hours=5)"), + ("@addMinutes('2024-01-01', 30)", "timedelta(minutes=30)"), + ("@addSeconds('2024-01-01', 90)", "timedelta(seconds=90)"), + ] + for expression, fragment in cases: + result = resolve_expression(expression, TranslationContext()) + assert result is not None + assert result.kind == "notebook_code" + assert fragment in result.value + + def test_now_offset_handlers_emit_correct_sign(self): + from flowx.models.ir import TranslationContext + from flowx.parser.expression_parser import resolve_expression + + future = resolve_expression("@getFutureTime(1, 'Day')", TranslationContext()) + past = resolve_expression("@getPastTime(1, 'Day')", TranslationContext()) + assert future is not None and past is not None + assert "+ timedelta" in future.value + assert "- timedelta" in past.value diff --git a/tests/unit/test_motifs.py b/tests/unit/test_motifs.py new file mode 100644 index 0000000..1be2e9b --- /dev/null +++ b/tests/unit/test_motifs.py @@ -0,0 +1,252 @@ +"""Tests for motif detection and collapsing.""" + +from __future__ import annotations + +from flowx.models.adf_ast import ( + AdfActivity, + AdfDefinitions, + AdfDependency, + AdfPipeline, +) +from flowx.models.ir import ( + Activity, + Dependency, + MotifActivity, + Pipeline, +) +from flowx.models.motifs import MOTIF_METADATA_DRIVEN_BULK_COPY +from flowx.motifs.collapser import collapse_motifs +from flowx.motifs.detector import detect_motifs + +_EMPTY_DEFS = AdfDefinitions(pipelines=[], datasets={}, linked_services={}, triggers=[]) + + +def _adf_activity( + name: str, + adf_type: str, + depends_on: list[str] | None = None, + type_properties: dict | None = None, + activities: list[AdfActivity] | None = None, +) -> AdfActivity: + deps = ( + [AdfDependency(activity=dep, dependency_conditions=["Succeeded"]) for dep in depends_on] if depends_on else None + ) + return AdfActivity( + name=name, + type=adf_type, + depends_on=deps, + type_properties=type_properties, + activities=activities, + ) + + +def _ir_activity(name: str, depends_on: list[str] | None = None) -> Activity: + deps = [Dependency(task_key=dep) for dep in depends_on] if depends_on else None + return Activity(name=name, task_key=name, depends_on=deps) + + +class TestDetectorMetadataDrivenBulkCopy: + def test_detects_lookup_foreach_copy_pattern(self): + copy_child = _adf_activity("CopyTable", "Copy") + pipeline = AdfPipeline( + name="test_pipeline", + activities=[ + _adf_activity( + "GetTableList", + "Lookup", + type_properties={ + "source": {"sqlReaderQuery": "SELECT table_name FROM config.control_table"}, + }, + ), + _adf_activity( + "ForEachTable", + "ForEach", + depends_on=["GetTableList"], + activities=[copy_child], + ), + ], + ) + motifs = detect_motifs(pipeline, _EMPTY_DEFS) + assert len(motifs) == 1 + assert motifs[0].definition.motif_id == "metadata_driven_bulk_copy" + assert "GetTableList" in motifs[0].matched_activities + assert "ForEachTable" in motifs[0].matched_activities + + def test_no_match_when_foreach_has_execute_pipeline(self): + exec_child = _adf_activity("RunChild", "ExecutePipeline") + pipeline = AdfPipeline( + name="test_pipeline", + activities=[ + _adf_activity("GetList", "Lookup"), + _adf_activity( + "Loop", + "ForEach", + depends_on=["GetList"], + activities=[exec_child], + ), + ], + ) + motifs = detect_motifs(pipeline, _EMPTY_DEFS) + bulk_copy = [m for m in motifs if m.definition.motif_id == "metadata_driven_bulk_copy"] + assert len(bulk_copy) == 0 + + +class TestDetectorCopyAndNotify: + def test_detects_copy_then_web_notification(self): + pipeline = AdfPipeline( + name="test_pipeline", + activities=[ + _adf_activity("CopyData", "Copy"), + _adf_activity( + "NotifySuccess", + "WebActivity", + depends_on=["CopyData"], + type_properties={ + "url": "https://hooks.slack.com/services/T00/B00/xxx", + "method": "POST", + }, + ), + ], + ) + motifs = detect_motifs(pipeline, _EMPTY_DEFS) + assert len(motifs) == 1 + assert motifs[0].definition.motif_id == "copy_and_notify" + + +class TestDetectorParentChild: + def test_detects_lookup_foreach_execute_pipeline(self): + exec_child = _adf_activity("RunChildPipeline", "ExecutePipeline") + pipeline = AdfPipeline( + name="test_pipeline", + activities=[ + _adf_activity("GetWorkItems", "Lookup"), + _adf_activity( + "ProcessItems", + "ForEach", + depends_on=["GetWorkItems"], + activities=[exec_child], + ), + ], + ) + motifs = detect_motifs(pipeline, _EMPTY_DEFS) + assert len(motifs) == 1 + assert motifs[0].definition.motif_id == "parent_child_orchestration" + + +class TestCollapser: + def test_collapse_replaces_activities_with_motif(self): + pipeline = Pipeline( + name="test", + tasks=[ + _ir_activity("GetTableList"), + _ir_activity("ForEachTable", depends_on=["GetTableList"]), + _ir_activity("PostProcessing", depends_on=["ForEachTable"]), + ], + ) + motif = MOTIF_METADATA_DRIVEN_BULK_COPY + from flowx.models.motifs import DetectedMotif + + detected = DetectedMotif( + definition=motif, + matched_activities=["GetTableList", "ForEachTable"], + source_type_hint="database", + confidence_notes=["Test match"], + ) + result = collapse_motifs(pipeline, [detected]) + + assert len(result.tasks) == 2 + motif_task = result.tasks[0] + assert isinstance(motif_task, MotifActivity) + assert motif_task.motif_id == "metadata_driven_bulk_copy" + assert motif_task.databricks_replacement == "for_each_ingestion" + assert "GetTableList" in motif_task.matched_activity_names + + post = result.tasks[1] + assert post.name == "PostProcessing" + assert post.depends_on is not None + assert post.depends_on[0].task_key == motif_task.task_key + + def test_collapse_no_motifs_returns_unchanged(self): + pipeline = Pipeline( + name="test", + tasks=[_ir_activity("A"), _ir_activity("B", depends_on=["A"])], + ) + result = collapse_motifs(pipeline, []) + assert len(result.tasks) == 2 + assert result.tasks[0].name == "A" + + def test_collapse_preserves_unclaimed_activities(self): + pipeline = Pipeline( + name="test", + tasks=[ + _ir_activity("Unclaimed1"), + _ir_activity("GetList"), + _ir_activity("Loop", depends_on=["GetList"]), + _ir_activity("Unclaimed2", depends_on=["Loop"]), + ], + ) + from flowx.models.motifs import DetectedMotif + + detected = DetectedMotif( + definition=MOTIF_METADATA_DRIVEN_BULK_COPY, + matched_activities=["GetList", "Loop"], + ) + result = collapse_motifs(pipeline, [detected]) + + names = [t.name for t in result.tasks] + assert "Unclaimed1" in names + assert "Unclaimed2" in names + motif_tasks = [t for t in result.tasks if isinstance(t, MotifActivity)] + assert len(motif_tasks) == 1 + + def test_collapse_handles_activity_name_distinct_from_task_key(self): + """Regression: motif collapser must compare against sanitised task_keys. + + Activity names with spaces (or other characters that get sanitised + in task_keys) used to confuse ``_collect_external_dependencies`` and + ``_rewire_dependencies``: both compared raw activity names against + ``Dependency.task_key`` (which is sanitised by the translator), so + internal motif edges leaked through as "external" deps and + downstream rewires never matched. + """ + from flowx.models.motifs import DetectedMotif + + pipeline = Pipeline( + name="test", + tasks=[ + Activity(name="Setup Probe", task_key="Setup_Probe"), + Activity( + name="Get Table List", + task_key="Get_Table_List", + depends_on=[Dependency(task_key="Setup_Probe")], + ), + Activity( + name="For Each Table", + task_key="For_Each_Table", + depends_on=[Dependency(task_key="Get_Table_List")], + ), + Activity( + name="Notify Done", + task_key="Notify_Done", + depends_on=[Dependency(task_key="For_Each_Table")], + ), + ], + ) + detected = DetectedMotif( + definition=MOTIF_METADATA_DRIVEN_BULK_COPY, + matched_activities=["Get Table List", "For Each Table"], + source_type_hint="database", + ) + result = collapse_motifs(pipeline, [detected]) + + motif = next(t for t in result.tasks if isinstance(t, MotifActivity)) + external_keys = {dep.task_key for dep in motif.depends_on or []} + assert external_keys == {"Setup_Probe"}, ( + "Internal edge GetTableList -> ForEachTable should NOT appear as an external dep" + ) + + notify = next(t for t in result.tasks if t.name == "Notify Done") + downstream_keys = {dep.task_key for dep in notify.depends_on or []} + assert downstream_keys == {motif.task_key}, ( + f"Downstream task should be rewired to point at the motif, got {downstream_keys}" + ) diff --git a/tests/unit/test_preparers.py b/tests/unit/test_preparers.py new file mode 100644 index 0000000..891a497 --- /dev/null +++ b/tests/unit/test_preparers.py @@ -0,0 +1,776 @@ +"""Unit tests for activity preparers and the workflow_preparer module.""" + +from __future__ import annotations + +import ast +from typing import Any + +import pytest + +from flowx.models.ir import ( + AppendVariableActivity, + CopyActivity, + DeleteActivity, + Dependency, + ExecutePipelineActivity, + FilterActivity, + ForEachActivity, + LookupActivity, + NotebookActivity, + Pipeline, + PlaceholderActivity, + RunJobActivity, + SetVariableActivity, + SparkJarActivity, + SparkPythonActivity, + SwitchActivity, + SwitchCase, + WaitActivity, + WebActivity, +) +from flowx.preparer.workflow_preparer import ( + PreparedActivity, + PreparedWorkflow, + prepare_activity, + prepare_workflow, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_base(name: str = "test", task_key: str = "test") -> dict[str, Any]: + return { + "name": name, + "task_key": task_key, + "description": None, + "timeout_seconds": None, + "max_retries": None, + "min_retry_interval_millis": None, + "depends_on": None, + "cluster": None, + } + + +# --------------------------------------------------------------------------- +# Individual activity preparer tests +# --------------------------------------------------------------------------- + + +class TestNotebookPreparer: + def test_prepare_notebook_task_structure(self): + activity = NotebookActivity( + **_make_base("Run NB", "run_nb"), + notebook_path="/Shared/ETL/transform", + base_parameters={"env": "dev"}, + ) + prepared = prepare_activity(activity) + assert isinstance(prepared, PreparedActivity) + assert prepared.task["task_key"] == "run_nb" + assert "notebook_task" in prepared.task + # Existing notebook at an absolute workspace path is referenced + # in place -- no placeholder is synthesized into the bundle src + # tree because the notebook already exists in the workspace. + assert prepared.task["notebook_task"]["notebook_path"] == "/Shared/ETL/transform" + assert prepared.task["notebook_task"]["base_parameters"] == {"env": "dev"} + assert prepared.notebooks == [] + + def test_prepare_notebook_no_params(self): + activity = NotebookActivity( + **_make_base("NB", "nb"), + notebook_path="/Shared/simple", + base_parameters=None, + ) + prepared = prepare_activity(activity) + assert "base_parameters" not in prepared.task.get("notebook_task", {}) + # No placeholder for an absolute workspace path. + assert prepared.notebooks == [] + + def test_prepare_notebook_resolves_expression_params(self): + """ADF expression params are mapped to DAB dynamic value references.""" + activity = NotebookActivity( + **_make_base("Expr NB", "expr_nb"), + notebook_path="/Shared/nb", + base_parameters={ + "run_id": "@pipeline().RunId", + "env": "dev", + "trigger_time": {"type": "Expression", "value": "@pipeline().TriggerTime"}, + }, + ) + prepared = prepare_activity(activity) + params = prepared.task["notebook_task"]["base_parameters"] + assert params["run_id"] == "{{job.run_id}}" + assert params["env"] == "dev" + assert params["trigger_time"] == "{{job.start_time.iso_datetime}}" + + +class TestCopyPreparer: + def test_prepare_copy_generates_notebook(self): + activity = CopyActivity( + **_make_base("Copy Data", "copy_data"), + source_type="BlobSource", + sink_type="DeltaSink", + ) + prepared = prepare_activity(activity) + assert "notebook_task" in prepared.task + assert len(prepared.notebooks) == 1 + assert prepared.notebooks[0].relative_path == "notebooks/copy_data.py" + # Verify generated content is valid Python + content = prepared.notebooks[0].content + assert "Databricks notebook source" in content + + def test_prepare_copy_db_source_creates_secrets(self): + activity = CopyActivity( + **_make_base("Copy SQL", "copy_sql"), + source_type="AzureSqlSource", + sink_type="DeltaSink", + ) + prepared = prepare_activity(activity) + assert len(prepared.secrets) >= 2 + scopes = {s.key for s in prepared.secrets} + assert "jdbc-url" in scopes + assert "jdbc-password" in scopes + + +class TestSparkJarPreparer: + def test_prepare_spark_jar_task(self): + activity = SparkJarActivity( + **_make_base("Jar Task", "jar_task"), + main_class_name="com.example.Main", + parameters=["--arg1"], + libraries=[{"jar": "dbfs:/libs/my.jar"}], + ) + prepared = prepare_activity(activity) + assert "spark_jar_task" in prepared.task + assert prepared.task["spark_jar_task"]["main_class_name"] == "com.example.Main" + # Libraries are rewritten to bundle-relative paths + assert prepared.task["libraries"] == [{"jar": "../lib/my.jar"}] + # Placeholder readme is generated + assert len(prepared.notebooks) == 1 + assert "jar_task_README.txt" in prepared.notebooks[0].relative_path + + +class TestSparkPythonPreparer: + def test_prepare_spark_python_task(self): + activity = SparkPythonActivity( + **_make_base("Py Task", "py_task"), + python_file="dbfs:/scripts/etl.py", + parameters=["--mode", "batch"], + ) + prepared = prepare_activity(activity) + assert "spark_python_task" in prepared.task + # Path is rewritten to bundle-relative + assert prepared.task["spark_python_task"]["python_file"] == "../src/scripts/etl.py" + # Placeholder script is generated + assert len(prepared.notebooks) == 1 + assert "scripts/etl.py" in prepared.notebooks[0].relative_path + assert "dbfs:/scripts/etl.py" in prepared.notebooks[0].content + + +class TestLookupPreparer: + def test_prepare_lookup_generates_notebook(self): + activity = LookupActivity( + **_make_base("Lookup", "lookup"), + source_type="AzureSqlSource", + first_row_only=True, + source_query="SELECT 1", + ) + prepared = prepare_activity(activity) + assert len(prepared.notebooks) == 1 + assert "notebook_task" in prepared.task + assert prepared.task["notebook_task"]["base_parameters"]["first_row_only"] == "true" + + +class TestWebActivityPreparer: + def test_prepare_web_activity_generates_notebook(self): + activity = WebActivity( + **_make_base("Call API", "call_api"), + url="https://api.example.com", + method="GET", + ) + prepared = prepare_activity(activity) + assert len(prepared.notebooks) == 1 + assert "notebook_task" in prepared.task + assert prepared.task["notebook_task"]["base_parameters"]["url"] == "https://api.example.com" + assert prepared.task["notebook_task"]["base_parameters"]["method"] == "GET" + + def test_prepare_web_activity_with_auth_creates_secrets(self): + activity = WebActivity( + **_make_base("Auth API", "auth_api"), + url="https://api.example.com", + method="POST", + authentication={"type": "ServicePrincipal"}, + ) + prepared = prepare_activity(activity) + assert len(prepared.secrets) >= 1 + assert any(s.key == "auth-credential" for s in prepared.secrets) + + +class TestDeletePreparer: + def test_prepare_delete_generates_notebook(self): + activity = DeleteActivity( + **_make_base("Delete Files", "delete_files"), + dataset_name="ds_staging", + recursive=True, + ) + prepared = prepare_activity(activity) + assert len(prepared.notebooks) == 1 + assert "notebook_task" in prepared.task + + +class TestSetVariablePreparer: + def test_prepare_set_variable_generates_notebook(self): + activity = SetVariableActivity( + **_make_base("Set Var", "set_var"), + variable_name="status", + variable_value="completed", + value_kind="literal", + ) + prepared = prepare_activity(activity) + assert "notebook_task" in prepared.task + assert len(prepared.notebooks) == 1 + # Literal value should be in base_parameters + params = prepared.task["notebook_task"]["base_parameters"] + assert params["value"] == "completed" + + def test_prepare_set_variable_dab_ref(self): + activity = SetVariableActivity( + **_make_base("Set Env", "set_env"), + variable_name="env", + variable_value="{{job.parameters.environment}}", + value_kind="dab_ref", + ) + prepared = prepare_activity(activity) + params = prepared.task["notebook_task"]["base_parameters"] + assert params["value"] == "{{job.parameters.environment}}" + + def test_prepare_set_variable_notebook_code_not_in_params(self): + """notebook_code values must NOT appear in base_parameters.""" + activity = SetVariableActivity( + **_make_base("Set Date", "set_date"), + variable_name="runDate", + variable_value="datetime.now(timezone.utc).strftime('%Y-%m-%d')", + value_kind="notebook_code", + notebook_code="datetime.now(timezone.utc).strftime('%Y-%m-%d')", + notebook_imports=["from datetime import datetime, timezone"], + ) + prepared = prepare_activity(activity) + params = prepared.task["notebook_task"]["base_parameters"] + # Should NOT have 'value' key with Python code + assert "value" not in params + # But should have variable_name + assert params["variable_name"] == "runDate" + # Notebook should contain the code + content = prepared.notebooks[0].content + assert "strftime" in content + assert "datetime" in content + + +class TestAppendVariablePreparer: + def test_prepare_append_variable_generates_notebook(self): + activity = AppendVariableActivity( + **_make_base("Append Var", "append_var"), + variable_name="logEntries", + append_value="step1 done", + value_kind="literal", + ) + prepared = prepare_activity(activity) + assert "notebook_task" in prepared.task + assert len(prepared.notebooks) == 1 + params = prepared.task["notebook_task"]["base_parameters"] + assert params["value"] == "step1 done" + + def test_prepare_append_variable_notebook_code_not_in_params(self): + """notebook_code values must NOT appear in base_parameters.""" + activity = AppendVariableActivity( + **_make_base("Append TS", "append_ts"), + variable_name="timestamps", + append_value="datetime.now(timezone.utc).isoformat()", + value_kind="notebook_code", + notebook_code="datetime.now(timezone.utc).isoformat()", + notebook_imports=["from datetime import datetime, timezone"], + ) + prepared = prepare_activity(activity) + params = prepared.task["notebook_task"]["base_parameters"] + assert "value" not in params + + +class TestFilterPreparer: + def test_prepare_filter_generates_notebook(self): + activity = FilterActivity( + **_make_base("Filter Items", "filter_items"), + items_expression="@variables('myList')", + condition_expression="@not(empty(item()))", + ) + prepared = prepare_activity(activity) + assert "notebook_task" in prepared.task + assert len(prepared.notebooks) == 1 + + +class TestWaitPreparer: + def test_prepare_wait_generates_notebook(self): + activity = WaitActivity( + **_make_base("Wait 30s", "wait_30s"), + wait_time_seconds=30, + ) + prepared = prepare_activity(activity) + assert "notebook_task" in prepared.task + assert len(prepared.notebooks) == 1 + assert prepared.task["notebook_task"]["base_parameters"]["wait_seconds"] == "30" + + +class TestForEachPreparer: + def test_prepare_for_each_wraps_inner(self): + inner = WaitActivity(**_make_base("Inner", "inner"), wait_time_seconds=1) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[inner], + concurrency=10, + ) + prepared = prepare_activity(activity) + assert "for_each_task" in prepared.task + assert prepared.task["for_each_task"]["concurrency"] == 10 + assert prepared.task["for_each_task"]["inputs"] == "@output.value" + + +class TestExecutePipelinePreparer: + def test_prepare_execute_pipeline_task(self): + activity = ExecutePipelineActivity( + **_make_base("Run Child", "run_child"), + pipeline_name="child_pipeline", + parameters={"date": "2024-01-01"}, + wait_on_completion=True, + ) + prepared = prepare_activity(activity) + assert "run_job_task" in prepared.task + + +class TestRunJobPreparer: + def test_prepare_databricks_job_task(self): + activity = RunJobActivity( + **_make_base("Run Job", "run_job"), + job_name="nightly-agg", + existing_job_id="12345", + ) + prepared = prepare_activity(activity) + assert "run_job_task" in prepared.task + + def test_run_job_round_trips_through_translation_report_json(self, tmp_path): + """RunJobActivity.job_parameters survive the JSON serialise/reload cycle. + + Regression: the engine used to omit ``job_parameters`` from the + serialised IR, and dab_writer's reload path read ``parameters`` (which + only ExecutePipelineActivity emits), silently dropping every + RunJobActivity's parameters when bundles were produced via the CLI + ``--report`` flow. + """ + import json + + import yaml + + from flowx.bundler.dab_writer import _load_report, write_bundle + from flowx.translator.engine import _activity_to_dict, _pipeline_to_dict + + run_job = RunJobActivity( + **_make_base("Nightly Aggregator", "nightly_aggregator"), + job_name="nightly-agg", + existing_job_id="12345", + job_parameters={"window_start": "2024-01-01", "table": "orders"}, + ) + pipeline = Pipeline(name="rj_pipeline", tasks=[run_job]) + pipeline_dict = _pipeline_to_dict(pipeline) + # Sanity: serialiser must include job_parameters. + run_job_dict = next(t for t in pipeline_dict["tasks"] if t["task_key"] == "nightly_aggregator") + assert run_job_dict["job_parameters"] == {"window_start": "2024-01-01", "table": "orders"} + assert _activity_to_dict(run_job)["job_parameters"] == run_job.job_parameters + + report_path = tmp_path / "rj.json" + report_path.write_text(json.dumps(pipeline_dict)) + workflows = _load_report(report_path) + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + write_bundle(workflows[0], bundle_dir) + + job_yml = next((bundle_dir / "resources").iterdir()) + job = yaml.safe_load(job_yml.read_text()) + task = job["resources"]["jobs"]["rj_pipeline"]["tasks"][0] + assert task["run_job_task"]["job_parameters"] == { + "window_start": "2024-01-01", + "table": "orders", + } + + +class TestMotifPreparer: + def test_motif_preparer_registered(self): + """prepare_activity dispatches MotifActivity to the motif preparer. + + Regression: the motif system was added without a registered preparer, + so any pipeline whose translation matched a motif raised + ``ValueError("No preparer registered for activity type MotifActivity")`` + when prepare_workflow ran. + """ + from flowx.models.ir import MotifActivity + + activity = MotifActivity( + **_make_base("Bulk Ingest", "bulk_ingest"), + motif_id="metadata_driven_bulk_copy", + display_name="Metadata-driven bulk copy", + databricks_replacement="for_each_ingestion", + matched_activity_names=["GetTableList", "ForEachTable", "CopyTable"], + source_type_hint="database", + confidence_notes=["Lookup feeds ForEach feeds Copy"], + original_activities=[], + motif_config={"sink_table": "raw.{schema_name}_{table_name}"}, + ) + prepared = prepare_activity(activity) + assert "notebook_task" in prepared.task + assert prepared.task["notebook_task"]["notebook_path"].endswith("bulk_ingest.py") + assert len(prepared.notebooks) == 1 + assert "metadata_driven_bulk_copy" in prepared.notebooks[0].content + + +class TestSwitchPreparer: + def test_prepare_switch_generates_condition_task(self): + inner = WaitActivity(**_make_base("CaseWait", "case_wait"), wait_time_seconds=1) + default_inner = WaitActivity(**_make_base("DefaultWait", "default_wait"), wait_time_seconds=2) + activity = SwitchActivity( + **_make_base("Route", "route"), + on_expression="@item().type", + cases=[SwitchCase(value="full", activities=[inner])], + default_activities=[default_inner], + ) + prepared = prepare_activity(activity) + # The main task is the first case condition: ``_case_``. + # ``task_key_remap`` rewires upstream depends_on edges from the + # original switch key to this new key. + assert prepared.task["task_key"] == "route_case_full" + assert prepared.task_key_remap == {"route": "route_case_full"} + cond = prepared.task["condition_task"] + assert cond["op"] == "EQUAL_TO" + assert "left" in cond + assert cond["right"] == "full" + assert "if_true" not in cond + assert "if_false" not in cond + # Case body is a sibling task gated on outcome="true". + case_task_keys = {task["task_key"] for task in prepared.extra_tasks} + assert "case_wait" in case_task_keys + case_task = next(task for task in prepared.extra_tasks if task["task_key"] == "case_wait") + assert case_task["depends_on"] == [{"task_key": "route_case_full", "outcome": "true"}] + # Default body is gated on outcome="false" of the last case. + default_task = next(task for task in prepared.extra_tasks if task["task_key"] == "default_wait") + assert default_task["depends_on"] == [{"task_key": "route_case_full", "outcome": "false"}] + + def test_prepare_switch_multi_case_chains_conditions(self): + """Each case becomes a chained condition task linked by outcome=false.""" + inner1 = WaitActivity(**_make_base("Wait1", "wait1"), wait_time_seconds=1) + inner2 = WaitActivity(**_make_base("Wait2", "wait2"), wait_time_seconds=2) + inner3 = WaitActivity(**_make_base("Wait3", "wait3"), wait_time_seconds=3) + default = WaitActivity(**_make_base("Default", "default_wait"), wait_time_seconds=5) + activity = SwitchActivity( + **_make_base("Route", "route"), + on_expression="@pipeline().parameters.env", + cases=[ + SwitchCase(value="dev", activities=[inner1]), + SwitchCase(value="staging", activities=[inner2]), + SwitchCase(value="prod", activities=[inner3]), + ], + default_activities=[default], + ) + prepared = prepare_activity(activity) + # First case condition is the main task: ``route_case_dev``. No + # if_true/if_false nesting -- branches hang off as siblings. + assert prepared.task["task_key"] == "route_case_dev" + cond = prepared.task["condition_task"] + assert cond["right"] == "dev" + assert "if_true" not in cond and "if_false" not in cond + # Subsequent case conditions live as siblings, chained via outcome=false. + extra_by_key = {task["task_key"]: task for task in prepared.extra_tasks} + assert "route_case_staging" in extra_by_key + assert "route_case_prod" in extra_by_key + assert extra_by_key["route_case_staging"]["depends_on"] == [{"task_key": "route_case_dev", "outcome": "false"}] + assert extra_by_key["route_case_prod"]["depends_on"] == [{"task_key": "route_case_staging", "outcome": "false"}] + # Case bodies hang off their own condition with outcome=true. + assert extra_by_key["wait1"]["depends_on"] == [{"task_key": "route_case_dev", "outcome": "true"}] + assert extra_by_key["wait2"]["depends_on"] == [{"task_key": "route_case_staging", "outcome": "true"}] + assert extra_by_key["wait3"]["depends_on"] == [{"task_key": "route_case_prod", "outcome": "true"}] + # Default hangs off the last case's outcome=false. + assert extra_by_key["default_wait"]["depends_on"] == [{"task_key": "route_case_prod", "outcome": "false"}] + + def test_prepare_switch_resolves_variables_expression(self): + """Switch on @variables('x') resolves to a DAB task value ref.""" + inner = WaitActivity(**_make_base("CaseWait", "case_wait"), wait_time_seconds=1) + activity = SwitchActivity( + **_make_base("Route", "route"), + on_expression="@variables('sourceType')", + cases=[SwitchCase(value="SQL", activities=[inner])], + default_activities=[], + ) + prepared = prepare_activity(activity) + cond = prepared.task["condition_task"] + # Should be resolved to a DAB ref (fallback: variable name used as task key) + assert "tasks." in cond["left"] + assert "sourceType" in cond["left"] + + def test_prepare_switch_resolves_pipeline_param(self): + """Switch on @pipeline().parameters.X resolves to a DAB job parameter ref.""" + inner = WaitActivity(**_make_base("CaseWait", "case_wait"), wait_time_seconds=1) + activity = SwitchActivity( + **_make_base("Route", "route"), + on_expression="@pipeline().parameters.env", + cases=[SwitchCase(value="dev", activities=[inner])], + default_activities=[], + ) + prepared = prepare_activity(activity) + cond = prepared.task["condition_task"] + assert cond["left"] == "{{job.parameters.env}}" + + def test_reload_path_resolves_unresolved_on_expression(self, tmp_path): + """dab_writer's reload path resolves a leftover @variables() before emitting YAML. + + Regression: ``_handle_switch`` used to read ``on_expression`` straight + from the IR JSON, so a hand-edited or future-translator-produced IR + carrying an unresolved ``@variables(...)`` leaked the raw ADF syntax + into ``condition_task.left``, which ``databricks bundle validate`` + would reject. + """ + import json + + import yaml + + from flowx.bundler.dab_writer import _load_report, write_bundle + + pipeline_dict = { + "name": "switch_pipeline", + "tasks": [ + { + "type": "SwitchActivity", + "name": "Route", + "task_key": "route", + "on_expression": "@pipeline().parameters.env", + "cases": [ + { + "value": "dev", + "activities": [ + { + "type": "WaitActivity", + "name": "Wait", + "task_key": "wait", + "wait_time_seconds": 1, + } + ], + } + ], + "default_activities": [], + } + ], + } + report_path = tmp_path / "switch.json" + report_path.write_text(json.dumps(pipeline_dict)) + workflows = _load_report(report_path) + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + write_bundle(workflows[0], bundle_dir) + + job_yml = next((bundle_dir / "resources").iterdir()) + job = yaml.safe_load(job_yml.read_text()) + switch_task = next(t for t in job["resources"]["jobs"]["switch_pipeline"]["tasks"] if "condition_task" in t) + assert switch_task["condition_task"]["left"] == "{{job.parameters.env}}" + assert switch_task["task_key"].endswith("_case_dev") + + +class TestInjectOutcomeDependency: + def test_preserves_external_dependencies(self): + """Regression: branch tasks with external deps keep them after gating. + + Previously ``inject_outcome_dependency`` clobbered ``depends_on`` with + a single outcome edge whenever the task didn't depend on a sibling, + silently dropping any dependency on a task outside the branch (e.g. + a global setup task). + """ + from flowx.preparer.activity_preparers.if_condition import inject_outcome_dependency + + external_dep = {"task_key": "global_setup"} + branch_tasks = [ + {"task_key": "branch_root", "depends_on": [external_dep]}, + {"task_key": "branch_child", "depends_on": [{"task_key": "branch_root"}]}, + ] + inject_outcome_dependency(branch_tasks, "if_check", "true") + + root_deps = branch_tasks[0]["depends_on"] + assert {"task_key": "if_check", "outcome": "true"} in root_deps + assert external_dep in root_deps, "external dep lost when gating branch root" + # Internal child task still depends only on its sibling. + assert branch_tasks[1]["depends_on"] == [{"task_key": "branch_root"}] + + def test_idempotent(self): + """Calling twice with the same outcome doesn't duplicate the edge.""" + from flowx.preparer.activity_preparers.if_condition import inject_outcome_dependency + + branch_tasks = [{"task_key": "branch_root"}] + inject_outcome_dependency(branch_tasks, "if_check", "true") + inject_outcome_dependency(branch_tasks, "if_check", "true") + assert branch_tasks[0]["depends_on"] == [{"task_key": "if_check", "outcome": "true"}] + + +class TestPlaceholderPreparer: + def test_prepare_placeholder_generates_stub(self): + activity = PlaceholderActivity( + **_make_base("Unknown Act", "unknown_act"), + original_type="SomeFutureType", + comment="This activity requires manual implementation.", + ) + prepared = prepare_activity(activity) + assert "notebook_task" in prepared.task + assert len(prepared.notebooks) == 1 + assert "NotImplementedError" in prepared.notebooks[0].content + + +# --------------------------------------------------------------------------- +# Notebook content validity +# --------------------------------------------------------------------------- + + +class TestNotebookContentValidity: + """Verify that generated notebooks are syntactically valid Python.""" + + def _get_all_notebooks(self) -> list[tuple[str, str]]: + """Build all activity types and collect generated notebooks.""" + activities = [ + CopyActivity(**_make_base("c", "c"), source_type="BlobSource", sink_type="DeltaSink"), + LookupActivity(**_make_base("l", "l"), source_type="AzureSqlSource", first_row_only=True), + WebActivity(**_make_base("w", "w"), url="https://example.com", method="GET"), + DeleteActivity(**_make_base("d", "d"), dataset_name="ds", recursive=True), + SetVariableActivity( + **_make_base("sv", "sv"), variable_name="x", variable_value="completed", value_kind="literal" + ), + AppendVariableActivity( + **_make_base("av", "av"), variable_name="arr", append_value="step1_done", value_kind="literal" + ), + FilterActivity( + **_make_base("f", "f"), items_expression="@vars('list')", condition_expression="@not(empty(item()))" + ), + WaitActivity(**_make_base("wait", "wait"), wait_time_seconds=5), + PlaceholderActivity(**_make_base("ph", "ph"), original_type="FutureType", comment="TODO"), + ] + results = [] + for act in activities: + prepared = prepare_activity(act) + for nb in prepared.notebooks: + results.append((nb.relative_path, nb.content)) + return results + + def test_all_generated_notebooks_are_valid_python(self): + """Every generated notebook passes ast.parse without SyntaxError.""" + notebooks = self._get_all_notebooks() + assert len(notebooks) > 0, "Expected at least some generated notebooks" + for path, content in notebooks: + # Strip Databricks magic comments for Python parsing + lines = [] + for line in content.split("\n"): + stripped = line.lstrip() + if stripped.startswith("# MAGIC") or stripped.startswith("# COMMAND"): + continue + if stripped == "# Databricks notebook source": + continue + lines.append(line) + python_code = "\n".join(lines) + try: + ast.parse(python_code) + except SyntaxError as exc: + pytest.fail(f"Notebook {path} has invalid Python syntax: {exc}") + + +# --------------------------------------------------------------------------- +# prepare_workflow (full pipeline) +# --------------------------------------------------------------------------- + + +class TestPrepareWorkflow: + def test_prepare_workflow_aggregates(self): + pipeline = Pipeline( + name="test_pipeline", + tasks=[ + CopyActivity(**_make_base("Copy", "copy"), source_type="BlobSource", sink_type="DeltaSink"), + WaitActivity(**_make_base("Wait", "wait"), wait_time_seconds=10), + NotebookActivity(**_make_base("NB", "nb"), notebook_path="/Shared/nb"), + ], + ) + wf = prepare_workflow(pipeline) + assert isinstance(wf, PreparedWorkflow) + assert wf.name == "test_pipeline" + assert len(wf.tasks) == 3 + # Copy and Wait synthesize bundled notebooks; the NotebookActivity + # references an existing absolute workspace path so it does not + # contribute a bundle artifact. + assert len(wf.notebooks) == 2 + relative_paths = {nb.relative_path for nb in wf.notebooks} + assert relative_paths == {"notebooks/copy.py", "notebooks/wait.py"} + + def test_prepare_workflow_deduplicates_secrets(self): + """Duplicate secrets across tasks are deduplicated.""" + pipeline = Pipeline( + name="dup_secrets", + tasks=[ + CopyActivity(**_make_base("C1", "c1"), source_type="AzureSqlSource", sink_type="DeltaSink"), + CopyActivity(**_make_base("C2", "c2"), source_type="AzureSqlSource", sink_type="DeltaSink"), + ], + ) + wf = prepare_workflow(pipeline) + # Each copy generates its own scope, but within each scope secrets are deduped + scope_keys = [(s.scope, s.key) for s in wf.secrets] + assert len(scope_keys) == len(set(scope_keys)), "Secret (scope, key) pairs should be unique" + + def test_prepare_workflow_task_keys_unique(self): + """Every task has a unique task_key.""" + pipeline = Pipeline( + name="unique_keys", + tasks=[ + WaitActivity(**_make_base("A", "a"), wait_time_seconds=1), + WaitActivity(**_make_base("B", "b"), wait_time_seconds=2), + WaitActivity(**_make_base("C", "c"), wait_time_seconds=3), + ], + ) + wf = prepare_workflow(pipeline) + task_keys = [t["task_key"] for t in wf.tasks] + assert len(task_keys) == len(set(task_keys)) + + def test_prepare_workflow_with_dependencies(self): + """Dependencies are preserved in prepared tasks.""" + pipeline = Pipeline( + name="deps", + tasks=[ + WaitActivity(**_make_base("First", "first"), wait_time_seconds=1), + WaitActivity( + **{**_make_base("Second", "second"), "depends_on": [Dependency(task_key="first")]}, + wait_time_seconds=2, + ), + ], + ) + wf = prepare_workflow(pipeline) + second_task = next(t for t in wf.tasks if t["task_key"] == "second") + assert "depends_on" in second_task + assert second_task["depends_on"][0]["task_key"] == "first" + + def test_prepare_workflow_with_retries(self): + """Retry settings are carried through.""" + pipeline = Pipeline( + name="retries", + tasks=[ + WaitActivity( + name="Retry Me", + task_key="retry_me", + timeout_seconds=3600, + max_retries=3, + min_retry_interval_millis=60000, + wait_time_seconds=10, + ), + ], + ) + wf = prepare_workflow(pipeline) + task = wf.tasks[0] + assert task["timeout_seconds"] == 3600 + assert task["max_retries"] == 3 + assert task["min_retry_interval_millis"] == 60000 + assert task["retry_on_timeout"] is True diff --git a/tests/unit/test_resolve_field.py b/tests/unit/test_resolve_field.py new file mode 100644 index 0000000..a025c81 --- /dev/null +++ b/tests/unit/test_resolve_field.py @@ -0,0 +1,149 @@ +"""Unit tests for the resolve.py helpers (resolve_field, resolve_field_int, resolve_dict_values).""" + +from __future__ import annotations + +from types import MappingProxyType + +from flowx.models.ir import TranslationContext +from flowx.translator.activity_translators.resolve import ( + resolve_dict_values, + resolve_field, + resolve_field_int, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _ctx(**variable_mappings: str) -> TranslationContext: + """Build a context with optional variable -> task_key mappings.""" + vc = MappingProxyType(variable_mappings) if variable_mappings else MappingProxyType({}) + return TranslationContext(variable_cache=vc) + + +# --------------------------------------------------------------------------- +# resolve_field +# --------------------------------------------------------------------------- + + +class TestResolveField: + def test_none_returns_empty_string(self): + assert resolve_field(None, _ctx()) == "" + + def test_plain_string_literal(self): + assert resolve_field("hello", _ctx()) == "hello" + + def test_integer_literal(self): + result = resolve_field(42, _ctx()) + assert result == "42" + + def test_expression_dict_pipeline_param(self): + value = {"type": "Expression", "value": "@pipeline().parameters.env"} + result = resolve_field(value, _ctx()) + assert result == "{{job.parameters.env}}" + + def test_pipeline_run_id(self): + result = resolve_field("@pipeline().RunId", _ctx()) + assert result == "{{job.run_id}}" + + def test_interpolation_string(self): + result = resolve_field("@{pipeline().parameters.env}", _ctx()) + assert "env" in result + + def test_expression_dict_with_unsupported_expression(self): + """Unsupported expression dict falls back to the raw value string.""" + value = {"type": "Expression", "value": "@dataUri('hello')"} + result = resolve_field(value, _ctx()) + assert result == "@dataUri('hello')" + + def test_non_expression_dict_returns_str(self): + """Non-expression dict without resolve falls back to str.""" + value = {"type": "Other"} + result = resolve_field(value, _ctx()) + assert isinstance(result, str) + + def test_activity_output_ref(self): + result = resolve_field("@activity('Lookup').output.firstRow.cnt", _ctx()) + assert "tasks.Lookup.values.cnt" in result + + def test_boolean_value(self): + result = resolve_field(True, _ctx()) + assert result == "True" + + def test_variables_with_context(self): + result = resolve_field("@variables('runDate')", _ctx(runDate="SetRunDate")) + assert "tasks.SetRunDate.values.runDate" in result + + +# --------------------------------------------------------------------------- +# resolve_field_int +# --------------------------------------------------------------------------- + + +class TestResolveFieldInt: + def test_integer_value(self): + assert resolve_field_int(42, _ctx()) == 42 + + def test_integer_as_string(self): + assert resolve_field_int("100", _ctx()) == 100 + + def test_expression_resolving_to_literal(self): + """A literal string that happens to be an integer.""" + assert resolve_field_int("5", _ctx()) == 5 + + def test_non_numeric_returns_default(self): + assert resolve_field_int("not_a_number", _ctx()) == 0 + + def test_custom_default(self): + assert resolve_field_int("not_a_number", _ctx(), default=99) == 99 + + def test_none_returns_default(self): + # resolve_field(None) returns "", int("") fails, so default is returned + assert resolve_field_int(None, _ctx()) == 0 + + def test_expression_dict_non_numeric(self): + """Expression dict that resolves to a DAB ref (non-numeric) returns default.""" + value = {"type": "Expression", "value": "@pipeline().parameters.env"} + assert resolve_field_int(value, _ctx(), default=10) == 10 + + +# --------------------------------------------------------------------------- +# resolve_dict_values +# --------------------------------------------------------------------------- + + +class TestResolveDictValues: + def test_none_returns_empty_dict(self): + assert resolve_dict_values(None, _ctx()) == {} + + def test_empty_dict_returns_empty(self): + assert resolve_dict_values({}, _ctx()) == {} + + def test_literal_values(self): + result = resolve_dict_values({"env": "dev", "mode": "batch"}, _ctx()) + assert result == {"env": "dev", "mode": "batch"} + + def test_mixed_literal_and_expression(self): + result = resolve_dict_values( + { + "env": "dev", + "run_id": "@pipeline().RunId", + "date": {"type": "Expression", "value": "@pipeline().parameters.date"}, + }, + _ctx(), + ) + assert result["env"] == "dev" + assert result["run_id"] == "{{job.run_id}}" + assert result["date"] == "{{job.parameters.date}}" + + def test_all_expressions(self): + result = resolve_dict_values( + { + "run_id": "@pipeline().RunId", + "name": "@pipeline().Pipeline", + }, + _ctx(), + ) + assert result["run_id"] == "{{job.run_id}}" + assert result["name"] == "{{job.name}}" diff --git a/tests/unit/test_translators.py b/tests/unit/test_translators.py new file mode 100644 index 0000000..d213604 --- /dev/null +++ b/tests/unit/test_translators.py @@ -0,0 +1,642 @@ +"""Unit tests for individual activity translators and the translation engine.""" + +from __future__ import annotations + +from types import MappingProxyType +from typing import Any + +from flowx.models.adf_ast import ( + AdfActivity, + AdfDatasetReference, + AdfDefinitions, + AdfDependency, + AdfLinkedServiceReference, + AdfPolicy, +) +from flowx.models.ir import ( + AppendVariableActivity, + CopyActivity, + DeleteActivity, + ExecutePipelineActivity, + FilterActivity, + ForEachActivity, + IfConditionActivity, + LookupActivity, + NotebookActivity, + Pipeline, + PlaceholderActivity, + RunJobActivity, + SetVariableActivity, + SparkJarActivity, + SparkPythonActivity, + SwitchActivity, + TranslationContext, + WaitActivity, + WebActivity, +) +from flowx.translator.engine import translate_pipeline + +_EMPTY_DEFS = AdfDefinitions(pipelines=[], datasets={}, linked_services={}, triggers=[]) + + +def _base_kwargs(name: str = "test_activity") -> dict[str, Any]: + """Minimal base_kwargs for translator functions.""" + return { + "name": name, + "task_key": name.replace(" ", "_"), + "description": None, + "timeout_seconds": None, + "max_retries": None, + "min_retry_interval_millis": None, + "depends_on": None, + "cluster": None, + } + + +def _context() -> TranslationContext: + return TranslationContext( + activity_cache=MappingProxyType({}), + registry=MappingProxyType({}), + variable_cache=MappingProxyType({}), + ) + + +def _make_activity( + name: str, + adf_type: str, + type_properties: dict[str, Any] | None = None, + *, + depends_on: list[AdfDependency] | None = None, + inputs: list[AdfDatasetReference] | None = None, + outputs: list[AdfDatasetReference] | None = None, + linked_service_name: AdfLinkedServiceReference | None = None, + if_true_activities: list[AdfActivity] | None = None, + if_false_activities: list[AdfActivity] | None = None, + activities: list[AdfActivity] | None = None, + policy: AdfPolicy | None = None, +) -> AdfActivity: + return AdfActivity( + name=name, + type=adf_type, + type_properties=type_properties, + depends_on=depends_on, + inputs=inputs, + outputs=outputs, + linked_service_name=linked_service_name, + if_true_activities=if_true_activities, + if_false_activities=if_false_activities, + activities=activities, + policy=policy, + ) + + +class TestCopyTranslator: + def test_translate_copy_basic(self): + from flowx.translator.activity_translators.copy import translate + + activity = _make_activity( + "Copy Data", + "Copy", + { + "source": {"type": "BlobSource", "recursive": True}, + "sink": {"type": "DeltaSink", "writeBatchSize": 10000}, + }, + ) + result = translate(activity, _base_kwargs("Copy Data"), _context(), _EMPTY_DEFS) + assert isinstance(result, CopyActivity) + assert result.source_type == "BlobSource" + assert result.sink_type == "DeltaSink" + assert result.source_properties["recursive"] is True + assert result.sink_properties["writeBatchSize"] == 10000 + + def test_translate_copy_with_column_mapping(self): + from flowx.translator.activity_translators.copy import translate + + activity = _make_activity( + "Copy Mapped", + "Copy", + { + "source": {"type": "AzureSqlSource"}, + "sink": {"type": "DeltaSink"}, + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": {"name": "id", "type": "Int32"}, + "sink": {"name": "id", "type": "Int64"}, + }, + { + "source": {"name": "name", "type": "String"}, + "sink": {"name": "full_name", "type": "String"}, + }, + ], + }, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, CopyActivity) + assert result.column_mapping is not None + assert len(result.column_mapping) == 2 + assert result.column_mapping[0]["source_name"] == "id" + assert result.column_mapping[1]["sink_name"] == "full_name" + + def test_translate_copy_empty_type_properties(self): + from flowx.translator.activity_translators.copy import translate + + activity = _make_activity("Empty Copy", "Copy", {}) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, CopyActivity) + assert result.source_type is None + assert result.sink_type is None + + +class TestNotebookTranslator: + def test_translate_notebook_basic(self): + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/ETL/transform", "baseParameters": {"env": "dev"}}, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.notebook_path == "/Shared/ETL/transform" + assert result.base_parameters == {"env": "dev"} + + def test_translate_notebook_no_params(self): + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/ETL/simple"}, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.base_parameters == {} + + +class TestSparkJarTranslator: + def test_translate_spark_jar(self): + from flowx.translator.activity_translators.spark_jar import translate + + activity = _make_activity( + "Run Jar", + "DatabricksSparkJar", + { + "mainClassName": "com.example.MainJob", + "parameters": ["--input", "/mnt/data"], + "libraries": [{"jar": "dbfs:/libs/my-job.jar"}], + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, SparkJarActivity) + assert result.main_class_name == "com.example.MainJob" + assert result.parameters == ["--input", "/mnt/data"] + assert result.libraries == [{"jar": "dbfs:/libs/my-job.jar"}] + + +class TestSparkPythonTranslator: + def test_translate_spark_python(self): + from flowx.translator.activity_translators.spark_python import translate + + activity = _make_activity( + "Run Python", + "DatabricksSparkPython", + {"pythonFile": "dbfs:/scripts/etl.py", "parameters": ["--mode", "batch"]}, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, SparkPythonActivity) + assert result.python_file == "dbfs:/scripts/etl.py" + assert result.parameters == ["--mode", "batch"] + + +class TestLookupTranslator: + def test_translate_lookup_first_row(self): + from flowx.translator.activity_translators.lookup import translate + + activity = _make_activity( + "Lookup Config", + "Lookup", + { + "source": {"type": "AzureSqlSource", "sqlReaderQuery": "SELECT TOP 1 * FROM config"}, + "firstRowOnly": True, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, LookupActivity) + assert result.first_row_only is True + assert result.source_type == "AzureSqlSource" + assert result.source_query == "SELECT TOP 1 * FROM config" + + def test_translate_lookup_all_rows(self): + from flowx.translator.activity_translators.lookup import translate + + activity = _make_activity( + "Lookup All", + "Lookup", + { + "source": {"type": "AzureSqlSource", "sqlReaderQuery": "SELECT * FROM tables"}, + "firstRowOnly": False, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, LookupActivity) + assert result.first_row_only is False + + +class TestWebActivityTranslator: + def test_translate_web_activity_get(self): + from flowx.translator.activity_translators.web_activity import translate + + activity = _make_activity( + "Call API", + "WebActivity", + {"url": "https://api.example.com/data", "method": "GET"}, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, WebActivity) + assert result.url == "https://api.example.com/data" + assert result.method == "GET" + + def test_translate_web_activity_post(self): + from flowx.translator.activity_translators.web_activity import translate + + activity = _make_activity( + "Post Data", + "WebActivity", + { + "url": "https://api.example.com/submit", + "method": "POST", + "body": {"key": "value"}, + "headers": {"Content-Type": "application/json"}, + "authentication": {"type": "ServicePrincipal", "resource": "https://api.example.com"}, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, WebActivity) + assert result.method == "POST" + assert result.body == {"key": "value"} + assert result.headers == {"Content-Type": "application/json"} + assert result.authentication["type"] == "ServicePrincipal" + + +class TestDeleteTranslator: + def test_translate_delete(self): + from flowx.translator.activity_translators.delete import translate + + activity = _make_activity( + "Delete Files", + "Delete", + {"recursive": True}, + inputs=[AdfDatasetReference(reference_name="ds_staging_folder")], + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, DeleteActivity) + assert result.dataset_name == "ds_staging_folder" + assert result.recursive is True + + +class TestExecutePipelineTranslator: + def test_translate_execute_pipeline(self): + from flowx.translator.activity_translators.execute_pipeline import translate + + activity = _make_activity( + "Run Child", + "ExecutePipeline", + { + "pipeline": {"referenceName": "child_pipeline", "type": "PipelineReference"}, + "parameters": {"date": "2024-01-01"}, + "waitOnCompletion": True, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, ExecutePipelineActivity) + assert result.pipeline_name == "child_pipeline" + assert result.parameters == {"date": "2024-01-01"} + assert result.wait_on_completion is True + + +class TestDatabricksJobTranslator: + def test_translate_databricks_job(self): + from flowx.translator.activity_translators.databricks_job import translate + + activity = _make_activity( + "Run Job", + "DatabricksJob", + {"jobName": "nightly-agg", "jobId": "12345"}, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, RunJobActivity) + assert result.job_name == "nightly-agg" + assert result.existing_job_id == "12345" + + +class TestWaitTranslator: + def test_translate_wait(self): + from flowx.translator.activity_translators.wait import translate + + activity = _make_activity( + "Pause", + "Wait", + {"waitTimeInSeconds": 60}, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, WaitActivity) + assert result.wait_time_seconds == 60 + + def test_translate_wait_defaults_to_zero(self): + from flowx.translator.activity_translators.wait import translate + + activity = _make_activity("Pause", "Wait", {}) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, WaitActivity) + assert result.wait_time_seconds == 0 + + +class TestFilterTranslator: + def test_translate_filter(self): + from flowx.translator.activity_translators.filter import translate + + activity = _make_activity( + "Filter Items", + "Filter", + { + "items": {"type": "Expression", "value": "@variables('myList')"}, + "condition": {"type": "Expression", "value": "@not(empty(item()))"}, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, FilterActivity) + assert result.items_expression is not None + assert result.condition_expression is not None + + def test_translate_filter_lowers_simple_condition(self): + """``@equals(item().X, 'Y')`` lowers to a Python expression with item.get(X).""" + from flowx.translator.activity_translators.filter import translate + + activity = _make_activity( + "Filter Active", + "Filter", + { + "items": {"type": "Expression", "value": "@activity('Lookup').output.value"}, + "condition": {"type": "Expression", "value": "@equals(item().status, 'active')"}, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert result.condition_code is not None + assert "item.get('status')" in result.condition_code + assert "dbutils.widgets.get" not in result.condition_code + + def test_translate_filter_falls_back_to_placeholder_for_unresolvable(self): + """A condition that doesn't lower cleanly leaves condition_code=None.""" + from flowx.translator.activity_translators.filter import translate + + activity = _make_activity( + "Filter Mystery", + "Filter", + { + "items": {"type": "Expression", "value": "@activity('X').output.value"}, + "condition": {"type": "Expression", "value": "@nonexistent_function(item())"}, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert result.condition_code is None + + +class TestForEachTranslator: + def test_translate_foreach_basic(self): + from flowx.translator.activity_translators.for_each import translate + + inner_activity = _make_activity( + "InnerCopy", "Copy", {"source": {"type": "BlobSource"}, "sink": {"type": "DeltaSink"}} + ) + activity = _make_activity( + "Loop Items", + "ForEach", + { + "items": {"type": "Expression", "value": "@activity('GetList').output.value"}, + "isSequential": False, + "batchCount": 5, + }, + activities=[inner_activity], + ) + result, context = translate(activity, _base_kwargs("Loop_Items"), _context(), _EMPTY_DEFS) + assert isinstance(result, ForEachActivity) + # The translator now resolves to a DAB ref + assert result.items_expression == "{{tasks.GetList.values.result}}" + assert result.concurrency == 5 + + def test_translate_foreach_sequential(self): + from flowx.translator.activity_translators.for_each import translate + + inner_activity = _make_activity("InnerWait", "Wait", {"waitTimeInSeconds": 1}) + activity = _make_activity( + "Seq Loop", + "ForEach", + {"items": "@items", "isSequential": True}, + activities=[inner_activity], + ) + result, context = translate(activity, _base_kwargs("Seq_Loop"), _context(), _EMPTY_DEFS) + assert isinstance(result, ForEachActivity) + assert result.concurrency == 1 + + +class TestIfConditionTranslator: + def test_translate_if_condition_equals(self): + from flowx.translator.activity_translators.if_condition import translate + + true_act = _make_activity("TrueAct", "Wait", {"waitTimeInSeconds": 1}) + false_act = _make_activity("FalseAct", "Wait", {"waitTimeInSeconds": 2}) + + def _mock_translate(activities, context, definitions): + """Mock translate callback that wraps ADF activities as WaitActivity IRs.""" + results = [] + for child in activities: + results.append(WaitActivity(name=child.name, task_key=child.name, wait_time_seconds=1)) + return results, context + + activity = _make_activity( + "Branch", + "IfCondition", + {"expression": {"type": "Expression", "value": "@equals(pipeline().parameters.env, 'prod')"}}, + if_true_activities=[true_act], + if_false_activities=[false_act], + ) + result, context = translate( + activity, + _base_kwargs("Branch"), + _context(), + _EMPTY_DEFS, + translate_activities_fn=_mock_translate, + ) + assert isinstance(result, IfConditionActivity) + assert result.op == "EQUAL_TO" + assert len(result.if_true_activities) == 1 + assert len(result.if_false_activities) == 1 + + def test_translate_if_condition_greater(self): + from flowx.translator.activity_translators.if_condition import translate + + activity = _make_activity( + "Check Count", + "IfCondition", + {"expression": {"type": "Expression", "value": "@greater(activity('Copy').output.rowsCopied, 0)"}}, + ) + result, context = translate(activity, _base_kwargs("Check_Count"), _context(), _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + assert result.op == "GREATER_THAN" + assert "tasks.Copy.values.rowsCopied" in result.left + assert result.right == "0" + + +class TestSetVariableTranslator: + def test_translate_set_variable_literal(self): + from flowx.translator.activity_translators.set_variable import translate + + activity = _make_activity( + "Set Status", + "SetVariable", + {"variableName": "status", "value": "completed"}, + ) + result, context = translate(activity, _base_kwargs("Set_Status"), _context(), _EMPTY_DEFS) + assert isinstance(result, SetVariableActivity) + assert result.variable_name == "status" + assert result.variable_value == "completed" + assert result.value_kind == "literal" + assert result.notebook_code is None + # Context should have the variable mapped + assert context.get_variable_task_key("status") == "Set_Status" + + def test_translate_set_variable_utcnow(self): + from flowx.translator.activity_translators.set_variable import translate + + activity = _make_activity( + "SetRunDate", + "SetVariable", + {"variableName": "runDate", "value": {"type": "Expression", "value": "@utcNow('yyyy-MM-dd')"}}, + ) + result, context = translate(activity, _base_kwargs("SetRunDate"), _context(), _EMPTY_DEFS) + assert isinstance(result, SetVariableActivity) + assert result.value_kind == "notebook_code" + assert result.notebook_code is not None + assert "strftime" in result.notebook_code + assert "datetime" in result.notebook_imports[0] + + def test_translate_set_variable_pipeline_param(self): + from flowx.translator.activity_translators.set_variable import translate + + activity = _make_activity( + "Set Env", + "SetVariable", + {"variableName": "env", "value": "@pipeline().parameters.environment"}, + ) + result, context = translate(activity, _base_kwargs("Set_Env"), _context(), _EMPTY_DEFS) + assert isinstance(result, SetVariableActivity) + assert result.value_kind == "dab_ref" + assert result.variable_value == "{{job.parameters.environment}}" + assert result.notebook_code is None + + +class TestAppendVariableTranslator: + def test_translate_append_variable(self): + from flowx.translator.activity_translators.append_variable import translate + + activity = _make_activity( + "Append Log", + "AppendVariable", + {"variableName": "logEntries", "value": "step1 done"}, + ) + result, context = translate(activity, _base_kwargs("Append_Log"), _context(), _EMPTY_DEFS) + assert isinstance(result, AppendVariableActivity) + assert result.variable_name == "logEntries" + assert result.value_kind == "literal" + assert result.append_value == "step1 done" + assert context.get_variable_task_key("logEntries") == "Append_Log" + + +class TestSwitchTranslator: + def test_translate_switch_with_cases(self): + from flowx.translator.activity_translators.switch import translate + + case_act = _make_activity("CaseWait", "Wait", {"waitTimeInSeconds": 1}) + default_act = _make_activity("DefaultWait", "Wait", {"waitTimeInSeconds": 2}) + + def _mock_translate(activities, context, definitions): + results = [] + for child in activities: + results.append(WaitActivity(name=child.name, task_key=child.name, wait_time_seconds=1)) + return results, context + + activity = _make_activity( + "Route", + "Switch", + { + "on": {"type": "Expression", "value": "@item().load_type"}, + "cases": [ + {"value": "full", "activities": [case_act]}, + {"value": "incremental", "activities": [case_act]}, + ], + "defaultActivities": [default_act], + }, + ) + result, context = translate( + activity, + _base_kwargs("Route"), + _context(), + _EMPTY_DEFS, + translate_activities_fn=_mock_translate, + ) + assert isinstance(result, SwitchActivity) + assert result.on_expression == "{{input.load_type}}" + assert len(result.cases) == 2 + assert result.cases[0].value == "full" + assert result.cases[1].value == "incremental" + assert len(result.default_activities) == 1 + + +class TestTranslateEngine: + def test_translate_pipeline_produces_report(self, adf_definitions): + """translate_pipeline returns a TranslationReport for every pipeline.""" + for pipeline in adf_definitions.pipelines: + report = translate_pipeline(pipeline, adf_definitions) + assert report.pipeline is not None + assert isinstance(report.pipeline, Pipeline) + assert report.pipeline.name == pipeline.name + total = report.deterministic_count + report.agentic_count + report.unsupported_count + assert total > 0 + + def test_translate_pipeline_gaps_tracked(self, adf_definitions): + """Agentic and unsupported types produce gap entries.""" + # pipeline_mixed_agentic has ExecuteDataFlow, SqlServerStoredProcedure, etc. + mixed = next(pl for pl in adf_definitions.pipelines if pl.name == "pipeline_mixed_agentic") + report = translate_pipeline(mixed, adf_definitions) + assert report.agentic_count > 0 or report.unsupported_count > 0 + assert len(report.gaps) > 0 + + def test_translate_pipeline_deterministic_only(self, adf_definitions): + """Pipeline with only deterministic types has zero gaps.""" + basic = next(pl for pl in adf_definitions.pipelines if pl.name == "pipeline_notebook_basic") + report = translate_pipeline(basic, adf_definitions) + assert report.deterministic_count > 0 + assert report.agentic_count == 0 + assert report.unsupported_count == 0 + assert len(report.gaps) == 0 + + def test_translate_pipeline_preserves_parameters(self, adf_definitions): + """Pipeline parameters are forwarded to the IR.""" + csv = next(pl for pl in adf_definitions.pipelines if pl.name == "pipeline_copy_csv_to_delta") + report = translate_pipeline(csv, adf_definitions) + param_names = {param["name"] for param in report.pipeline.parameters} if report.pipeline.parameters else set() + assert "sourceFolderPath" in param_names + assert "triggerDate" in param_names + + def test_translate_pipeline_placeholder_for_agentic(self, adf_definitions): + """Agentic activities produce PlaceholderActivity in the IR.""" + mixed = next(pl for pl in adf_definitions.pipelines if pl.name == "pipeline_mixed_agentic") + report = translate_pipeline(mixed, adf_definitions) + placeholders = [task for task in report.pipeline.tasks if isinstance(task, PlaceholderActivity)] + assert len(placeholders) > 0 + for placeholder in placeholders: + assert placeholder.original_type is not None diff --git a/tests/unit/test_workspace_downloader.py b/tests/unit/test_workspace_downloader.py new file mode 100644 index 0000000..73a6a78 --- /dev/null +++ b/tests/unit/test_workspace_downloader.py @@ -0,0 +1,52 @@ +"""Unit tests for workspace_downloader.py graceful failure paths.""" + +from __future__ import annotations + +from unittest.mock import patch + +from flowx.preparer.workspace_downloader import download_dbfs_file, download_notebook + + +class TestDownloadNotebook: + def test_returns_none_when_sdk_not_available(self): + """download_notebook returns None when databricks-sdk is not installed.""" + with patch.dict("sys.modules", {"databricks": None, "databricks.sdk": None}): + result = download_notebook("/Shared/flowx/transform") + assert result is None + + def test_returns_none_on_import_error(self): + """download_notebook returns None when the SDK import raises ImportError.""" + # Force an ImportError by making the module import fail + + original = __builtins__.__import__ if hasattr(__builtins__, "__import__") else __import__ + + def mock_import(name, *args, **kwargs): + if "databricks.sdk" in name: + raise ImportError("No module named 'databricks.sdk'") + return original(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=mock_import): + result = download_notebook("/Shared/flowx/transform") + assert result is None + + +class TestDownloadDbfsFile: + def test_returns_none_when_sdk_not_available(self): + """download_dbfs_file returns None when databricks-sdk is not installed.""" + with patch.dict("sys.modules", {"databricks": None, "databricks.sdk": None}): + result = download_dbfs_file("dbfs:/scripts/etl.py") + assert result is None + + def test_returns_none_on_import_error(self): + """download_dbfs_file returns None when the SDK import raises ImportError.""" + + original = __builtins__.__import__ if hasattr(__builtins__, "__import__") else __import__ + + def mock_import(name, *args, **kwargs): + if "databricks.sdk" in name: + raise ImportError("No module named 'databricks.sdk'") + return original(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=mock_import): + result = download_dbfs_file("dbfs:/scripts/etl.py") + assert result is None diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..d78a867 --- /dev/null +++ b/uv.lock @@ -0,0 +1,387 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.13.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", size = 219554, upload-time = "2026-03-17T10:30:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", size = 219908, upload-time = "2026-03-17T10:30:43.906Z" }, + { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", size = 251419, upload-time = "2026-03-17T10:30:45.545Z" }, + { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", size = 254159, upload-time = "2026-03-17T10:30:47.204Z" }, + { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", size = 255270, upload-time = "2026-03-17T10:30:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", size = 257538, upload-time = "2026-03-17T10:30:50.77Z" }, + { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", size = 251821, upload-time = "2026-03-17T10:30:52.5Z" }, + { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", size = 253191, upload-time = "2026-03-17T10:30:54.543Z" }, + { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", size = 251337, upload-time = "2026-03-17T10:30:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", size = 255404, upload-time = "2026-03-17T10:30:58.427Z" }, + { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", size = 250903, upload-time = "2026-03-17T10:31:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", size = 252780, upload-time = "2026-03-17T10:31:01.916Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", size = 222093, upload-time = "2026-03-17T10:31:03.642Z" }, + { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", size = 222900, upload-time = "2026-03-17T10:31:05.651Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", size = 221515, upload-time = "2026-03-17T10:31:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" }, + { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" }, + { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" }, + { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" }, + { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" }, + { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" }, + { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" }, + { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" }, + { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" }, + { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" }, + { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" }, + { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" }, + { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" }, + { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" }, + { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" }, + { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" }, + { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" }, + { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" }, + { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" }, + { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" }, + { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" }, + { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" }, + { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" }, + { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" }, + { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" }, + { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" }, + { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" }, + { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" }, + { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" }, + { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" }, + { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" }, + { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" }, + { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" }, + { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" }, + { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "librt" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/9c/b4b0c54d84da4a94b37bd44151e46d5e583c9534c7e02250b961b1b6d8a8/librt-0.8.1.tar.gz", hash = "sha256:be46a14693955b3bd96014ccbdb8339ee8c9346fbe11c1b78901b55125f14c73", size = 177471, upload-time = "2026-02-17T16:13:06.101Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/21/d39b0a87ac52fc98f621fb6f8060efb017a767ebbbac2f99fbcbc9ddc0d7/librt-0.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a28f2612ab566b17f3698b0da021ff9960610301607c9a5e8eaca62f5e1c350a", size = 66516, upload-time = "2026-02-17T16:11:41.604Z" }, + { url = "https://files.pythonhosted.org/packages/69/f1/46375e71441c43e8ae335905e069f1c54febee63a146278bcee8782c84fd/librt-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:60a78b694c9aee2a0f1aaeaa7d101cf713e92e8423a941d2897f4fa37908dab9", size = 68634, upload-time = "2026-02-17T16:11:43.268Z" }, + { url = "https://files.pythonhosted.org/packages/0a/33/c510de7f93bf1fa19e13423a606d8189a02624a800710f6e6a0a0f0784b3/librt-0.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:758509ea3f1eba2a57558e7e98f4659d0ea7670bff49673b0dde18a3c7e6c0eb", size = 198941, upload-time = "2026-02-17T16:11:44.28Z" }, + { url = "https://files.pythonhosted.org/packages/dd/36/e725903416409a533d92398e88ce665476f275081d0d7d42f9c4951999e5/librt-0.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:039b9f2c506bd0ab0f8725aa5ba339c6f0cd19d3b514b50d134789809c24285d", size = 209991, upload-time = "2026-02-17T16:11:45.462Z" }, + { url = "https://files.pythonhosted.org/packages/30/7a/8d908a152e1875c9f8eac96c97a480df425e657cdb47854b9efaa4998889/librt-0.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bb54f1205a3a6ab41a6fd71dfcdcbd278670d3a90ca502a30d9da583105b6f7", size = 224476, upload-time = "2026-02-17T16:11:46.542Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b8/a22c34f2c485b8903a06f3fe3315341fe6876ef3599792344669db98fcff/librt-0.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:05bd41cdee35b0c59c259f870f6da532a2c5ca57db95b5f23689fcb5c9e42440", size = 217518, upload-time = "2026-02-17T16:11:47.746Z" }, + { url = "https://files.pythonhosted.org/packages/79/6f/5c6fea00357e4f82ba44f81dbfb027921f1ab10e320d4a64e1c408d035d9/librt-0.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adfab487facf03f0d0857b8710cf82d0704a309d8ffc33b03d9302b4c64e91a9", size = 225116, upload-time = "2026-02-17T16:11:49.298Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a0/95ced4e7b1267fe1e2720a111685bcddf0e781f7e9e0ce59d751c44dcfe5/librt-0.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:153188fe98a72f206042be10a2c6026139852805215ed9539186312d50a8e972", size = 217751, upload-time = "2026-02-17T16:11:50.49Z" }, + { url = "https://files.pythonhosted.org/packages/93/c2/0517281cb4d4101c27ab59472924e67f55e375bc46bedae94ac6dc6e1902/librt-0.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:dd3c41254ee98604b08bd5b3af5bf0a89740d4ee0711de95b65166bf44091921", size = 218378, upload-time = "2026-02-17T16:11:51.783Z" }, + { url = "https://files.pythonhosted.org/packages/43/e8/37b3ac108e8976888e559a7b227d0ceac03c384cfd3e7a1c2ee248dbae79/librt-0.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e0d138c7ae532908cbb342162b2611dbd4d90c941cd25ab82084aaf71d2c0bd0", size = 241199, upload-time = "2026-02-17T16:11:53.561Z" }, + { url = "https://files.pythonhosted.org/packages/4b/5b/35812d041c53967fedf551a39399271bbe4257e681236a2cf1a69c8e7fa1/librt-0.8.1-cp312-cp312-win32.whl", hash = "sha256:43353b943613c5d9c49a25aaffdba46f888ec354e71e3529a00cca3f04d66a7a", size = 54917, upload-time = "2026-02-17T16:11:54.758Z" }, + { url = "https://files.pythonhosted.org/packages/de/d1/fa5d5331b862b9775aaf2a100f5ef86854e5d4407f71bddf102f4421e034/librt-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:ff8baf1f8d3f4b6b7257fcb75a501f2a5499d0dda57645baa09d4d0d34b19444", size = 62017, upload-time = "2026-02-17T16:11:55.748Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7c/c614252f9acda59b01a66e2ddfd243ed1c7e1deab0293332dfbccf862808/librt-0.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:0f2ae3725904f7377e11cc37722d5d401e8b3d5851fb9273d7f4fe04f6b3d37d", size = 52441, upload-time = "2026-02-17T16:11:56.801Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3c/f614c8e4eaac7cbf2bbdf9528790b21d89e277ee20d57dc6e559c626105f/librt-0.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e6bad1cd94f6764e1e21950542f818a09316645337fd5ab9a7acc45d99a8f35", size = 66529, upload-time = "2026-02-17T16:11:57.809Z" }, + { url = "https://files.pythonhosted.org/packages/ab/96/5836544a45100ae411eda07d29e3d99448e5258b6e9c8059deb92945f5c2/librt-0.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cf450f498c30af55551ba4f66b9123b7185362ec8b625a773b3d39aa1a717583", size = 68669, upload-time = "2026-02-17T16:11:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/06/53/f0b992b57af6d5531bf4677d75c44f095f2366a1741fb695ee462ae04b05/librt-0.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:eca45e982fa074090057132e30585a7e8674e9e885d402eae85633e9f449ce6c", size = 199279, upload-time = "2026-02-17T16:11:59.862Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ad/4848cc16e268d14280d8168aee4f31cea92bbd2b79ce33d3e166f2b4e4fc/librt-0.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c3811485fccfda840861905b8c70bba5ec094e02825598bb9d4ca3936857a04", size = 210288, upload-time = "2026-02-17T16:12:00.954Z" }, + { url = "https://files.pythonhosted.org/packages/52/05/27fdc2e95de26273d83b96742d8d3b7345f2ea2bdbd2405cc504644f2096/librt-0.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e4af413908f77294605e28cfd98063f54b2c790561383971d2f52d113d9c363", size = 224809, upload-time = "2026-02-17T16:12:02.108Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d0/78200a45ba3240cb042bc597d6f2accba9193a2c57d0356268cbbe2d0925/librt-0.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5212a5bd7fae98dae95710032902edcd2ec4dc994e883294f75c857b83f9aba0", size = 218075, upload-time = "2026-02-17T16:12:03.631Z" }, + { url = "https://files.pythonhosted.org/packages/af/72/a210839fa74c90474897124c064ffca07f8d4b347b6574d309686aae7ca6/librt-0.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e692aa2d1d604e6ca12d35e51fdc36f4cda6345e28e36374579f7ef3611b3012", size = 225486, upload-time = "2026-02-17T16:12:04.725Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c1/a03cc63722339ddbf087485f253493e2b013039f5b707e8e6016141130fa/librt-0.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4be2a5c926b9770c9e08e717f05737a269b9d0ebc5d2f0060f0fe3fe9ce47acb", size = 218219, upload-time = "2026-02-17T16:12:05.828Z" }, + { url = "https://files.pythonhosted.org/packages/58/f5/fff6108af0acf941c6f274a946aea0e484bd10cd2dc37610287ce49388c5/librt-0.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fd1a720332ea335ceb544cf0a03f81df92abd4bb887679fd1e460976b0e6214b", size = 218750, upload-time = "2026-02-17T16:12:07.09Z" }, + { url = "https://files.pythonhosted.org/packages/71/67/5a387bfef30ec1e4b4f30562c8586566faf87e47d696768c19feb49e3646/librt-0.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2af9e01e0ef80d95ae3c720be101227edae5f2fe7e3dc63d8857fadfc5a1d", size = 241624, upload-time = "2026-02-17T16:12:08.43Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/24f8502db11d405232ac1162eb98069ca49c3306c1d75c6ccc61d9af8789/librt-0.8.1-cp313-cp313-win32.whl", hash = "sha256:086a32dbb71336627e78cc1d6ee305a68d038ef7d4c39aaff41ae8c9aa46e91a", size = 54969, upload-time = "2026-02-17T16:12:09.633Z" }, + { url = "https://files.pythonhosted.org/packages/5c/73/c9fdf6cb2a529c1a092ce769a12d88c8cca991194dfe641b6af12fa964d2/librt-0.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:e11769a1dbda4da7b00a76cfffa67aa47cfa66921d2724539eee4b9ede780b79", size = 62000, upload-time = "2026-02-17T16:12:10.632Z" }, + { url = "https://files.pythonhosted.org/packages/d3/97/68f80ca3ac4924f250cdfa6e20142a803e5e50fca96ef5148c52ee8c10ea/librt-0.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:924817ab3141aca17893386ee13261f1d100d1ef410d70afe4389f2359fea4f0", size = 52495, upload-time = "2026-02-17T16:12:11.633Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6a/907ef6800f7bca71b525a05f1839b21f708c09043b1c6aa77b6b827b3996/librt-0.8.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6cfa7fe54fd4d1f47130017351a959fe5804bda7a0bc7e07a2cdbc3fdd28d34f", size = 66081, upload-time = "2026-02-17T16:12:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/1b/18/25e991cd5640c9fb0f8d91b18797b29066b792f17bf8493da183bf5caabe/librt-0.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:228c2409c079f8c11fb2e5d7b277077f694cb93443eb760e00b3b83cb8b3176c", size = 68309, upload-time = "2026-02-17T16:12:13.756Z" }, + { url = "https://files.pythonhosted.org/packages/a4/36/46820d03f058cfb5a9de5940640ba03165ed8aded69e0733c417bb04df34/librt-0.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7aae78ab5e3206181780e56912d1b9bb9f90a7249ce12f0e8bf531d0462dd0fc", size = 196804, upload-time = "2026-02-17T16:12:14.818Z" }, + { url = "https://files.pythonhosted.org/packages/59/18/5dd0d3b87b8ff9c061849fbdb347758d1f724b9a82241aa908e0ec54ccd0/librt-0.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:172d57ec04346b047ca6af181e1ea4858086c80bdf455f61994c4aa6fc3f866c", size = 206907, upload-time = "2026-02-17T16:12:16.513Z" }, + { url = "https://files.pythonhosted.org/packages/d1/96/ef04902aad1424fd7299b62d1890e803e6ab4018c3044dca5922319c4b97/librt-0.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b1977c4ea97ce5eb7755a78fae68d87e4102e4aaf54985e8b56806849cc06a3", size = 221217, upload-time = "2026-02-17T16:12:17.906Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ff/7e01f2dda84a8f5d280637a2e5827210a8acca9a567a54507ef1c75b342d/librt-0.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10c42e1f6fd06733ef65ae7bebce2872bcafd8d6e6b0a08fe0a05a23b044fb14", size = 214622, upload-time = "2026-02-17T16:12:19.108Z" }, + { url = "https://files.pythonhosted.org/packages/1e/8c/5b093d08a13946034fed57619742f790faf77058558b14ca36a6e331161e/librt-0.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4c8dfa264b9193c4ee19113c985c95f876fae5e51f731494fc4e0cf594990ba7", size = 221987, upload-time = "2026-02-17T16:12:20.331Z" }, + { url = "https://files.pythonhosted.org/packages/d3/cc/86b0b3b151d40920ad45a94ce0171dec1aebba8a9d72bb3fa00c73ab25dd/librt-0.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:01170b6729a438f0dedc4a26ed342e3dc4f02d1000b4b19f980e1877f0c297e6", size = 215132, upload-time = "2026-02-17T16:12:21.54Z" }, + { url = "https://files.pythonhosted.org/packages/fc/be/8588164a46edf1e69858d952654e216a9a91174688eeefb9efbb38a9c799/librt-0.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7b02679a0d783bdae30d443025b94465d8c3dc512f32f5b5031f93f57ac32071", size = 215195, upload-time = "2026-02-17T16:12:23.073Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f2/0b9279bea735c734d69344ecfe056c1ba211694a72df10f568745c899c76/librt-0.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:190b109bb69592a3401fe1ffdea41a2e73370ace2ffdc4a0e8e2b39cdea81b78", size = 237946, upload-time = "2026-02-17T16:12:24.275Z" }, + { url = "https://files.pythonhosted.org/packages/e9/cc/5f2a34fbc8aeb35314a3641f9956fa9051a947424652fad9882be7a97949/librt-0.8.1-cp314-cp314-win32.whl", hash = "sha256:e70a57ecf89a0f64c24e37f38d3fe217a58169d2fe6ed6d70554964042474023", size = 50689, upload-time = "2026-02-17T16:12:25.766Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/cd4d010ab2147339ca2b93e959c3686e964edc6de66ddacc935c325883d7/librt-0.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:7e2f3edca35664499fbb36e4770650c4bd4a08abc1f4458eab9df4ec56389730", size = 57875, upload-time = "2026-02-17T16:12:27.465Z" }, + { url = "https://files.pythonhosted.org/packages/84/0f/2143cb3c3ca48bd3379dcd11817163ca50781927c4537345d608b5045998/librt-0.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:0d2f82168e55ddefd27c01c654ce52379c0750ddc31ee86b4b266bcf4d65f2a3", size = 48058, upload-time = "2026-02-17T16:12:28.556Z" }, + { url = "https://files.pythonhosted.org/packages/d2/0e/9b23a87e37baf00311c3efe6b48d6b6c168c29902dfc3f04c338372fd7db/librt-0.8.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2c74a2da57a094bd48d03fa5d196da83d2815678385d2978657499063709abe1", size = 68313, upload-time = "2026-02-17T16:12:29.659Z" }, + { url = "https://files.pythonhosted.org/packages/db/9a/859c41e5a4f1c84200a7d2b92f586aa27133c8243b6cac9926f6e54d01b9/librt-0.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a355d99c4c0d8e5b770313b8b247411ed40949ca44e33e46a4789b9293a907ee", size = 70994, upload-time = "2026-02-17T16:12:31.516Z" }, + { url = "https://files.pythonhosted.org/packages/4c/28/10605366ee599ed34223ac2bf66404c6fb59399f47108215d16d5ad751a8/librt-0.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2eb345e8b33fb748227409c9f1233d4df354d6e54091f0e8fc53acdb2ffedeb7", size = 220770, upload-time = "2026-02-17T16:12:33.294Z" }, + { url = "https://files.pythonhosted.org/packages/af/8d/16ed8fd452dafae9c48d17a6bc1ee3e818fd40ef718d149a8eff2c9f4ea2/librt-0.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9be2f15e53ce4e83cc08adc29b26fb5978db62ef2a366fbdf716c8a6c8901040", size = 235409, upload-time = "2026-02-17T16:12:35.443Z" }, + { url = "https://files.pythonhosted.org/packages/89/1b/7bdf3e49349c134b25db816e4a3db6b94a47ac69d7d46b1e682c2c4949be/librt-0.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:785ae29c1f5c6e7c2cde2c7c0e148147f4503da3abc5d44d482068da5322fd9e", size = 246473, upload-time = "2026-02-17T16:12:36.656Z" }, + { url = "https://files.pythonhosted.org/packages/4e/8a/91fab8e4fd2a24930a17188c7af5380eb27b203d72101c9cc000dbdfd95a/librt-0.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d3a7da44baf692f0c6aeb5b2a09c5e6fc7a703bca9ffa337ddd2e2da53f7732", size = 238866, upload-time = "2026-02-17T16:12:37.849Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e0/c45a098843fc7c07e18a7f8a24ca8496aecbf7bdcd54980c6ca1aaa79a8e/librt-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5fc48998000cbc39ec0d5311312dda93ecf92b39aaf184c5e817d5d440b29624", size = 250248, upload-time = "2026-02-17T16:12:39.445Z" }, + { url = "https://files.pythonhosted.org/packages/82/30/07627de23036640c952cce0c1fe78972e77d7d2f8fd54fa5ef4554ff4a56/librt-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e96baa6820280077a78244b2e06e416480ed859bbd8e5d641cf5742919d8beb4", size = 240629, upload-time = "2026-02-17T16:12:40.889Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c1/55bfe1ee3542eba055616f9098eaf6eddb966efb0ca0f44eaa4aba327307/librt-0.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31362dbfe297b23590530007062c32c6f6176f6099646bb2c95ab1b00a57c382", size = 239615, upload-time = "2026-02-17T16:12:42.446Z" }, + { url = "https://files.pythonhosted.org/packages/2b/39/191d3d28abc26c9099b19852e6c99f7f6d400b82fa5a4e80291bd3803e19/librt-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc3656283d11540ab0ea01978378e73e10002145117055e03722417aeab30994", size = 263001, upload-time = "2026-02-17T16:12:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/b9/eb/7697f60fbe7042ab4e88f4ee6af496b7f222fffb0a4e3593ef1f29f81652/librt-0.8.1-cp314-cp314t-win32.whl", hash = "sha256:738f08021b3142c2918c03692608baed43bc51144c29e35807682f8070ee2a3a", size = 51328, upload-time = "2026-02-17T16:12:45.148Z" }, + { url = "https://files.pythonhosted.org/packages/7c/72/34bf2eb7a15414a23e5e70ecb9440c1d3179f393d9349338a91e2781c0fb/librt-0.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:89815a22daf9c51884fb5dbe4f1ef65ee6a146e0b6a8df05f753e2e4a9359bf4", size = 58722, upload-time = "2026-02-17T16:12:46.85Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c8/d148e041732d631fc76036f8b30fae4e77b027a1e95b7a84bb522481a940/librt-0.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:bf512a71a23504ed08103a13c941f763db13fb11177beb3d9244c98c29fb4a61", size = 48755, upload-time = "2026-02-17T16:12:47.943Z" }, +] + +[[package]] +name = "mypy" +version = "1.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/5c/b0089fe7fef0a994ae5ee07029ced0526082c6cfaaa4c10d40a10e33b097/mypy-1.20.0.tar.gz", hash = "sha256:eb96c84efcc33f0b5e0e04beacf00129dd963b67226b01c00b9dfc8affb464c3", size = 3815028, upload-time = "2026-03-31T16:55:14.959Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/dd/3afa29b58c2e57c79116ed55d700721c3c3b15955e2b6251dd165d377c0e/mypy-1.20.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:002b613ae19f4ac7d18b7e168ffe1cb9013b37c57f7411984abbd3b817b0a214", size = 14509525, upload-time = "2026-03-31T16:55:01.824Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/227b516ab8cad9f2a13c5e7a98d28cd6aa75e9c83e82776ae6c1c4c046c7/mypy-1.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9336b5e6712f4adaf5afc3203a99a40b379049104349d747eb3e5a3aa23ac2e", size = 13326469, upload-time = "2026-03-31T16:51:41.23Z" }, + { url = "https://files.pythonhosted.org/packages/57/d4/1ddb799860c1b5ac6117ec307b965f65deeb47044395ff01ab793248a591/mypy-1.20.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f13b3e41bce9d257eded794c0f12878af3129d80aacd8a3ee0dee51f3a978651", size = 13705953, upload-time = "2026-03-31T16:48:55.69Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b7/54a720f565a87b893182a2a393370289ae7149e4715859e10e1c05e49154/mypy-1.20.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9804c3ad27f78e54e58b32e7cb532d128b43dbfb9f3f9f06262b821a0f6bd3f5", size = 14710363, upload-time = "2026-03-31T16:53:26.948Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2a/74810274848d061f8a8ea4ac23aaad43bd3d8c1882457999c2e568341c57/mypy-1.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:697f102c5c1d526bdd761a69f17c6070f9892eebcb94b1a5963d679288c09e78", size = 14947005, upload-time = "2026-03-31T16:50:17.591Z" }, + { url = "https://files.pythonhosted.org/packages/77/91/21b8ba75f958bcda75690951ce6fa6b7138b03471618959529d74b8544e2/mypy-1.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ecd63f75fdd30327e4ad8b5704bd6d91fc6c1b2e029f8ee14705e1207212489", size = 10880616, upload-time = "2026-03-31T16:52:19.986Z" }, + { url = "https://files.pythonhosted.org/packages/8a/15/3d8198ef97c1ca03aea010cce4f1d4f3bc5d9849e8c0140111ca2ead9fdd/mypy-1.20.0-cp312-cp312-win_arm64.whl", hash = "sha256:f194db59657c58593a3c47c6dfd7bad4ef4ac12dbc94d01b3a95521f78177e33", size = 9813091, upload-time = "2026-03-31T16:53:44.385Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a7/f64ea7bd592fa431cb597418b6dec4a47f7d0c36325fec7ac67bc8402b94/mypy-1.20.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b20c8b0fd5877abdf402e79a3af987053de07e6fb208c18df6659f708b535134", size = 14485344, upload-time = "2026-03-31T16:49:16.78Z" }, + { url = "https://files.pythonhosted.org/packages/bb/72/8927d84cfc90c6abea6e96663576e2e417589347eb538749a464c4c218a0/mypy-1.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:367e5c993ba34d5054d11937d0485ad6dfc60ba760fa326c01090fc256adf15c", size = 13327400, upload-time = "2026-03-31T16:53:08.02Z" }, + { url = "https://files.pythonhosted.org/packages/ab/4a/11ab99f9afa41aa350178d24a7d2da17043228ea10f6456523f64b5a6cf6/mypy-1.20.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f799d9db89fc00446f03281f84a221e50018fc40113a3ba9864b132895619ebe", size = 13706384, upload-time = "2026-03-31T16:52:28.577Z" }, + { url = "https://files.pythonhosted.org/packages/42/79/694ca73979cfb3535ebfe78733844cd5aff2e63304f59bf90585110d975a/mypy-1.20.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555658c611099455b2da507582ea20d2043dfdfe7f5ad0add472b1c6238b433f", size = 14700378, upload-time = "2026-03-31T16:48:45.527Z" }, + { url = "https://files.pythonhosted.org/packages/84/24/a022ccab3a46e3d2cdf2e0e260648633640eb396c7e75d5a42818a8d3971/mypy-1.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:efe8d70949c3023698c3fca1e94527e7e790a361ab8116f90d11221421cd8726", size = 14932170, upload-time = "2026-03-31T16:49:36.038Z" }, + { url = "https://files.pythonhosted.org/packages/d8/9b/549228d88f574d04117e736f55958bd4908f980f9f5700a07aeb85df005b/mypy-1.20.0-cp313-cp313-win_amd64.whl", hash = "sha256:f49590891d2c2f8a9de15614e32e459a794bcba84693c2394291a2038bbaaa69", size = 10888526, upload-time = "2026-03-31T16:50:59.827Z" }, + { url = "https://files.pythonhosted.org/packages/91/17/15095c0e54a8bc04d22d4ff06b2139d5f142c2e87520b4e39010c4862771/mypy-1.20.0-cp313-cp313-win_arm64.whl", hash = "sha256:76a70bf840495729be47510856b978f1b0ec7d08f257ca38c9d932720bf6b43e", size = 9816456, upload-time = "2026-03-31T16:49:59.537Z" }, + { url = "https://files.pythonhosted.org/packages/4e/0e/6ca4a84cbed9e62384bc0b2974c90395ece5ed672393e553996501625fc5/mypy-1.20.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0f42dfaab7ec1baff3b383ad7af562ab0de573c5f6edb44b2dab016082b89948", size = 14483331, upload-time = "2026-03-31T16:52:57.999Z" }, + { url = "https://files.pythonhosted.org/packages/7d/c5/5fe9d8a729dd9605064691816243ae6c49fde0bd28f6e5e17f6a24203c43/mypy-1.20.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:31b5dbb55293c1bd27c0fc813a0d2bb5ceef9d65ac5afa2e58f829dab7921fd5", size = 13342047, upload-time = "2026-03-31T16:54:21.555Z" }, + { url = "https://files.pythonhosted.org/packages/4c/33/e18bcfa338ca4e6b2771c85d4c5203e627d0c69d9de5c1a2cf2ba13320ba/mypy-1.20.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49d11c6f573a5a08f77fad13faff2139f6d0730ebed2cfa9b3d2702671dd7188", size = 13719585, upload-time = "2026-03-31T16:51:53.89Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8d/93491ff7b79419edc7eabf95cb3b3f7490e2e574b2855c7c7e7394ff933f/mypy-1.20.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d3243c406773185144527f83be0e0aefc7bf4601b0b2b956665608bf7c98a83", size = 14685075, upload-time = "2026-03-31T16:54:04.464Z" }, + { url = "https://files.pythonhosted.org/packages/b5/9d/d924b38a4923f8d164bf2b4ec98bf13beaf6e10a5348b4b137eadae40a6e/mypy-1.20.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a79c1eba7ac4209f2d850f0edd0a2f8bba88cbfdfefe6fb76a19e9d4fe5e71a2", size = 14919141, upload-time = "2026-03-31T16:54:51.785Z" }, + { url = "https://files.pythonhosted.org/packages/59/98/1da9977016678c0b99d43afe52ed00bb3c1a0c4c995d3e6acca1a6ebb9b4/mypy-1.20.0-cp314-cp314-win_amd64.whl", hash = "sha256:00e047c74d3ec6e71a2eb88e9ea551a2edb90c21f993aefa9e0d2a898e0bb732", size = 11050925, upload-time = "2026-03-31T16:51:30.758Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e3/ba0b7a3143e49a9c4f5967dde6ea4bf8e0b10ecbbcca69af84027160ee89/mypy-1.20.0-cp314-cp314-win_arm64.whl", hash = "sha256:931a7630bba591593dcf6e97224a21ff80fb357e7982628d25e3c618e7f598ef", size = 10001089, upload-time = "2026-03-31T16:49:43.632Z" }, + { url = "https://files.pythonhosted.org/packages/12/28/e617e67b3be9d213cda7277913269c874eb26472489f95d09d89765ce2d8/mypy-1.20.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:26c8b52627b6552f47ff11adb4e1509605f094e29815323e487fc0053ebe93d1", size = 15534710, upload-time = "2026-03-31T16:52:12.506Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0c/3b5f2d3e45dc7169b811adce8451679d9430399d03b168f9b0489f43adaa/mypy-1.20.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:39362cdb4ba5f916e7976fccecaab1ba3a83e35f60fa68b64e9a70e221bb2436", size = 14393013, upload-time = "2026-03-31T16:54:41.186Z" }, + { url = "https://files.pythonhosted.org/packages/a3/49/edc8b0aa145cc09c1c74f7ce2858eead9329931dcbbb26e2ad40906daa4e/mypy-1.20.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34506397dbf40c15dc567635d18a21d33827e9ab29014fb83d292a8f4f8953b6", size = 15047240, upload-time = "2026-03-31T16:54:31.955Z" }, + { url = "https://files.pythonhosted.org/packages/42/37/a946bb416e37a57fa752b3100fd5ede0e28df94f92366d1716555d47c454/mypy-1.20.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555493c44a4f5a1b58d611a43333e71a9981c6dbe26270377b6f8174126a0526", size = 15858565, upload-time = "2026-03-31T16:53:36.997Z" }, + { url = "https://files.pythonhosted.org/packages/2f/99/7690b5b5b552db1bd4ff362e4c0eb3107b98d680835e65823fbe888c8b78/mypy-1.20.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2721f0ce49cb74a38f00c50da67cb7d36317b5eda38877a49614dc018e91c787", size = 16087874, upload-time = "2026-03-31T16:52:48.313Z" }, + { url = "https://files.pythonhosted.org/packages/aa/76/53e893a498138066acd28192b77495c9357e5a58cc4be753182846b43315/mypy-1.20.0-cp314-cp314t-win_amd64.whl", hash = "sha256:47781555a7aa5fedcc2d16bcd72e0dc83eb272c10dd657f9fb3f9cc08e2e6abb", size = 12572380, upload-time = "2026-03-31T16:49:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/76/9c/6dbdae21f01b7aacddc2c0bbf3c5557aa547827fdf271770fe1e521e7093/mypy-1.20.0-cp314-cp314t-win_arm64.whl", hash = "sha256:c70380fe5d64010f79fb863b9081c7004dd65225d2277333c219d93a10dad4dd", size = 10381174, upload-time = "2026-03-31T16:51:20.179Z" }, + { url = "https://files.pythonhosted.org/packages/21/66/4d734961ce167f0fd8380769b3b7c06dbdd6ff54c2190f3f2ecd22528158/mypy-1.20.0-py3-none-any.whl", hash = "sha256:a6e0641147cbfa7e4e94efdb95c2dab1aff8cfc159ded13e07f308ddccc8c48e", size = 2636365, upload-time = "2026-03-31T16:51:44.911Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "flowx" +version = "0.2.0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage" }, + { name = "mypy" }, + { name = "pytest" }, + { name = "ruff" }, + { name = "types-pyyaml" }, +] + +[package.metadata] +requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", specifier = ">=7.6.1,<8" }, + { name = "mypy", specifier = ">=1.18.2,<2" }, + { name = "pytest", specifier = ">=8.3.3,<9" }, + { name = "ruff", specifier = ">=0.14.0,<1" }, + { name = "types-pyyaml", specifier = ">=6.0.12.20250915,<7" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pathspec" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/b0/73cf7550861e2b4824950b8b52eebdcc5adc792a00c514406556c5b80817/ruff-0.15.8.tar.gz", hash = "sha256:995f11f63597ee362130d1d5a327a87cb6f3f5eae3094c620bcc632329a4d26e", size = 4610921, upload-time = "2026-03-26T18:39:38.675Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/92/c445b0cd6da6e7ae51e954939cb69f97e008dbe750cfca89b8cedc081be7/ruff-0.15.8-py3-none-linux_armv6l.whl", hash = "sha256:cbe05adeba76d58162762d6b239c9056f1a15a55bd4b346cfd21e26cd6ad7bc7", size = 10527394, upload-time = "2026-03-26T18:39:41.566Z" }, + { url = "https://files.pythonhosted.org/packages/eb/92/f1c662784d149ad1414cae450b082cf736430c12ca78367f20f5ed569d65/ruff-0.15.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d3e3d0b6ba8dca1b7ef9ab80a28e840a20070c4b62e56d675c24f366ef330570", size = 10905693, upload-time = "2026-03-26T18:39:30.364Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f2/7a631a8af6d88bcef997eb1bf87cc3da158294c57044aafd3e17030613de/ruff-0.15.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6ee3ae5c65a42f273f126686353f2e08ff29927b7b7e203b711514370d500de3", size = 10323044, upload-time = "2026-03-26T18:39:33.37Z" }, + { url = "https://files.pythonhosted.org/packages/67/18/1bf38e20914a05e72ef3b9569b1d5c70a7ef26cd188d69e9ca8ef588d5bf/ruff-0.15.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdce027ada77baa448077ccc6ebb2fa9c3c62fd110d8659d601cf2f475858d94", size = 10629135, upload-time = "2026-03-26T18:39:44.142Z" }, + { url = "https://files.pythonhosted.org/packages/d2/e9/138c150ff9af60556121623d41aba18b7b57d95ac032e177b6a53789d279/ruff-0.15.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12e617fc01a95e5821648a6df341d80456bd627bfab8a829f7cfc26a14a4b4a3", size = 10348041, upload-time = "2026-03-26T18:39:52.178Z" }, + { url = "https://files.pythonhosted.org/packages/02/f1/5bfb9298d9c323f842c5ddeb85f1f10ef51516ac7a34ba446c9347d898df/ruff-0.15.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:432701303b26416d22ba696c39f2c6f12499b89093b61360abc34bcc9bf07762", size = 11121987, upload-time = "2026-03-26T18:39:55.195Z" }, + { url = "https://files.pythonhosted.org/packages/10/11/6da2e538704e753c04e8d86b1fc55712fdbdcc266af1a1ece7a51fff0d10/ruff-0.15.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d910ae974b7a06a33a057cb87d2a10792a3b2b3b35e33d2699fdf63ec8f6b17a", size = 11951057, upload-time = "2026-03-26T18:39:19.18Z" }, + { url = "https://files.pythonhosted.org/packages/83/f0/c9208c5fd5101bf87002fed774ff25a96eea313d305f1e5d5744698dc314/ruff-0.15.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2033f963c43949d51e6fdccd3946633c6b37c484f5f98c3035f49c27395a8ab8", size = 11464613, upload-time = "2026-03-26T18:40:06.301Z" }, + { url = "https://files.pythonhosted.org/packages/f8/22/d7f2fabdba4fae9f3b570e5605d5eb4500dcb7b770d3217dca4428484b17/ruff-0.15.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f29b989a55572fb885b77464cf24af05500806ab4edf9a0fd8977f9759d85b1", size = 11257557, upload-time = "2026-03-26T18:39:57.972Z" }, + { url = "https://files.pythonhosted.org/packages/71/8c/382a9620038cf6906446b23ce8632ab8c0811b8f9d3e764f58bedd0c9a6f/ruff-0.15.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ac51d486bf457cdc985a412fb1801b2dfd1bd8838372fc55de64b1510eff4bec", size = 11169440, upload-time = "2026-03-26T18:39:22.205Z" }, + { url = "https://files.pythonhosted.org/packages/4d/0d/0994c802a7eaaf99380085e4e40c845f8e32a562e20a38ec06174b52ef24/ruff-0.15.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c9861eb959edab053c10ad62c278835ee69ca527b6dcd72b47d5c1e5648964f6", size = 10605963, upload-time = "2026-03-26T18:39:46.682Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/d624b86f5b0aad7cef6bbf9cd47a6a02dfdc4f72c92a337d724e39c9d14b/ruff-0.15.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8d9a5b8ea13f26ae90838afc33f91b547e61b794865374f114f349e9036835fb", size = 10357484, upload-time = "2026-03-26T18:39:49.176Z" }, + { url = "https://files.pythonhosted.org/packages/35/c3/e0b7835d23001f7d999f3895c6b569927c4d39912286897f625736e1fd04/ruff-0.15.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c2a33a529fb3cbc23a7124b5c6ff121e4d6228029cba374777bd7649cc8598b8", size = 10830426, upload-time = "2026-03-26T18:40:03.702Z" }, + { url = "https://files.pythonhosted.org/packages/f0/51/ab20b322f637b369383adc341d761eaaa0f0203d6b9a7421cd6e783d81b9/ruff-0.15.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:75e5cd06b1cf3f47a3996cfc999226b19aa92e7cce682dcd62f80d7035f98f49", size = 11345125, upload-time = "2026-03-26T18:39:27.799Z" }, + { url = "https://files.pythonhosted.org/packages/37/e6/90b2b33419f59d0f2c4c8a48a4b74b460709a557e8e0064cf33ad894f983/ruff-0.15.8-py3-none-win32.whl", hash = "sha256:bc1f0a51254ba21767bfa9a8b5013ca8149dcf38092e6a9eb704d876de94dc34", size = 10571959, upload-time = "2026-03-26T18:39:36.117Z" }, + { url = "https://files.pythonhosted.org/packages/1f/a2/ef467cb77099062317154c63f234b8a7baf7cb690b99af760c5b68b9ee7f/ruff-0.15.8-py3-none-win_amd64.whl", hash = "sha256:04f79eff02a72db209d47d665ba7ebcad609d8918a134f86cb13dd132159fc89", size = 11743893, upload-time = "2026-03-26T18:39:25.01Z" }, + { url = "https://files.pythonhosted.org/packages/15/e2/77be4fff062fa78d9b2a4dea85d14785dac5f1d0c1fb58ed52331f0ebe28/ruff-0.15.8-py3-none-win_arm64.whl", hash = "sha256:cf891fa8e3bb430c0e7fac93851a5978fc99c8fa2c053b57b118972866f8e5f2", size = 11048175, upload-time = "2026-03-26T18:40:01.06Z" }, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.20250915" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/69/3c51b36d04da19b92f9e815be12753125bd8bc247ba0470a982e6979e71c/types_pyyaml-6.0.12.20250915.tar.gz", hash = "sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3", size = 17522, upload-time = "2025-09-15T03:01:00.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/e0/1eed384f02555dde685fff1a1ac805c1c7dcb6dd019c916fe659b1c1f9ec/types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6", size = 20338, upload-time = "2025-09-15T03:00:59.218Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] From d2fe94b14bb4ce70a97d0b155fb3a0f69c056425 Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Tue, 5 May 2026 13:46:33 -0400 Subject: [PATCH 02/77] Update documentation (#4) --- docs/content/docs/{usage-guide.mdx => guide.mdx} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/content/docs/{usage-guide.mdx => guide.mdx} (97%) diff --git a/docs/content/docs/usage-guide.mdx b/docs/content/docs/guide.mdx similarity index 97% rename from docs/content/docs/usage-guide.mdx rename to docs/content/docs/guide.mdx index 720417a..5f8d572 100644 --- a/docs/content/docs/usage-guide.mdx +++ b/docs/content/docs/guide.mdx @@ -1,6 +1,6 @@ --- title: Usage Guide -description: Use flowx to translate a pipeline from Azure Data Factory to Lakeflow Jobs. +description: Translate pipelines to Lakeflow Jobs from Azure Data Factory. --- import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; From 7150eb33e78f271e90e1f7ad3c53342bd8545ee4 Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Thu, 7 May 2026 13:22:39 -0400 Subject: [PATCH 03/77] Update documentation (#5) * Update documentation * Restore docs navigation entries Co-authored-by: ghanse <163584195+ghanse@users.noreply.github.com> * Fix docs guide navigation links Co-authored-by: ghanse <163584195+ghanse@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- docs/content/docs/guide.mdx | 2 +- docs/content/docs/index.mdx | 2 +- docs/content/docs/meta.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx index 5f8d572..e6b482d 100644 --- a/docs/content/docs/guide.mdx +++ b/docs/content/docs/guide.mdx @@ -1,6 +1,6 @@ --- title: Usage Guide -description: Translate pipelines to Lakeflow Jobs from Azure Data Factory. +description: Translate a pipeline from Azure Data Factory to Lakeflow Jobs. --- import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index 643662c..0a6e022 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -32,5 +32,5 @@ Each phase can be run independently, maintains its own input/output contract, pr ## Where to go next - **[Installation](/docs/installation)** — install the flowx plugin in your agentic tool of choice. -- **[Usage Guide](/docs/usage-guide)** — an end-to-end walkthrough from raw ADF JSON to a deployable bundle. +- **[Usage Guide](/docs/guide)** — an end-to-end walkthrough from raw ADF JSON to a deployable bundle. - **[AI Tools & Skills](/docs/ai-tools-skills)** — the agent skills shipped with flowx and how each tool loads them. diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index e65d2ac..667f5fb 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -4,7 +4,7 @@ "index", "how-it-works", "installation", - "usage-guide", + "guide", "ai-tools-skills" ] } From 6ec052f26cb4d19fdc9fd21bb150c6b5b5ca17ac Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Wed, 13 May 2026 12:09:00 -0400 Subject: [PATCH 04/77] Update documentation (#6) --- docs/content/docs/index.mdx | 7 +++---- docs/content/docs/installation.mdx | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index 0a6e022..ababf16 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -29,8 +29,7 @@ can be deployed to a Databricks workspace. Bundles include job configuration fil Each phase can be run independently, maintains its own input/output contract, produces artifacts you can inspect before moving to the next phase. -## Where to go next +## Next steps -- **[Installation](/docs/installation)** — install the flowx plugin in your agentic tool of choice. -- **[Usage Guide](/docs/guide)** — an end-to-end walkthrough from raw ADF JSON to a deployable bundle. -- **[AI Tools & Skills](/docs/ai-tools-skills)** — the agent skills shipped with flowx and how each tool loads them. +- **[Installation](/flowx/docs/installation)** — install the flowx plugin in your agentic tool of choice. +- **[Usage Guide](/flowx/docs/guide)** — an end-to-end walkthrough from raw ADF JSON to a deployable bundle. diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index 84a17d1..21b44ee 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -44,7 +44,7 @@ You can also copy the skill folders into your local `/.claude/skills` folder: cp -R skills/{ingest,translate,prepare,migrate} ~/.claude/skills/ ``` -Once installed, the skills are can be invoked using `/flowx:migrate`, `/flowx:ingest`, etc. Claude Code can also invoke the skills when prompted. See the [Claude Code Plugins documentation](https://docs.claude.com/en/docs/claude-code/plugins) for more details. +Once installed, the skills can be invoked using `/flowx:migrate`, `/flowx:ingest`, etc. Claude Code can also invoke the skills when prompted. See the [Claude Code Plugins documentation](https://docs.claude.com/en/docs/claude-code/plugins) for more details. @@ -54,8 +54,8 @@ Any tool that supports the [Agent Skills](https://agentskills.io/) open standard 2. Make sure the path contains `SKILL.md` directly, 3. Restart the tool if it caches skill metadata at startup. - -If your tool expects a single Markdown file instead of a directory tree, use the following command to concatenate the skills: + +If your tool expects a single Markdown file instead of a directory tree, use the following command to flatten Flowx's skills files: ```bash cat skills/*/SKILL.md > flowx-skills.md @@ -64,7 +64,7 @@ cat skills/*/SKILL.md > flowx-skills.md -## Verifying the install +## Verifying the installation Open your agent and ask: From 43db10d5a8ad7b9850eee525951b7c0aa6dc1e3d Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Thu, 14 May 2026 09:24:26 -0400 Subject: [PATCH 05/77] Vendor workspace artifacts into prepared bundles (#7) * Vendor workspace artifacts into prepared bundles Lets the prepare phase download notebook source (and existing JAR/Python support) from a Databricks workspace so the generated DAB is self-contained and deployable across environments without depending on the source workspace holding the original notebook tree. * Opt-in toggle in `workspace_downloader` (`enable_workspace_downloads()` / `workspace_downloads_enabled()`) so library callers preserve current behavior; the CLI flips it on by default. * Pre-flight auth gate: `auth_available()` checks `.databrickscfg`, `DATABRICKS_CONFIG_PROFILE`, and `DATABRICKS_HOST`+`DATABRICKS_TOKEN`. `prompt_for_auth_if_missing()` surfaces an interactive prompt with explicit `databricks auth login` instructions before the prepare pass runs. * `dab_writer` walks the report ahead of time, collects workspace-resident paths from nested control-flow activities, prompts (and aborts on `n`), then enables downloads. New CLI flags: `--profile`, `--no-vendor-workspace-files`. * `NotebookActivity` preparer downloads the workspace source on hit, vendors to `src/notebooks/.py`, rewrites `notebook_task.notebook_path` to the bundle-relative path, and binds `job_cluster_key=default_cluster` so the downloaded code keeps classic-compute parity (the post-process bind step skips `../src/` paths reserved for orchestra-generated serverless notebooks). * `workspace_notebook_filename()` preserves the workspace basename verbatim (case, underscores, digits); falls back to `notebook_filename()` only when the path yields no usable segment. Strips `.py`, folds other extensions into the stem to avoid collisions between e.g. `runner.sql` and `runner.py`. * `write_notebooks` now coalesces identical writes, disambiguates content collisions with a `__N` suffix, and logs a warning so users know two workspace paths share a basename rather than silently overwriting one. * SKILL.md documents the new behavior, the auth prompt, and the two flags. * Tests: new `test_naming.py`, `test_notebook_writer.py`; updated `test_preparers.py` and `test_workspace_downloader.py` to cover the toggle, auth helpers, vendor-success branch, in-place fallback, basename preservation, and the verbatim test_notebook_001 case. Co-authored-by: Isaac * Format --- skills/prepare/SKILL.md | 24 ++++- src/orchestra/bundler/dab_writer.py | 81 ++++++++++++++++ src/orchestra/bundler/notebook_writer.py | 61 +++++++++++- .../preparer/activity_preparers/naming.py | 30 ++++++ .../preparer/activity_preparers/notebook.py | 23 ++++- .../preparer/workspace_downloader.py | 94 +++++++++++++++++++ tests/unit/test_naming.py | 66 +++++++++++++ tests/unit/test_notebook_writer.py | 50 ++++++++++ tests/unit/test_preparers.py | 62 ++++++++++++ tests/unit/test_workspace_downloader.py | 70 +++++++++++++- 10 files changed, 555 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_naming.py create mode 100644 tests/unit/test_notebook_writer.py diff --git a/skills/prepare/SKILL.md b/skills/prepare/SKILL.md index 2ba7353..66ef4bc 100644 --- a/skills/prepare/SKILL.md +++ b/skills/prepare/SKILL.md @@ -50,6 +50,7 @@ Ask the user for the following (provide defaults): | Bundle name | Name for the DABs project | derived from first pipeline name | | Target environments | Deployment targets to configure | `dev, staging, prod` | | Warehouse ID | SQL warehouse for SQL tasks (optional) | prompt if SQL tasks exist | +| Databricks CLI profile | Profile used to download workspace-resident notebooks / JARs / Python files (`--profile`). Required only when the bundle references absolute workspace paths. | resolved from `~/.databrickscfg` (auto-prompt if multiple) | ### Step 3 — Run bundle generation @@ -62,7 +63,8 @@ python3 /src/flowx/bundler/dab_writer.py \ --catalog \ --schema \ --bundle-name \ - --targets + [--profile ] \ + [--no-vendor-workspace-files] ``` Where: @@ -70,6 +72,26 @@ Where: - `` is the path to `translation_report.json` - Other parameters are from step 2 +**Workspace artifact vendoring (default: enabled).** When the report references workspace-resident notebooks (`/Shared/...`), DBFS Spark JARs (`dbfs:/...`), or Spark Python files, the preparer downloads them via the Databricks CLI auth so the resulting bundle is self-contained and deployable across environments. Downloaded notebooks are vendored under `src/notebooks/` and bound to the default `job_cluster` (since they may rely on classic-compute features). The original `notebook_path` in the resource YAML is rewritten to the bundle-relative path `../src/notebooks/.py`. + +If no Databricks CLI auth is detected on the host (`~/.databrickscfg` empty AND no `DATABRICKS_CONFIG_PROFILE` / `DATABRICKS_HOST`+`DATABRICKS_TOKEN` env vars), the CLI prints the workspace paths it was about to download and prompts: + +``` +Workspace downloads are enabled but no Databricks CLI auth was found. + Looked for profiles in: /Users//.databrickscfg + Artifacts to vendor: /Shared/ETL/transform, … + +To authenticate, run one of: + databricks auth login --host https://.cloud.databricks.com + databricks configure --token + +Continue with placeholders (downloads will be skipped)? [y/N]: +``` + +Answering `n` aborts with exit code 2 so the user can authenticate and re-run. Answering `y` continues with placeholder notebooks (legacy in-place workspace paths). In non-interactive sessions the prompt defaults to placeholders. + +Use `--no-vendor-workspace-files` to opt out entirely; the bundle then keeps original workspace paths exactly as in the IR. + ### Step 4 — Present the generated file tree Show the user what was generated: diff --git a/src/orchestra/bundler/dab_writer.py b/src/orchestra/bundler/dab_writer.py index 29da7d5..22d7524 100644 --- a/src/orchestra/bundler/dab_writer.py +++ b/src/orchestra/bundler/dab_writer.py @@ -43,6 +43,11 @@ WebActivity, ) from flowx.preparer.workflow_preparer import PreparedWorkflow, prepare_workflow +from flowx.preparer.workspace_downloader import ( + enable_workspace_downloads, + prompt_for_auth_if_missing, + set_profile, +) from flowx.utils import normalize_task_key @@ -277,12 +282,40 @@ def main() -> None: default=None, help="Override the bundle name (defaults to the workflow name).", ) + parser.add_argument( + "--profile", + type=str, + default=None, + help="Databricks CLI profile to use when downloading workspace artifacts.", + ) + parser.add_argument( + "--no-vendor-workspace-files", + action="store_true", + help=( + "Skip downloading workspace-resident notebooks / Python files / JARs. " + "Tasks keep their original workspace paths and the bundle is not self-contained." + ), + ) args = parser.parse_args() if not args.report.exists(): print(f"Error: Report file not found: {args.report}", file=sys.stderr) sys.exit(1) + if args.profile: + set_profile(args.profile) + + if not args.no_vendor_workspace_files: + workspace_paths = _collect_workspace_artifact_paths(args.report) + if workspace_paths: + if not prompt_for_auth_if_missing(workspace_paths): + print( + "Aborted. Run `databricks auth login --host ` and retry.", + file=sys.stderr, + ) + sys.exit(2) + enable_workspace_downloads(True) + print(f"Loading translation report: {args.report}") workflows = _load_report(args.report) @@ -743,6 +776,54 @@ def _normalize_base_parameters( return resolved +def _collect_workspace_artifact_paths(report_path: Path) -> list[str]: + """Return workspace-resident artifact paths the bundler would try to download. + + Used as a pre-flight before invoking the preparers: when the report + contains any absolute workspace paths (``/Shared/...``, ``/Workspace/...``) + or DBFS / Volume URIs, we want to surface them to the user so they can + authenticate before the prepare pass. + """ + try: + with open(report_path, encoding="utf-8") as report_file: + report = json.load(report_file) + except (OSError, json.JSONDecodeError): + return [] + + candidates: list[str] = [] + + def _walk_tasks(tasks: list[dict[str, Any]] | None) -> None: + for task in tasks or []: + task_type = task.get("type") + if task_type == "NotebookActivity": + path = task.get("notebook_path") or "" + if isinstance(path, str) and path.startswith("/") and not path.startswith("../"): + candidates.append(path) + elif task_type == "SparkPythonActivity": + path = task.get("python_file") or "" + if isinstance(path, str) and (path.startswith("dbfs:") or path.startswith("/")): + candidates.append(path) + elif task_type == "SparkJarActivity": + for lib in task.get("libraries") or []: + jar = lib.get("jar") if isinstance(lib, dict) else None + if isinstance(jar, str) and (jar.startswith("dbfs:") or jar.startswith("/")): + candidates.append(jar) + _walk_tasks(task.get("inner_activities")) + _walk_tasks(task.get("if_true_activities")) + _walk_tasks(task.get("if_false_activities")) + for case in task.get("cases") or []: + _walk_tasks(case.get("activities")) + _walk_tasks(task.get("default_activities")) + + if "tasks" in report: + _walk_tasks(report.get("tasks")) + for translation in report.get("translations") or []: + ir = translation.get("ir") or {} + _walk_tasks(ir.get("tasks")) + + return candidates + + def _load_report(report_path: Path) -> list[PreparedWorkflow]: """Loads a translation report and reconstruct PreparedWorkflow objects. diff --git a/src/orchestra/bundler/notebook_writer.py b/src/orchestra/bundler/notebook_writer.py index 7817b7a..05285bb 100644 --- a/src/orchestra/bundler/notebook_writer.py +++ b/src/orchestra/bundler/notebook_writer.py @@ -2,21 +2,78 @@ from __future__ import annotations +import logging from pathlib import Path from flowx.models.dab import DabNotebook +logger = logging.getLogger(__name__) + + +def _content_signature(notebook: DabNotebook) -> object: + """Return a value suitable for comparing two notebooks for byte-equivalence.""" + if notebook.binary_content is not None: + return ("binary", notebook.binary_content) + return ("text", notebook.content) + + +def _disambiguate(relative_path: str, taken: set[str]) -> str: + """Append a numeric suffix to ``relative_path`` until it is unique within ``taken``.""" + if relative_path not in taken: + return relative_path + parent, _, name = relative_path.rpartition("/") + stem, dot, ext = name.rpartition(".") + if not stem: + stem, dot, ext = name, "", "" + n = 1 + while True: + candidate_name = f"{stem}__{n}{dot}{ext}" if dot else f"{stem}__{n}" + candidate = f"{parent}/{candidate_name}" if parent else candidate_name + if candidate not in taken: + return candidate + n += 1 + def write_notebooks(notebooks: list[DabNotebook], output_dir: Path) -> list[Path]: - """Writes each notebook to ``output_dir/`` and returns the absolute paths.""" + """Writes each notebook to ``output_dir/`` and returns the absolute paths. + + Two notebooks may legitimately share a ``relative_path`` (e.g., when two + ADF activities reference the same workspace notebook). Identical writes + are coalesced into one. When the contents differ — which usually means + two workspace paths produced the same basename — the second write is + given a ``__N`` suffix and a warning is logged so the user can decide + whether to disambiguate the source activity names or workspace paths. + """ created: list[Path] = [] + written_signatures: dict[str, object] = {} + taken_paths: set[str] = set() + for notebook in notebooks: - destination = output_dir / notebook.relative_path + target = notebook.relative_path + signature = _content_signature(notebook) + existing = written_signatures.get(target) + if existing is not None: + if existing == signature: + # Identical write — skip the duplicate file operation. + continue + new_target = _disambiguate(target, taken_paths) + logger.warning( + "Two notebooks resolved to the same bundle path %s with different " + "contents; writing the second copy to %s. This typically means " + "two workspace notebook paths share a basename — rename one in " + "the workspace or adjust the ADF activity name to disambiguate.", + target, + new_target, + ) + target = new_target + destination = output_dir / target destination.parent.mkdir(parents=True, exist_ok=True) if notebook.binary_content is not None: destination.write_bytes(notebook.binary_content) else: content = notebook.content if notebook.content.endswith("\n") else notebook.content + "\n" destination.write_text(content, encoding="utf-8") + written_signatures[target] = signature + taken_paths.add(target) created.append(destination.resolve()) return created diff --git a/src/orchestra/preparer/activity_preparers/naming.py b/src/orchestra/preparer/activity_preparers/naming.py index 30214fb..416a9d2 100644 --- a/src/orchestra/preparer/activity_preparers/naming.py +++ b/src/orchestra/preparer/activity_preparers/naming.py @@ -28,3 +28,33 @@ def notebook_filename(task_key: str, activity_name: str | None = None) -> str: if not snake: snake = task_key.lower() or "notebook" return f"{snake}.py" + + +def workspace_notebook_filename(workspace_path: str) -> str: + """Derive a bundle filename from a workspace notebook path's basename. + + Preserves the workspace name verbatim (case, underscores, digits) so a + downloaded notebook lands at ``src/notebooks/.py`` instead of + being renamed to the ADF activity's task_key. Sanitises filesystem-unsafe + characters and ensures a ``.py`` extension. + + Returns an empty string when the path yields no usable segment so the + caller can fall back to the activity-name-based naming. + """ + if not workspace_path: + return "" + basename = workspace_path.rsplit("/", 1)[-1].strip() + if not basename: + return "" + stem, _, ext = basename.rpartition(".") + if not stem: + stem, ext = basename, "" + safe_stem = re.sub(r"[^A-Za-z0-9._-]+", "_", stem).strip("._-") + if not safe_stem: + return "" + if ext.lower() == "py": + return f"{safe_stem}.py" + if ext: + safe_ext = re.sub(r"[^A-Za-z0-9]+", "", ext) + return f"{safe_stem}_{safe_ext}.py" if safe_ext else f"{safe_stem}.py" + return f"{safe_stem}.py" diff --git a/src/orchestra/preparer/activity_preparers/notebook.py b/src/orchestra/preparer/activity_preparers/notebook.py index 3f8e460..79f0227 100644 --- a/src/orchestra/preparer/activity_preparers/notebook.py +++ b/src/orchestra/preparer/activity_preparers/notebook.py @@ -7,9 +7,9 @@ from flowx.models.dab import DabNotebook from flowx.models.ir import TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string -from flowx.preparer.activity_preparers.naming import notebook_filename +from flowx.preparer.activity_preparers.naming import notebook_filename, workspace_notebook_filename from flowx.preparer.workflow_preparer import PreparedActivity, build_common_task_fields -from flowx.preparer.workspace_downloader import download_notebook +from flowx.preparer.workspace_downloader import download_notebook, workspace_downloads_enabled if TYPE_CHECKING: from flowx.models.ir import NotebookActivity @@ -114,6 +114,25 @@ def prepare( ) if is_existing_notebook: + downloaded = download_notebook(resolved_path) if workspace_downloads_enabled() else None + if downloaded is not None: + # Preserve the workspace basename so the bundle file mirrors the + # source notebook name; fall back to the activity-derived snake + # case when the workspace path is unusable (empty trailing + # segment, all special chars, etc.). + filename = workspace_notebook_filename(resolved_path) or notebook_filename(activity.task_key, activity.name) + notebook_relative_path = f"notebooks/{filename}" + task["notebook_task"] = {"notebook_path": f"../src/{notebook_relative_path}"} + if base_parameters is not None: + task["notebook_task"]["base_parameters"] = base_parameters + # Downloaded notebooks were authored for classic compute and may + # use init scripts or DBR-only features that serverless can't run. + # _bind_cluster_to_notebook_tasks skips "../src/" paths because + # flowx-generated notebooks target serverless, so bind here. + task["job_cluster_key"] = "default_cluster" + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=downloaded)] + return PreparedActivity(task=task, notebooks=notebooks) + task["notebook_task"] = {"notebook_path": resolved_path} if base_parameters is not None: task["notebook_task"]["base_parameters"] = base_parameters diff --git a/src/orchestra/preparer/workspace_downloader.py b/src/orchestra/preparer/workspace_downloader.py index fcf0503..e5167a1 100644 --- a/src/orchestra/preparer/workspace_downloader.py +++ b/src/orchestra/preparer/workspace_downloader.py @@ -6,6 +6,8 @@ import configparser import logging import os +import sys +from collections.abc import Iterable from pathlib import Path logger = logging.getLogger(__name__) @@ -17,6 +19,11 @@ _resolved_profile: str | None = None _profile_resolved: bool = False +# When False (default), preparers preserve workspace artifact paths in-place +# instead of attempting a network download. The CLI flips this on so that +# `databricks bundle deploy` can ship the source files across environments. +_downloads_enabled: bool = False + def _get_databrickscfg_path() -> Path: """Return the path to the Databricks CLI config file.""" @@ -208,3 +215,90 @@ def download_dbfs_file(dbfs_path: str) -> bytes | None: except Exception as e: logger.warning("Failed to download DBFS file %s: %s", dbfs_path, e) return None + + +# --------------------------------------------------------------------------- +# Opt-in toggle + auth gating +# --------------------------------------------------------------------------- + + +def enable_workspace_downloads(enabled: bool = True) -> None: + """Globally enable or disable workspace artifact downloads.""" + global _downloads_enabled # noqa: PLW0603 + _downloads_enabled = bool(enabled) + + +def workspace_downloads_enabled() -> bool: + """Return True iff preparers should attempt to download workspace artifacts.""" + return _downloads_enabled + + +def auth_available() -> bool: + """Return True iff there is any usable Databricks authentication on this host. + + A resolvable ``.databrickscfg`` profile, ``DATABRICKS_CONFIG_PROFILE``, or + the standard ``DATABRICKS_HOST`` + ``DATABRICKS_TOKEN`` env-var pair will + all satisfy this check. This is a pre-flight signal — it does not validate + that the credentials actually authorize against any specific workspace. + """ + if os.environ.get("DATABRICKS_CONFIG_PROFILE"): + return True + if os.environ.get("DATABRICKS_HOST") and os.environ.get("DATABRICKS_TOKEN"): + return True + return bool(_list_profiles()) + + +def prompt_for_auth_if_missing( + sample_paths: Iterable[str], + *, + interactive: bool | None = None, +) -> bool: + """Warn the user when auth is missing and confirm how to proceed. + + Args: + sample_paths: Workspace paths the preparer is about to try to download. + Used in the on-screen instructions so the user knows where the + artifacts they're missing live. + interactive: Force interactive prompting on/off. Default ``None`` + auto-detects via ``sys.stdin.isatty()``. + + Returns: + ``True`` if the user wants to continue with placeholders, ``False`` + if the caller should abort so the user can run ``databricks auth login``. + """ + if auth_available(): + return True + + paths = [p for p in sample_paths if p] + cfg_path = _get_databrickscfg_path() + + print( + "\nWorkspace downloads are enabled but no Databricks CLI auth was found.", + file=sys.stderr, + ) + print(f" Looked for profiles in: {cfg_path}", file=sys.stderr) + if paths: + preview = ", ".join(paths[:3]) + suffix = ", …" if len(paths) > 3 else "" + print(f" Artifacts to vendor: {preview}{suffix}", file=sys.stderr) + print( + "\nTo authenticate, run one of:\n" + " databricks auth login --host https://.cloud.databricks.com\n" + " databricks configure --token # legacy PAT flow\n", + file=sys.stderr, + ) + + if interactive is None: + interactive = sys.stdin.isatty() + if not interactive: + print( + "Non-interactive session; skipping downloads and using placeholders.", + file=sys.stderr, + ) + return True + + try: + choice = input("Continue with placeholders (downloads will be skipped)? [y/N]: ").strip().lower() + except (EOFError, KeyboardInterrupt): + return True + return choice in ("y", "yes") diff --git a/tests/unit/test_naming.py b/tests/unit/test_naming.py new file mode 100644 index 0000000..0045f66 --- /dev/null +++ b/tests/unit/test_naming.py @@ -0,0 +1,66 @@ +"""Unit tests for the naming helpers used by activity preparers.""" + +from __future__ import annotations + +import pytest + +from flowx.preparer.activity_preparers.naming import ( + notebook_filename, + to_snake_case, + workspace_notebook_filename, +) + + +class TestToSnakeCase: + @pytest.mark.parametrize( + ("source", "expected"), + [ + ("BronzeIngest", "bronze_ingest"), + ("copySQLToBlob", "copy_sql_to_blob"), + ("ETL_Main", "etl_main"), + ("spaces and dashes-here", "spaces_and_dashes_here"), + ], + ) + def test_converts_to_snake(self, source: str, expected: str) -> None: + assert to_snake_case(source) == expected + + +class TestNotebookFilename: + def test_uses_activity_name_when_provided(self) -> None: + assert notebook_filename("bronze_ingest", "BronzeIngest") == "bronze_ingest.py" + + def test_falls_back_to_task_key_when_no_activity_name(self) -> None: + assert notebook_filename("bronze_ingest", None) == "bronze_ingest.py" + + def test_default_when_both_inputs_empty(self) -> None: + assert notebook_filename("", None) == "notebook.py" + + +class TestWorkspaceNotebookFilename: + def test_preserves_basename_verbatim(self) -> None: + """Underscores, digits, and case are preserved exactly — no snake casing.""" + assert workspace_notebook_filename("/Shared/test_notebook_001") == "test_notebook_001.py" + assert workspace_notebook_filename("/Workspace/Users/foo/MyNotebook") == "MyNotebook.py" + + def test_strips_existing_py_extension(self) -> None: + """A workspace path that already has ``.py`` is not double-suffixed.""" + assert workspace_notebook_filename("/Shared/etl/runner.py") == "runner.py" + + def test_non_py_extension_is_folded_into_stem(self) -> None: + """A non-``.py`` extension is kept as part of the bundle filename stem to + preserve disambiguation between e.g. ``foo.sql`` and ``foo.py``.""" + assert workspace_notebook_filename("/Shared/etl/runner.sql") == "runner_sql.py" + + def test_sanitises_special_characters(self) -> None: + """Spaces collapse to underscores; trailing separator characters are stripped.""" + assert workspace_notebook_filename("/Shared/My Folder/My Notebook!") == "My_Notebook.py" + assert workspace_notebook_filename("/Shared/foo bar baz") == "foo_bar_baz.py" + + def test_returns_empty_for_no_segment(self) -> None: + """Empty / trailing-slash paths return an empty string so the caller can + fall back to the activity-derived name.""" + assert workspace_notebook_filename("") == "" + assert workspace_notebook_filename("/Shared/") == "" + + def test_returns_empty_when_only_special_chars(self) -> None: + assert workspace_notebook_filename("/Shared/!!!") == "" diff --git a/tests/unit/test_notebook_writer.py b/tests/unit/test_notebook_writer.py new file mode 100644 index 0000000..9120a73 --- /dev/null +++ b/tests/unit/test_notebook_writer.py @@ -0,0 +1,50 @@ +"""Unit tests for the DAB notebook writer's collision handling.""" + +from __future__ import annotations + +import logging +from pathlib import Path + +from flowx.bundler.notebook_writer import write_notebooks +from flowx.models.dab import DabNotebook + + +class TestWriteNotebooks: + def test_writes_single_notebook(self, tmp_path: Path) -> None: + out = write_notebooks( + [DabNotebook(relative_path="notebooks/a.py", content="print('a')")], + tmp_path, + ) + assert len(out) == 1 + assert (tmp_path / "notebooks" / "a.py").read_text().rstrip() == "print('a')" + + def test_coalesces_identical_duplicates(self, tmp_path: Path) -> None: + """Two writes with identical content + path are written once, not twice.""" + notebooks = [ + DabNotebook(relative_path="notebooks/a.py", content="print('a')"), + DabNotebook(relative_path="notebooks/a.py", content="print('a')"), + ] + out = write_notebooks(notebooks, tmp_path) + assert len(out) == 1 + assert (tmp_path / "notebooks" / "a.py").read_text().rstrip() == "print('a')" + + def test_disambiguates_on_content_collision(self, tmp_path: Path, caplog) -> None: + """Same path + different content: the second write is suffixed __1 and a warning fires.""" + notebooks = [ + DabNotebook(relative_path="notebooks/run.py", content="print('first')"), + DabNotebook(relative_path="notebooks/run.py", content="print('second')"), + ] + with caplog.at_level(logging.WARNING, logger="flowx.bundler.notebook_writer"): + out = write_notebooks(notebooks, tmp_path) + assert len(out) == 2 + assert (tmp_path / "notebooks" / "run.py").read_text().rstrip() == "print('first')" + assert (tmp_path / "notebooks" / "run__1.py").read_text().rstrip() == "print('second')" + assert any("share a basename" in rec.message for rec in caplog.records) + + def test_binary_content_takes_precedence_over_text(self, tmp_path: Path) -> None: + out = write_notebooks( + [DabNotebook(relative_path="lib/x.jar", binary_content=b"\x01\x02\x03")], + tmp_path, + ) + assert len(out) == 1 + assert (tmp_path / "lib" / "x.jar").read_bytes() == b"\x01\x02\x03" diff --git a/tests/unit/test_preparers.py b/tests/unit/test_preparers.py index 891a497..17b318a 100644 --- a/tests/unit/test_preparers.py +++ b/tests/unit/test_preparers.py @@ -28,6 +28,7 @@ WaitActivity, WebActivity, ) +from flowx.preparer import workspace_downloader from flowx.preparer.workflow_preparer import ( PreparedActivity, PreparedWorkflow, @@ -87,6 +88,67 @@ def test_prepare_notebook_no_params(self): # No placeholder for an absolute workspace path. assert prepared.notebooks == [] + def test_prepare_notebook_vendors_downloaded_workspace_notebook(self, monkeypatch): + """When downloads are enabled and the SDK returns content, the workspace notebook + is vendored into src/notebooks/ under the workspace basename, and the task is + bound to the default cluster.""" + monkeypatch.setattr(workspace_downloader, "_downloads_enabled", True) + monkeypatch.setattr( + "flowx.preparer.activity_preparers.notebook.download_notebook", + lambda path: "# Databricks notebook source\nprint('hi from /Shared/ETL/transform')\n", + ) + activity = NotebookActivity( + **_make_base("Run NB", "run_nb"), + notebook_path="/Shared/ETL/transform", + base_parameters={"env": "dev"}, + ) + prepared = prepare_activity(activity) + # Path rewritten to bundle-local; basename mirrors the workspace name + # ("transform"), not the activity task_key ("run_nb"). + assert prepared.task["notebook_task"]["notebook_path"] == "../src/notebooks/transform.py" + # Cluster bound explicitly by the preparer (the post-process bind step + # skips ../src/ paths because flowx-generated notebooks are + # serverless-only; downloaded notebooks need classic compute). + assert prepared.task["job_cluster_key"] == "default_cluster" + # Notebook vendored under the workspace basename + assert len(prepared.notebooks) == 1 + assert prepared.notebooks[0].relative_path == "notebooks/transform.py" + assert "from /Shared/ETL/transform" in prepared.notebooks[0].content + # Params preserved + assert prepared.task["notebook_task"]["base_parameters"] == {"env": "dev"} + + def test_prepare_notebook_preserves_workspace_basename_verbatim(self, monkeypatch): + """Underscored / numbered workspace names like test_notebook_001 are preserved as-is.""" + monkeypatch.setattr(workspace_downloader, "_downloads_enabled", True) + monkeypatch.setattr( + "flowx.preparer.activity_preparers.notebook.download_notebook", + lambda path: "# Databricks notebook source\nprint('hello')\n", + ) + activity = NotebookActivity( + **_make_base("Bronze Ingest", "BronzeIngest"), + notebook_path="/Shared/test_notebook_001", + ) + prepared = prepare_activity(activity) + assert prepared.task["notebook_task"]["notebook_path"] == "../src/notebooks/test_notebook_001.py" + assert prepared.notebooks[0].relative_path == "notebooks/test_notebook_001.py" + + def test_prepare_notebook_falls_back_to_in_place_when_download_fails(self, monkeypatch): + """If downloads are enabled but the SDK returns None, behavior matches the + legacy in-place reference (no vendor, no cluster bind in the preparer).""" + monkeypatch.setattr(workspace_downloader, "_downloads_enabled", True) + monkeypatch.setattr( + "flowx.preparer.activity_preparers.notebook.download_notebook", + lambda path: None, + ) + activity = NotebookActivity( + **_make_base("NB", "nb"), + notebook_path="/Shared/missing", + ) + prepared = prepare_activity(activity) + assert prepared.task["notebook_task"]["notebook_path"] == "/Shared/missing" + assert "job_cluster_key" not in prepared.task + assert prepared.notebooks == [] + def test_prepare_notebook_resolves_expression_params(self): """ADF expression params are mapped to DAB dynamic value references.""" activity = NotebookActivity( diff --git a/tests/unit/test_workspace_downloader.py b/tests/unit/test_workspace_downloader.py index 73a6a78..da35d7f 100644 --- a/tests/unit/test_workspace_downloader.py +++ b/tests/unit/test_workspace_downloader.py @@ -4,7 +4,15 @@ from unittest.mock import patch -from flowx.preparer.workspace_downloader import download_dbfs_file, download_notebook +from flowx.preparer import workspace_downloader +from flowx.preparer.workspace_downloader import ( + auth_available, + download_dbfs_file, + download_notebook, + enable_workspace_downloads, + prompt_for_auth_if_missing, + workspace_downloads_enabled, +) class TestDownloadNotebook: @@ -50,3 +58,63 @@ def mock_import(name, *args, **kwargs): with patch("builtins.__import__", side_effect=mock_import): result = download_dbfs_file("dbfs:/scripts/etl.py") assert result is None + + +class TestDownloadsToggle: + def test_disabled_by_default(self, monkeypatch): + """The module-level toggle defaults to False so library users keep current behavior.""" + monkeypatch.setattr(workspace_downloader, "_downloads_enabled", False) + assert workspace_downloads_enabled() is False + + def test_enable_and_disable(self, monkeypatch): + monkeypatch.setattr(workspace_downloader, "_downloads_enabled", False) + enable_workspace_downloads(True) + try: + assert workspace_downloads_enabled() is True + enable_workspace_downloads(False) + assert workspace_downloads_enabled() is False + finally: + monkeypatch.setattr(workspace_downloader, "_downloads_enabled", False) + + +class TestAuthAvailable: + def test_returns_true_when_env_profile_set(self, monkeypatch): + monkeypatch.setenv("DATABRICKS_CONFIG_PROFILE", "DEFAULT") + monkeypatch.delenv("DATABRICKS_HOST", raising=False) + monkeypatch.delenv("DATABRICKS_TOKEN", raising=False) + assert auth_available() is True + + def test_returns_true_when_host_and_token_set(self, monkeypatch): + monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False) + monkeypatch.setenv("DATABRICKS_HOST", "https://example.cloud.databricks.com") + monkeypatch.setenv("DATABRICKS_TOKEN", "dapi-abc") + assert auth_available() is True + + def test_returns_false_when_no_env_and_no_profiles(self, monkeypatch): + monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False) + monkeypatch.delenv("DATABRICKS_HOST", raising=False) + monkeypatch.delenv("DATABRICKS_TOKEN", raising=False) + monkeypatch.setattr(workspace_downloader, "_list_profiles", lambda: []) + assert auth_available() is False + + +class TestPromptForAuthIfMissing: + def test_no_prompt_when_auth_available(self, monkeypatch): + monkeypatch.setattr(workspace_downloader, "auth_available", lambda: True) + assert prompt_for_auth_if_missing(["/Shared/foo"]) is True + + def test_non_interactive_falls_back_to_placeholders(self, monkeypatch, capsys): + monkeypatch.setattr(workspace_downloader, "auth_available", lambda: False) + assert prompt_for_auth_if_missing(["/Shared/foo"], interactive=False) is True + err = capsys.readouterr().err + assert "databricks auth login" in err + + def test_interactive_user_aborts(self, monkeypatch): + monkeypatch.setattr(workspace_downloader, "auth_available", lambda: False) + monkeypatch.setattr("builtins.input", lambda _prompt: "n") + assert prompt_for_auth_if_missing(["/Shared/foo"], interactive=True) is False + + def test_interactive_user_accepts(self, monkeypatch): + monkeypatch.setattr(workspace_downloader, "auth_available", lambda: False) + monkeypatch.setattr("builtins.input", lambda _prompt: "y") + assert prompt_for_auth_if_missing(["/Shared/foo"], interactive=True) is True From 91220d4ebb377a14a17c52d4c72f1d8d6f9e4d13 Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Thu, 28 May 2026 20:31:18 -0400 Subject: [PATCH 06/77] Add user-interface for specifying options (#8) --- .build-constraints.txt | 19 + Makefile | 32 +- docs/content/docs/index.mdx | 1 + docs/content/docs/meta.json | 1 + docs/content/docs/options.mdx | 74 ++ pyproject.toml | 5 + skills/migrate/SKILL.md | 95 +- skills/prepare/SKILL.md | 47 + skills/translate/SKILL.md | 105 ++ src/orchestra/adapter/__init__.py | 96 ++ src/orchestra/adapter/__main__.py | 594 +++++++++ src/orchestra/adapter/constants.py | 74 ++ src/orchestra/adapter/models.py | 284 ++++ src/orchestra/adapter/operations.py | 1125 ++++++++++++++++ src/orchestra/adapter/predicates.py | 243 ++++ src/orchestra/adapter/session.py | 431 ++++++ src/orchestra/bundler/constants.py | 21 + src/orchestra/bundler/dab_writer.py | 310 +++-- src/orchestra/models/ir.py | 31 +- src/orchestra/models/motifs.py | 16 + .../preparer/activity_preparers/copy.py | 302 ++++- .../preparer/activity_preparers/motif.py | 139 +- .../preparer/activity_preparers/notebook.py | 11 +- src/orchestra/preparer/code_generator.py | 105 +- src/orchestra/preparer/workflow_preparer.py | 104 +- .../translator/activity_translators/copy.py | 309 ++++- src/orchestra/translator/engine.py | 37 +- src/orchestra/translator/query_analysis.py | 287 ++++ tests/unit/test_adapter.py | 1158 +++++++++++++++++ tests/unit/test_query_analysis.py | 174 +++ uv.lock | 360 ++++- 31 files changed, 6463 insertions(+), 127 deletions(-) create mode 100644 .build-constraints.txt create mode 100644 docs/content/docs/options.mdx create mode 100644 src/orchestra/adapter/__init__.py create mode 100644 src/orchestra/adapter/__main__.py create mode 100644 src/orchestra/adapter/constants.py create mode 100644 src/orchestra/adapter/models.py create mode 100644 src/orchestra/adapter/operations.py create mode 100644 src/orchestra/adapter/predicates.py create mode 100644 src/orchestra/adapter/session.py create mode 100644 src/orchestra/bundler/constants.py create mode 100644 src/orchestra/translator/query_analysis.py create mode 100644 tests/unit/test_adapter.py create mode 100644 tests/unit/test_query_analysis.py diff --git a/.build-constraints.txt b/.build-constraints.txt new file mode 100644 index 0000000..48078f6 --- /dev/null +++ b/.build-constraints.txt @@ -0,0 +1,19 @@ +hatchling==1.29.0 \ + --hash=sha256:50af9343281f34785fab12da82e445ed987a6efb34fd8c2fc0f6e6630dbcc1b0 \ + --hash=sha256:793c31816d952cee405b83488ce001c719f325d9cda69f1fc4cd750527640ea6 +packaging==26.2 \ + --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ + --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 + # via hatchling +pathspec==1.1.1 \ + --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ + --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 + # via hatchling +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via hatchling +trove-classifiers==2026.5.20.19 \ + --hash=sha256:6e611993987ca9326968ad70452733dadd31471599d39896045b28970a9bb81e \ + --hash=sha256:7a173916960d0635fcbf610550d2c27bcc9125164d6f397adf46fc1ef6455c7c + # via hatchling diff --git a/Makefile b/Makefile index 34416d7..451dcde 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean dev ci test integration fmt help docs-install docs-clean docs-build docs-serve +.PHONY: clean dev ci test integration fmt help docs-install docs-clean docs-build docs-serve lock-dependencies clean: rm -rf .venv .pytest_cache .ruff_cache .mypy_cache __pycache__ @@ -34,15 +34,25 @@ docs-build: docs-install docs-serve: docs-build cd docs && bun run dev + +lock-dependencies: export UV_FROZEN := 0 +lock-dependencies: + uv lock --exclude-newer "7 days" + uv run --exact --all-extras --group yq tomlq -r '.["build-system"].requires[]' pyproject.toml | \ + uv pip compile --generate-hashes --universal --no-header - > build-constraints-new.txt + mv build-constraints-new.txt .build-constraints.txt + perl -pi -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple"|g' uv.lock + help: @echo "Available targets:" - @echo " dev Install dependencies" - @echo " ci Install dependencies (frozen lockfile)" - @echo " test Run unit tests" - @echo " integration Run integration tests" - @echo " fmt Format and lint code" - @echo " clean Remove build artifacts" - @echo " docs-install Install docs dependencies (bun)" - @echo " docs-clean Remove docs build artifacts" - @echo " docs-build Build the static docs site to docs/site" - @echo " docs-serve Run the docs dev server (next dev)" + @echo " dev Install dependencies" + @echo " ci Install dependencies (frozen lockfile)" + @echo " test Run unit tests" + @echo " integration Run integration tests" + @echo " fmt Format and lint code" + @echo " clean Remove build artifacts" + @echo " docs-install Install docs dependencies (bun)" + @echo " docs-clean Remove docs build artifacts" + @echo " docs-build Build the static docs site to docs/site" + @echo " docs-serve Run the docs dev server (next dev)" + @echo " lock-dependencies Write the uv.lock file" diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index ababf16..c8ed057 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -33,3 +33,4 @@ Each phase can be run independently, maintains its own input/output contract, pr - **[Installation](/flowx/docs/installation)** — install the flowx plugin in your agentic tool of choice. - **[Usage Guide](/flowx/docs/guide)** — an end-to-end walkthrough from raw ADF JSON to a deployable bundle. +- **[Options](/flowx/docs/options)** — reference documenting options for customizing output when translating pipelines with flowx. diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index 667f5fb..73219f4 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -5,6 +5,7 @@ "how-it-works", "installation", "guide", + "options", "ai-tools-skills" ] } diff --git a/docs/content/docs/options.mdx b/docs/content/docs/options.mdx new file mode 100644 index 0000000..dd1fd73 --- /dev/null +++ b/docs/content/docs/options.mdx @@ -0,0 +1,74 @@ +--- +title: Translation options +description: Control flowx's translation behavior and outputs +--- + +Flowx defers some architectural choices to allow users to specify properties of the output jobs. When options are available for controlling +translation, the agent session prompts the user for their preferences. + +## Available options + +### `copy_activity_paradigm` + +Controls how Copy Data activities whose sink resolves to a Delta table are translated. + +| Value | Default | Behavior | +|--------------|---------|--------------------------------------------------------------------------------------------| +| `notebook` | True | Generates a Notebook task that copies data using PySpark `read` and `write` methods. | +| `sdp` | False | Generates a Run Pipeline task that copies data using Lakeflow Spark Declarative Pipelines. | + +### `non_databricks_task_compute` + +Controls compute for non-Databricks tasks (e.g. Lookup, Web, Delete, Wait, Filter, and Set Variable activities). + +| Value | Default | Behavior | +|---------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `serverless` | True | Translated tasks run on serverless compute. | +| `classic` | False | Most tasks run on a single-node classic compute cluster; Translated Copy Data tasks run on a fixed-size multi-node cluster. Clusters referenced by tasks are included in the output bundle resources' `job_clusters`. | + +### `use_lakeflow_connectors` + +Controls whether eligible Copy Data activities are replaced with a managed Lakeflow Connect pipeline. + +| Value | Default | Behavior | +|--------------------|---------|------------------------------------------------------------------------------------------------------------------------| +| `existing` | True | Translates the Copy Data activity as a Notebook or Run Pipeline task using PySpark `read` and `write` methods. | +| `lakeflow_connect` | False | Translates the Copy Data activity as a Run Pipeline task that triggers a Lakeflow Connect managed ingestion pipeline. | + +### `lakeflow_connector_type` + +Controls how Lakeflow Connect reads from a database when copying data. + +| Value | Default | Behavior | +|---------------|-----------|---------------------------------------------------------------------------------------------------------------------------------------------------| +| `cdc` | True | Creates an ingestion pipeline using a [CDC-based connector](https://docs.databricks.com/aws/en/ingestion/lakeflow-connect/cdc-overview) | +| `query_based` | False | Creates an ingestion pipeline using a [query-based connector](https://docs.databricks.com/aws/en/ingestion/lakeflow-connect/query-based-overview) | + + +Copy activities that carry an explicit SQL query (`sqlReaderQuery`, `query`, or `sql_query` on `source_properties`) +must use a [query-based connector](https://docs.databricks.com/aws/en/ingestion/lakeflow-connect/query-based-overview). + + +### `databricks_task_compute` + +Controls the compute used to run Databricks Notebook and Spark Python tasks in the translated job. + +| Value | Default | Behavior | +|--------------|---------|-----------------------------------------------------------------------------------------------------------------------------------| +| `existing` | True | Uses the source pipeline's compute definition (e.g. a job cluster). Preserves init scripts, DBR-version, and other configuration. | +| `serverless` | False | Drops the source pipeline's cluster definition; The translated tasks run on serverless compute. | + +### `metadata_driven_consolidate` + +Configures flowx to detect and consolidate ingestion pipelines that read from configuration and run parameterized data copying. Allows users to create a consolidated pipeline. + +| Value | Default | Behavior | +|---------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `keep` | True | Preserves the source pipeline's metadata-driven patterns. | +| `consolidate` | False | Replaces metadata-driven patterns with a consolidated Lakeflow Connect ingestion pipeline whose `objects` list has one entry per row in the source configuration. | + + +The following are required to consolidate into a single ingestion pipeline: +- Access to query the file or database where metadata is stored or a CSV file containing the exported metadata +- The number of metadata rows or objects must be less than 250 + diff --git a/pyproject.toml b/pyproject.toml index 34e9e16..f5b9dac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,8 @@ classifiers = [ ] dependencies = [ "pyyaml>=6.0", + "databricks-sdk>=0.40", + "sqlglot>=25.0", ] [dependency-groups] @@ -32,6 +34,9 @@ dev = [ "mypy>=1.18.2,<2", "types-pyyaml>=6.0.12.20250915,<7", ] +yq = [ + "yq~=3.4.3", +] [build-system] requires = ["hatchling"] diff --git a/skills/migrate/SKILL.md b/skills/migrate/SKILL.md index 222e06d..5c3d910 100644 --- a/skills/migrate/SKILL.md +++ b/skills/migrate/SKILL.md @@ -31,6 +31,22 @@ Each phase builds on the output of the previous phase. The user is shown a summa Follow these steps in order: +### Step 0 — Gather phase inputs via the adapter + +Before invoking ingest, run the adapter inputs subcommand once per +phase so the agent surfaces the matching free-text prompts: + +```bash +python3 -m flowx.adapter inputs ingest +python3 -m flowx.adapter inputs translate +python3 -m flowx.adapter inputs prepare +``` + +Each response carries the questions for that phase plus their +descriptions and defaults. Collect answers from the user (or accept +the defaults), persist them to `//inputs.json`, and +thread the values into the downstream CLI calls. + ### Step 1 — Gather inputs Ask the user for all required inputs upfront: @@ -112,6 +128,57 @@ For failures, suggest: - Retry with additional context - Skip and add placeholder +### Step 5.1 — Gather just-in-time translation preferences + +Drive the loop multi-pass: re-run `inspect --answers ` +after each batch of answers so the adapter can surface chained +metadata-driven prompts. When the user opts to consolidate a +metadata-driven motif and the agent has a database tool, run the +lookup query directly and persist the rows to +`/translate/lookup_values.json`; otherwise prompt the user +for a CSV file or comma-separated string and run: + +```bash +python3 -m flowx.adapter materialize-lookup "" \ + --out /translate/lookup_values.json +``` + +Pass `--lookup-values` to the modify call when the file exists. + +#### Legacy flow details + +Before bundle generation, run the adapter inspect CLI on the translation +report to surface any pipeline-modifier questions the IR raises: + +```bash +python3 -m flowx.adapter inspect /translate/translation_report.json +``` + +For each question in the JSON output, prompt the user with the rationale, +options, and the affected task keys. Collect answers into +`/translate/answers.json` keyed by `question_id`, then apply +them to a stamped report: + +```bash +python3 -m flowx.adapter modify \ + /translate/translation_report.json \ + /translate/answers.json \ + --out /translate/translation_report.stamped.json +``` + +Use the stamped report (when produced) as the input to the prepare phase. +When inspect emits no questions for any pipeline, skip modify and use the +original report. + +The four questions the adapter raises: + +| `question_id` | Allowed values | Default | +|---|---|---| +| `copy_activity_paradigm` | `notebook`, `sdp` | `notebook` | +| `non_databricks_task_compute` | `serverless`, `classic` | `serverless` | +| `use_lakeflow_connectors` | `existing`, `lakeflow_connect` | `existing` | +| `databricks_task_compute` | `existing`, `serverless` | `existing` | + ### Step 6 — Checkpoint: confirm proceed to bundle generation > Translation is 91.5% complete. 4 activities could not be translated automatically. @@ -122,10 +189,36 @@ For failures, suggest: > > What would you like to do? +### Step 6.5 — Detect workspace artifacts and authenticate + +Before invoking the prepare phase, run the adapter's +`workspace-paths` subcommand to detect any absolute workspace paths +the bundle would need to vendor: + +```bash +python3 -m flowx.adapter workspace-paths \ + /translate/translation_report.stamped.json \ + --source-dir +``` + +When the response carries `needs_auth: true`: + +1. Confirm the workspace host with the user, defaulting to the first + entry in `suggested_hosts` (extracted from the Databricks linked + services in the ADF export). +2. Run `databricks auth login --host ` interactively to set up + a local profile. +3. Pass `--profile ` to the prepare invocation in Step 7 so + flowx downloads the referenced notebooks and vendors them under + `bundle/src/notebooks/` with the task references rewritten to the + relative `../src/notebooks/...` paths. + +Skip this step entirely when `needs_auth` is `false`. + ### Step 7 — Phase 3: Prepare Invoke the `flowx:prepare` skill with: -- Translation report: `/translate/translation_report.json` +- Translation report: `/translate/translation_report.stamped.json` if step 5.5 produced one, otherwise `/translate/translation_report.json` - Output dir: `/dab_output/` - Catalog: user-specified or `main` - Schema: user-specified or `default` diff --git a/skills/prepare/SKILL.md b/skills/prepare/SKILL.md index 66ef4bc..1e6ba9e 100644 --- a/skills/prepare/SKILL.md +++ b/skills/prepare/SKILL.md @@ -52,6 +52,53 @@ Ask the user for the following (provide defaults): | Warehouse ID | SQL warehouse for SQL tasks (optional) | prompt if SQL tasks exist | | Databricks CLI profile | Profile used to download workspace-resident notebooks / JARs / Python files (`--profile`). Required only when the bundle references absolute workspace paths. | resolved from `~/.databrickscfg` (auto-prompt if multiple) | +### Step 2.5 — Detect workspace artifacts and authenticate + +Before running the bundle writer, check whether the report references +absolute workspace paths (notebooks under `/Shared/`, SparkPython +files, SparkJar libraries) or DBFS paths that the bundle should +download to be self-contained: + +```bash +python3 -m flowx.adapter workspace-paths \ + \ + --source-dir +``` + +The command emits: + +```json +{ + "paths": ["/Shared/team/notebook_a", "/Shared/team/notebook_b"], + "suggested_hosts": ["https://adb-1234.5.azuredatabricks.net"], + "needs_auth": true +} +``` + +When `needs_auth` is `true`: + +1. Surface the suggested hosts to the user with `AskUserQuestion`. Use + the first `suggested_hosts` value as the default; allow the user to + override. When no host is suggested (no Databricks linked service + in the export), prompt for the host with no default. +2. Run the interactive Databricks CLI login command and wait for it to + complete: + + ```bash + databricks auth login --host + ``` + + This writes a profile into `~/.databrickscfg`. When the user has + chosen a specific profile name, append `--profile ` to both + the login and the prepare invocation below. + +3. Pass the resolved profile to step 3 via `--profile ` (default + profile name is `DEFAULT`). When `needs_auth` is `false` skip steps + 1–2 and omit `--profile` from step 3. + +The `paths` list is informational; you can echo it to the user so they +know which notebooks the bundle will vendor. + ### Step 3 — Run bundle generation Execute the DAB writer: diff --git a/skills/translate/SKILL.md b/skills/translate/SKILL.md index 283c9b9..5d719e8 100644 --- a/skills/translate/SKILL.md +++ b/skills/translate/SKILL.md @@ -30,6 +30,21 @@ This approach maximizes reliability while covering the long tail of ADF activity Follow these steps in order: +### Step 0 — Gather phase inputs + +Run the adapter inputs subcommand so the agent surfaces the free-text +questions the phase needs (inventory path, ADF source dir, output +directory): + +```bash +python3 -m flowx.adapter inputs translate +``` + +The JSON response carries the prompts and defaults; collect answers +from the user (or fall back to the defaults) and persist them to +`/translate/inputs.json` so later steps and subsequent +phases can read the same values. + ### Step 1 — Locate the inventory Read `inventory.json` from the ingest phase. If the path is not already in conversation context, ask the user: @@ -156,6 +171,96 @@ python3 /src/flowx/translator/engine.py \ This updates `translation_report.json` with the agentic results merged in, changing their status from `pending` to `translated` (or `failed` if the agentic skill could not produce a result). +### Step 6.1 — Gather just-in-time translation preferences + +The adapter raises several preference questions plus a chained set for +metadata-driven motifs. Drive the loop multi-pass: every time the user +answers a question whose value gates further prompts, re-run `inspect +--answers ` to surface the next batch. + +When the metadata-driven flow ends with `metadata_driven_lookup_tool=have` +and the agent has a database tool (Genie, MCP SQL, or a workspace SDK), +run the lookup query directly and write the rows to +`/lookup_values.json`. When the answer is `none`, prompt +the user for a CSV file or comma-separated string and call: + +```bash +python3 -m flowx.adapter materialize-lookup "" \ + --out /lookup_values.json +``` + +Then call `modify` with the lookup values: + +```bash +python3 -m flowx.adapter modify \ + \ + /answers.json \ + --lookup-values /lookup_values.json \ + --out /translation_report.stamped.json +``` + +When no metadata-driven motif is consolidated, `--lookup-values` is +omitted. + +#### Legacy flow details + +Before writing the final report, surface any pipeline-modifier questions the +IR raises (Copy Data paradigm, non-Databricks task compute, Lakeflow Connect +opt-in, Databricks task compute). Use the adapter CLI bridge: + +```bash +python3 -m flowx.adapter inspect +``` + +The command emits JSON: + +```json +{ + "pipelines": [ + { + "pipeline_name": "ETL_Main", + "questions": [ + { + "question_id": "copy_activity_paradigm", + "prompt": "How should Copy Data activities targeting Delta be implemented?", + "rationale": "...", + "options": [{"value": "notebook", "label": "...", "description": "..."}, ...], + "affected_task_keys": ["copy_orders", "copy_customers"], + "default": "notebook" + }, + ... + ] + } + ] +} +``` + +For each question, prompt the user with the rationale, options, and the +task keys it affects. Use the default when the user defers. Collect the +answers into a JSON file (`/answers.json`) shaped like: + +```json +{ + "copy_activity_paradigm": "sdp", + "non_databricks_task_compute": "serverless", + "use_lakeflow_connectors": "lakeflow_connect", + "databricks_task_compute": "existing" +} +``` + +Then apply the answers to produce a stamped report the prepare phase +consumes: + +```bash +python3 -m flowx.adapter modify \ + \ + /answers.json \ + --out /translation_report.stamped.json +``` + +The prepare phase (next skill) must be pointed at the stamped report. +When no questions are raised, the inspect output is `{"pipelines": [{"pipeline_name": "...", "questions": []}, ...]}` — skip the modify step and pass the original report straight through. + ### Step 7 — Present translation summary Display a summary to the user: diff --git a/src/orchestra/adapter/__init__.py b/src/orchestra/adapter/__init__.py new file mode 100644 index 0000000..f713c38 --- /dev/null +++ b/src/orchestra/adapter/__init__.py @@ -0,0 +1,96 @@ +"""Agent-facing surfaces and the matching pipeline modifier for flowx translation. + +This package draws a deliberate line between two roles: + +* **Agent adapter** -- :mod:`flowx.adapter.session` plus the question + shapes in :mod:`flowx.adapter.models`. This is the layer an agent + calls. It converts tool-call arguments into deterministic service calls + and maps "need more input" signals into structured objects (and the + :exc:`TranslationInputRequired` exception) the agent can hand back to + the user. + +* **Pipeline modifier** -- :mod:`flowx.adapter.operations`. The + deterministic transformation that consumes a validated + :class:`TranslationPreferences` snapshot and stamps concrete decisions + onto a Pipeline IR. It has no awareness of agents or user prompts and + is safely importable from non-agent contexts (CLI, tests, batch jobs). + +The package is organised into three primary modules plus the predicates +and session helpers: + +* :mod:`~flowx.adapter.models` -- StrEnums and dataclasses. +* :mod:`~flowx.adapter.operations` -- Free functions + (``gather_questions``, ``apply_preferences``, ``validate_answer``, + ``allowed_values_for``). +* :mod:`~flowx.adapter.constants` -- Question IDs, compute-mode + strings, replacement names, and other shared constants. +* :mod:`~flowx.adapter.predicates` -- Pure IR predicates used by + both ``operations`` and the bundler. +* :mod:`~flowx.adapter.session` -- The agent adapter class. +""" + +from __future__ import annotations + +from flowx.adapter.models import ( + DEFAULT_PREFERENCES, + CopyActivityParadigm, + DatabricksTaskCompute, + LakeflowConnectorType, + MetadataDrivenAccess, + MetadataDrivenConsolidate, + MetadataDrivenLookupTool, + MetadataDrivenSize, + MigrationInputQuestion, + NonDatabricksTaskCompute, + PendingMigrationInputs, + PendingQuestions, + QuestionOption, + TranslationPreferences, + TranslationQuestion, + UseLakeflowConnectors, +) +from flowx.adapter.operations import ( + allowed_values_for, + apply_preferences, + collect_workspace_artifact_paths, + detect_databricks_hosts, + enum_for, + gather_questions, + validate_answer, +) +from flowx.adapter.session import ( + MigrationInputSession, + TranslationInputRequired, + TranslationSession, + UnknownMigrationPhaseError, +) + +__all__ = [ + "DEFAULT_PREFERENCES", + "CopyActivityParadigm", + "DatabricksTaskCompute", + "LakeflowConnectorType", + "MetadataDrivenAccess", + "MetadataDrivenConsolidate", + "MetadataDrivenLookupTool", + "MetadataDrivenSize", + "MigrationInputQuestion", + "MigrationInputSession", + "NonDatabricksTaskCompute", + "PendingMigrationInputs", + "PendingQuestions", + "QuestionOption", + "TranslationInputRequired", + "TranslationPreferences", + "TranslationQuestion", + "TranslationSession", + "UnknownMigrationPhaseError", + "UseLakeflowConnectors", + "allowed_values_for", + "apply_preferences", + "collect_workspace_artifact_paths", + "detect_databricks_hosts", + "enum_for", + "gather_questions", + "validate_answer", +] diff --git a/src/orchestra/adapter/__main__.py b/src/orchestra/adapter/__main__.py new file mode 100644 index 0000000..be5823c --- /dev/null +++ b/src/orchestra/adapter/__main__.py @@ -0,0 +1,594 @@ +"""CLI bridge that lets the flowx skills drive the adapter via subprocesses. + +The skills (`/flowx:translate`, `/flowx:migrate`) cannot keep a +Python session alive across user prompts, so this module exposes two +stateless subcommands: + +* ``inspect`` reads a translation report and emits the pending questions + as JSON for the agent to surface to the user. +* ``modify`` reads the same report plus a JSON file of answers and writes + a preference-stamped report the prepare phase consumes verbatim. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from flowx.adapter.models import ( + DEFAULT_PREFERENCES, + CopyActivityParadigm, + DatabricksTaskCompute, + LakeflowConnectorType, + MetadataDrivenAccess, + MetadataDrivenConsolidate, + MetadataDrivenLookupTool, + MetadataDrivenSize, + NonDatabricksTaskCompute, + PendingQuestions, + TranslationPreferences, + TranslationQuestion, + UseLakeflowConnectors, +) +from flowx.adapter.operations import ( + apply_preferences, + collect_workspace_artifact_paths, + detect_databricks_hosts, + gather_questions, + validate_answer, +) +from flowx.bundler.dab_writer import pipeline_dict_to_ir +from flowx.translator.engine import _pipeline_to_dict + + +def main(argv: list[str] | None = None) -> int: + """Dispatches an ``inspect`` or ``modify`` subcommand. + + Args: + argv: CLI arguments to parse. Defaults to :data:`sys.argv` when + ``None``. + + Returns: + Exit code (0 on success, non-zero on usage or runtime errors). + """ + parser = _build_parser() + args = parser.parse_args(argv) + if args.command == "inspect": + return _run_inspect(args) + if args.command == "modify": + return _run_modify(args) + if args.command == "materialize-lookup": + return _run_materialize_lookup(args) + if args.command == "inputs": + return _run_inputs(args) + if args.command == "workspace-paths": + return _run_workspace_paths(args) + parser.print_help(sys.stderr) + return 2 + + +def _run_workspace_paths(args: argparse.Namespace) -> int: + """Implements the ``workspace-paths`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``report``, ``source_dir``, + and ``out``. + + Returns: + ``0`` on success. The command always succeeds when the report + can be read; missing or unreadable inputs simply produce empty + path / host lists so the skill can detect the no-op case. + """ + paths = collect_workspace_artifact_paths(args.report) + suggested_hosts = detect_databricks_hosts(args.source_dir) if args.source_dir else [] + payload = { + "paths": paths, + "suggested_hosts": suggested_hosts, + "needs_auth": bool(paths), + } + _emit_json(payload, args.out) + return 0 + + +def _run_inputs(args: argparse.Namespace) -> int: + """Implements the ``inputs`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``phase`` and ``out``. + + Returns: + ``0`` on success. The CLI never raises here because the phase + argument is constrained by argparse. + """ + from flowx.adapter.session import MigrationInputSession + + session = MigrationInputSession(phase=args.phase) + pending = session.pending() + payload = { + "phase": pending.phase, + "questions": [ + { + "question_id": question.question_id, + "prompt": question.prompt, + "description": question.description, + "default": question.default, + "required": question.required, + } + for question in pending.questions + ], + } + _emit_json(payload, args.out) + return 0 + + +def _build_parser() -> argparse.ArgumentParser: + """Builds the top-level argparse parser with the two subcommands. + + Returns: + Configured :class:`argparse.ArgumentParser`. + """ + parser = argparse.ArgumentParser( + prog="python -m flowx.adapter", + description="Inspect and modify a translated flowx pipeline IR.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + inspect = subparsers.add_parser( + "inspect", + help="Emit pending translation questions for a report as JSON.", + ) + inspect.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") + inspect.add_argument( + "--answers", + type=Path, + default=None, + help=( + "Optional path to a JSON file of answers already collected; " + "questions whose conditions depend on those answers will surface " + "only when their conditions are met." + ), + ) + inspect.add_argument( + "--out", + type=Path, + default=None, + help="Optional output file; defaults to stdout.", + ) + + modify = subparsers.add_parser( + "modify", + help="Apply collected answers to a translation report and write the stamped IR.", + ) + modify.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") + modify.add_argument("answers", type=Path, help="Path to a JSON file mapping question_id to answer string.") + modify.add_argument( + "--out", + type=Path, + required=True, + help="Destination path for the preference-stamped IR JSON.", + ) + modify.add_argument( + "--lookup-values", + type=Path, + default=None, + help=( + "Optional path to a JSON list of lookup-value rows that consolidated " + "metadata-driven motifs should ingest. Each row is a dict mirroring " + "a row from the source Lookup query." + ), + ) + + workspace_paths = subparsers.add_parser( + "workspace-paths", + help=( + "Detect absolute workspace paths in a stamped report and suggest " + "Databricks workspace hosts from the ADF linked services." + ), + ) + workspace_paths.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") + workspace_paths.add_argument( + "--source-dir", + type=Path, + default=None, + help=( + "Optional path to the ADF JSON export directory. When supplied, " + "the command reads ``linked_services/*.json`` to suggest the " + "workspace host that ``databricks auth login --host`` should use." + ), + ) + workspace_paths.add_argument( + "--out", + type=Path, + default=None, + help="Optional output file; defaults to stdout.", + ) + + inputs = subparsers.add_parser( + "inputs", + help="Emit the migration-phase input questions for an flowx phase as JSON.", + ) + inputs.add_argument( + "phase", + choices=("ingest", "translate", "prepare"), + help="Migration phase whose input prompts the agent should surface.", + ) + inputs.add_argument( + "--out", + type=Path, + default=None, + help="Optional output file; defaults to stdout.", + ) + + materialize = subparsers.add_parser( + "materialize-lookup", + help="Parse CSV-shaped lookup values into the JSON shape modify consumes.", + ) + materialize.add_argument( + "source", + help=( + "Either a path to a CSV file or a literal CSV string. The first row " + "is treated as headers and every subsequent row is emitted as one dict." + ), + ) + materialize.add_argument( + "--out", + type=Path, + required=True, + help="Destination path for the lookup-values JSON list.", + ) + return parser + + +def _run_inspect(args: argparse.Namespace) -> int: + """Implements the ``inspect`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``report``, ``answers``, and + ``out``. + + Returns: + ``0`` when the report was inspected successfully, ``1`` when the + report could not be loaded. + """ + pipelines = _load_pipelines(args.report) + if pipelines is None: + return 1 + answers = _read_answers_optional(args.answers) if getattr(args, "answers", None) else {} + payload = { + "pipelines": [_pending_to_payload(gather_questions(pipeline, [], answers=answers)) for pipeline in pipelines], + } + _emit_json(payload, args.out) + return 0 + + +def _read_answers_optional(answers_path: Path) -> dict[str, str]: + """Loads an answers JSON file supplied to ``inspect``. + + Args: + answers_path: Path to a JSON file mapping question_id to answer. + + Returns: + Mapping of question_id to answer string. Returns an empty dict + when the file is missing or unparseable so ``inspect`` still + succeeds (question gating just sees no prior answers). + """ + try: + raw = json.loads(answers_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return {key: str(value) for key, value in raw.items() if isinstance(value, str)} + + +def _run_modify(args: argparse.Namespace) -> int: + """Implements the ``modify`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``report``, ``answers``, + ``out``, and the optional ``lookup_values``. + + Returns: + ``0`` when the modified IR was written successfully, ``1`` when + the report could not be loaded, ``2`` when the answers failed + validation. + """ + pipelines = _load_pipelines(args.report) + if pipelines is None: + return 1 + try: + answers = _load_answers(args.answers) + preferences = _preferences_from_answers(answers) + except ValueError as error: + print(f"Invalid answers payload: {error}", file=sys.stderr) + return 2 + lookup_values = _load_lookup_values(args.lookup_values) if args.lookup_values else [] + stamped_pipelines = [ + _stamp_lookup_values_into_metadata_driven_motifs(apply_preferences(pipeline, preferences), lookup_values) + for pipeline in pipelines + ] + modified = [_pipeline_to_dict(pipeline) for pipeline in stamped_pipelines] + _write_modified_report(args.report, modified, args.out) + return 0 + + +def _run_materialize_lookup(args: argparse.Namespace) -> int: + """Implements the ``materialize-lookup`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``source`` (file path or + literal CSV string) and ``out``. + + Returns: + ``0`` when the JSON was written successfully, ``2`` when the + source could not be parsed as CSV. + """ + try: + rows = _parse_csv_source(args.source) + except ValueError as error: + print(f"Invalid CSV source: {error}", file=sys.stderr) + return 2 + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _parse_csv_source(source: str) -> list[dict[str, str]]: + """Parses a CSV file path or literal CSV string into a list of row dicts. + + Args: + source: Either a path to a CSV file or a literal CSV string with + a header row. + + Returns: + List of dicts, one per data row, keyed by the header names. + + Raises: + ValueError: When the CSV has no header row or is empty. + """ + import csv + + source_path = Path(source) + text = source_path.read_text(encoding="utf-8") if source_path.exists() else source + reader = csv.DictReader(text.splitlines()) + if reader.fieldnames is None: + raise ValueError("Source CSV is empty or missing a header row") + return [dict(row) for row in reader] + + +def _load_lookup_values(lookup_values_path: Path) -> list[dict[str, Any]]: + """Loads materialised lookup values from a JSON file. + + Args: + lookup_values_path: Path to a JSON list of row dicts. + + Returns: + The parsed list of row dicts. Returns an empty list when the + file is missing or unparseable so the modify pass still succeeds + (consolidated motifs will warn and fall back to the scaffold). + """ + try: + raw = json.loads(lookup_values_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return [] + if not isinstance(raw, list): + return [] + return [row for row in raw if isinstance(row, dict)] + + +def _stamp_lookup_values_into_metadata_driven_motifs(pipeline, lookup_values: list[dict[str, Any]]): + """Stamps lookup values onto every metadata-driven motif marked for consolidation. + + Args: + pipeline: Preference-stamped pipeline IR. + lookup_values: Rows materialised by the agent or the user. + + Returns: + A new :class:`Pipeline` whose metadata-driven motif activities + carry the supplied lookup rows. When *lookup_values* is empty + the pipeline is returned unchanged. + """ + if not lookup_values: + return pipeline + import dataclasses as _dataclasses + + from flowx.models.ir import MotifActivity as _MotifActivity + + stamped_tasks = [] + for task in pipeline.tasks: + if isinstance(task, _MotifActivity) and task.consolidate_metadata_driven: + stamped_tasks.append(_dataclasses.replace(task, lookup_values=list(lookup_values))) + else: + stamped_tasks.append(task) + return _dataclasses.replace(pipeline, tasks=stamped_tasks) + + +def _load_pipelines(report_path: Path) -> list[Any] | None: + """Loads every pipeline IR contained in a report file. + + Args: + report_path: Path to a translation report or pipeline IR JSON. + + Returns: + List of rehydrated :class:`Pipeline` objects, or ``None`` when + the file could not be parsed. + """ + try: + raw = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + print(f"Failed to read {report_path}: {error}", file=sys.stderr) + return None + pipeline_dicts = _extract_pipeline_dicts(raw) + return [pipeline_dict_to_ir(pipeline_dict)[0] for pipeline_dict in pipeline_dicts] + + +def _extract_pipeline_dicts(raw: Any) -> list[dict[str, Any]]: + """Normalises a translation report into a list of pipeline IR dicts. + + Args: + raw: Parsed JSON content from a report file. + + Returns: + List of dicts, each in the shape ``engine._pipeline_to_dict`` + produces. Empty when *raw* does not contain a recognisable + pipeline payload. + """ + if isinstance(raw, dict) and "tasks" in raw and "name" in raw: + return [raw] + if isinstance(raw, dict) and "translations" in raw: + return [ + {"name": entry["pipeline"], **entry["ir"]} + for entry in raw.get("translations", []) + if entry.get("status") == "translated" and entry.get("ir") + ] + return [] + + +def _load_answers(answers_path: Path) -> dict[str, str]: + """Loads a JSON answers file and validates its top-level shape. + + Args: + answers_path: Path to a JSON file mapping question_id to answer. + + Returns: + Mapping of question_id to answer string. + + Raises: + ValueError: When the file is not a JSON object of string values. + """ + raw = json.loads(answers_path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError(f"Expected a JSON object at {answers_path}; got {type(raw).__name__}") + coerced: dict[str, str] = {} + for key, value in raw.items(): + if not isinstance(value, str): + raise ValueError(f"Answer for {key!r} must be a string; got {type(value).__name__}") + coerced[key] = value + return coerced + + +def _preferences_from_answers(answers: dict[str, str]) -> TranslationPreferences: + """Builds a :class:`TranslationPreferences` from a validated answers dict. + + Args: + answers: Validated mapping of question_id to answer string. + + Returns: + Preferences with every answered field overridden and every + unanswered field defaulted. + + Raises: + ValueError: When an answer is not in the allowed set for its + question. + """ + validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} + return TranslationPreferences( + copy_activity_paradigm=CopyActivityParadigm( + validated.get("copy_activity_paradigm", DEFAULT_PREFERENCES.copy_activity_paradigm) + ), + non_databricks_task_compute=NonDatabricksTaskCompute( + validated.get("non_databricks_task_compute", DEFAULT_PREFERENCES.non_databricks_task_compute) + ), + use_lakeflow_connectors=UseLakeflowConnectors( + validated.get("use_lakeflow_connectors", DEFAULT_PREFERENCES.use_lakeflow_connectors) + ), + databricks_task_compute=DatabricksTaskCompute( + validated.get("databricks_task_compute", DEFAULT_PREFERENCES.databricks_task_compute) + ), + lakeflow_connector_type=LakeflowConnectorType( + validated.get("lakeflow_connector_type", DEFAULT_PREFERENCES.lakeflow_connector_type) + ), + metadata_driven_consolidate=MetadataDrivenConsolidate( + validated.get("metadata_driven_consolidate", DEFAULT_PREFERENCES.metadata_driven_consolidate) + ), + metadata_driven_access=MetadataDrivenAccess( + validated.get("metadata_driven_access", DEFAULT_PREFERENCES.metadata_driven_access) + ), + metadata_driven_size=MetadataDrivenSize( + validated.get("metadata_driven_size", DEFAULT_PREFERENCES.metadata_driven_size) + ), + metadata_driven_lookup_tool=MetadataDrivenLookupTool( + validated.get("metadata_driven_lookup_tool", DEFAULT_PREFERENCES.metadata_driven_lookup_tool) + ), + ) + + +def _pending_to_payload(pending: PendingQuestions) -> dict[str, Any]: + """Serialises pending questions for transmission over stdout. + + Args: + pending: Outstanding questions for a single pipeline. + + Returns: + JSON-friendly dict the agent can iterate over to prompt the user. + """ + return { + "pipeline_name": pending.pipeline_name, + "questions": [_question_to_payload(question) for question in pending.questions], + } + + +def _question_to_payload(question: TranslationQuestion) -> dict[str, Any]: + """Serialises a single :class:`TranslationQuestion` to a JSON-friendly dict. + + Args: + question: Question to serialise. + + Returns: + Dict containing the question's fields with options flattened to + plain dicts. + """ + return { + "question_id": question.question_id, + "prompt": question.prompt, + "rationale": question.rationale, + "options": [asdict(option) for option in question.options], + "affected_task_keys": list(question.affected_task_keys), + "default": question.default, + } + + +def _emit_json(payload: dict[str, Any], out: Path | None) -> None: + """Writes a JSON payload to a file or to stdout. + + Args: + payload: JSON-serialisable mapping to emit. + out: Destination path; ``None`` selects stdout. + """ + encoded = json.dumps(payload, indent=2, default=str) + if out is None: + print(encoded) + return + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(encoded + "\n", encoding="utf-8") + + +def _write_modified_report(report_path: Path, pipelines: list[dict[str, Any]], out: Path) -> None: + """Writes the preference-stamped IR to *out* using the input report's shape. + + Args: + report_path: Path the modified report was sourced from. Used + only to detect whether the input was a single pipeline IR + or an aggregated translation report. + pipelines: Stamped pipeline IR dicts to write. + out: Destination path for the modified report. + """ + raw = json.loads(report_path.read_text(encoding="utf-8")) + if isinstance(raw, dict) and "translations" in raw: + by_name = {pipeline["name"]: pipeline for pipeline in pipelines} + for entry in raw.get("translations", []): + stamped = by_name.get(entry.get("pipeline")) + if stamped is not None and entry.get("ir") is not None: + entry["ir"] = {key: value for key, value in stamped.items() if key != "name"} + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(raw, indent=2, default=str) + "\n", encoding="utf-8") + return + payload = pipelines[0] if len(pipelines) == 1 else {"pipelines": pipelines} + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/orchestra/adapter/constants.py b/src/orchestra/adapter/constants.py new file mode 100644 index 0000000..c74a434 --- /dev/null +++ b/src/orchestra/adapter/constants.py @@ -0,0 +1,74 @@ +"""String constants shared across the adapter and its bundler consumers. + +Every adapter-side string the modifier stamps onto an IR field or that +the bundler reads back from one is defined here. Modules in +``flowx.adapter``, ``flowx.bundler``, and the test suite import +from this module to avoid string-literal drift between the producer and +consumer ends of the same value. +""" + +from __future__ import annotations + +from typing import Final + +QUESTION_COPY_ACTIVITY_PARADIGM: Final[str] = "copy_activity_paradigm" +QUESTION_NON_DATABRICKS_TASK_COMPUTE: Final[str] = "non_databricks_task_compute" +QUESTION_USE_LAKEFLOW_CONNECTORS: Final[str] = "use_lakeflow_connectors" +QUESTION_DATABRICKS_TASK_COMPUTE: Final[str] = "databricks_task_compute" +QUESTION_LAKEFLOW_CONNECTOR_TYPE: Final[str] = "lakeflow_connector_type" +QUESTION_METADATA_DRIVEN_CONSOLIDATE: Final[str] = "metadata_driven_consolidate" +QUESTION_METADATA_DRIVEN_ACCESS: Final[str] = "metadata_driven_access" +QUESTION_METADATA_DRIVEN_SIZE: Final[str] = "metadata_driven_size" +QUESTION_METADATA_DRIVEN_LOOKUP_TOOL: Final[str] = "metadata_driven_lookup_tool" + +METADATA_DRIVEN_MOTIF_ID: Final[str] = "metadata_driven_bulk_copy" + +PHASE_INGEST: Final[str] = "ingest" +PHASE_TRANSLATE: Final[str] = "translate" +PHASE_PREPARE: Final[str] = "prepare" + +INPUT_ADF_SOURCE_PATH: Final[str] = "adf_source_path" +INPUT_ADF_RESOURCE_URL: Final[str] = "adf_resource_url" +INPUT_OUTPUT_DIR: Final[str] = "output_dir" +INPUT_INVENTORY_PATH: Final[str] = "inventory_path" +INPUT_TRANSLATION_REPORT_PATH: Final[str] = "translation_report_path" +INPUT_OUTPUT_BUNDLE_PATH: Final[str] = "output_bundle_path" +INPUT_CATALOG: Final[str] = "catalog" +INPUT_SCHEMA: Final[str] = "schema" +INPUT_BUNDLE_NAME: Final[str] = "bundle_name" +INPUT_DATABRICKS_PROFILE: Final[str] = "databricks_profile" + +LAKEFLOW_CONNECTOR_TYPE_QUERY_BASED: Final[str] = "query_based" +LAKEFLOW_CONNECTOR_TYPE_CDC: Final[str] = "cdc" + +COPY_SOURCE_QUERY_KEYS: Final[tuple[str, ...]] = ("query", "sqlReaderQuery", "sql_query") + +COMPUTE_MODE_SERVERLESS: Final[str] = "serverless" +COMPUTE_MODE_CLASSIC_SINGLE_NODE: Final[str] = "classic_single_node" +COMPUTE_MODE_CLASSIC_MULTI_NODE: Final[str] = "classic_multi_node" +COMPUTE_MODE_INHERIT: Final[str] = "inherit" + +LAKEFLOW_CONNECT_REPLACEMENT: Final[str] = "lakeflow_connect_database" + +DATABASE_SOURCE_TOKENS: Final[tuple[str, ...]] = ( + "sqlserver", + "azuresql", + "mysql", + "azuremysql", + "postgre", + "azurepostgre", +) + +DELTA_SINK_TOKENS: Final[tuple[str, ...]] = ("delta", "deltalake") + +LAKEFLOW_CONNECT_MOTIF_REPLACEMENTS: Final[frozenset[str]] = frozenset( + { + "auto_loader", + "auto_loader_file_notification", + "dlt_apply_changes", + "for_each_ingestion", + "spark_delta_write", + } +) + +DATABASE_SOURCE_TYPE_HINT: Final[str] = "database" diff --git a/src/orchestra/adapter/models.py b/src/orchestra/adapter/models.py new file mode 100644 index 0000000..3aff855 --- /dev/null +++ b/src/orchestra/adapter/models.py @@ -0,0 +1,284 @@ +"""Dataclasses and StrEnums shared across the adapter package.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import StrEnum +from types import MappingProxyType +from typing import Final + + +class CopyActivityParadigm(StrEnum): + """Code paradigm used to translate Copy Data activities whose sink is Delta.""" + + NOTEBOOK = "notebook" + SDP = "sdp" + + +class NonDatabricksTaskCompute(StrEnum): + """Compute mode used for non-Databricks tasks such as Copy, Web, or Lookup.""" + + SERVERLESS = "serverless" + CLASSIC = "classic" + + +class UseLakeflowConnectors(StrEnum): + """Whether to swap eligible database-source Copy patterns for Lakeflow Connect.""" + + LAKEFLOW_CONNECT = "lakeflow_connect" + EXISTING = "existing" + + +class DatabricksTaskCompute(StrEnum): + """Compute mode used for ADF DatabricksNotebook and DatabricksSparkPython tasks.""" + + SERVERLESS = "serverless" + EXISTING = "existing" + + +class LakeflowConnectorType(StrEnum): + """Lakeflow Connect connector flavour for an eligible Copy ingestion. + + Used only when ``use_lakeflow_connectors`` is ``lakeflow_connect``. The + modifier still routes Copy activities that read from a SQL query into + the query-based connector regardless of this preference; this enum + controls the default for table-based Copy activities. + """ + + QUERY_BASED = "query_based" + CDC = "cdc" + + +class MetadataDrivenConsolidate(StrEnum): + """Whether metadata-driven motifs should collapse into one managed pipeline.""" + + CONSOLIDATE = "consolidate" + KEEP = "keep" + + +class MetadataDrivenAccess(StrEnum): + """Whether the user can query the metadata source for lookup values.""" + + YES = "yes" + NO = "no" + + +class MetadataDrivenSize(StrEnum): + """T-shirt size for the number of metadata-driven configuration rows. + + The thresholds match the prompt rationale: ``small`` covers fewer than + 50 entries, ``medium`` covers fewer than 250, and ``large`` covers + 250 or more. ``large`` suppresses inline lookup materialisation + because the modifier cannot reliably enumerate the configuration in + one translation pass. + """ + + SMALL = "small" + MEDIUM = "medium" + LARGE = "large" + + +class MetadataDrivenLookupTool(StrEnum): + """Whether the agent has a tool that can run the lookup query.""" + + HAVE = "have" + NONE = "none" + + +FIELD_TO_ENUM: Final[MappingProxyType[str, type[StrEnum]]] = MappingProxyType( + { + "copy_activity_paradigm": CopyActivityParadigm, + "non_databricks_task_compute": NonDatabricksTaskCompute, + "use_lakeflow_connectors": UseLakeflowConnectors, + "databricks_task_compute": DatabricksTaskCompute, + "lakeflow_connector_type": LakeflowConnectorType, + "metadata_driven_consolidate": MetadataDrivenConsolidate, + "metadata_driven_access": MetadataDrivenAccess, + "metadata_driven_size": MetadataDrivenSize, + "metadata_driven_lookup_tool": MetadataDrivenLookupTool, + } +) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TranslationPreferences: + """Snapshot of user choices that shape downstream IR transformations. + + Each field accepts either a raw string or the corresponding enum + member; strings are coerced to enum members at construction. + + Attributes: + copy_activity_paradigm: Paradigm used for Copy Data activities whose + sink resolves to a Delta table. + non_databricks_task_compute: Compute mode for non-Databricks tasks. + use_lakeflow_connectors: Whether eligible database-source Copy + patterns are migrated to managed Lakeflow Connect pipelines. + databricks_task_compute: Compute mode for ADF DatabricksNotebook and + DatabricksSparkPython tasks. + per_task: Optional per-activity overrides keyed by task_key. Each + value is a partial mapping of the four fields above; only the + keys present win over the pipeline-wide defaults. + """ + + copy_activity_paradigm: CopyActivityParadigm = CopyActivityParadigm.NOTEBOOK + non_databricks_task_compute: NonDatabricksTaskCompute = NonDatabricksTaskCompute.SERVERLESS + use_lakeflow_connectors: UseLakeflowConnectors = UseLakeflowConnectors.EXISTING + databricks_task_compute: DatabricksTaskCompute = DatabricksTaskCompute.EXISTING + lakeflow_connector_type: LakeflowConnectorType = LakeflowConnectorType.CDC + metadata_driven_consolidate: MetadataDrivenConsolidate = MetadataDrivenConsolidate.KEEP + metadata_driven_access: MetadataDrivenAccess = MetadataDrivenAccess.NO + metadata_driven_size: MetadataDrivenSize = MetadataDrivenSize.LARGE + metadata_driven_lookup_tool: MetadataDrivenLookupTool = MetadataDrivenLookupTool.NONE + per_task: dict[str, dict[str, str]] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Coerces raw string inputs into their backing enum members. + + Raises: + ValueError: When a field value is not a member of the backing + :class:`StrEnum`. + """ + for field_name, enum_cls in FIELD_TO_ENUM.items(): + value = getattr(self, field_name) + if not isinstance(value, enum_cls): + object.__setattr__(self, field_name, enum_cls(value)) + + def effective_for(self, task_key: str) -> TranslationPreferences: + """Returns a preferences view where per-task overrides for *task_key* win. + + Args: + task_key: Sanitised task key of the activity being prepared. + + Returns: + A new :class:`TranslationPreferences` with overrides for + *task_key* applied on top of the pipeline-wide values, or + ``self`` unchanged when no overrides exist for *task_key*. + """ + override = self.per_task.get(task_key) + if not override: + return self + return TranslationPreferences( + copy_activity_paradigm=CopyActivityParadigm( + override.get("copy_activity_paradigm", self.copy_activity_paradigm) + ), + non_databricks_task_compute=NonDatabricksTaskCompute( + override.get("non_databricks_task_compute", self.non_databricks_task_compute) + ), + use_lakeflow_connectors=UseLakeflowConnectors( + override.get("use_lakeflow_connectors", self.use_lakeflow_connectors) + ), + databricks_task_compute=DatabricksTaskCompute( + override.get("databricks_task_compute", self.databricks_task_compute) + ), + lakeflow_connector_type=LakeflowConnectorType( + override.get("lakeflow_connector_type", self.lakeflow_connector_type) + ), + metadata_driven_consolidate=MetadataDrivenConsolidate( + override.get("metadata_driven_consolidate", self.metadata_driven_consolidate) + ), + metadata_driven_access=MetadataDrivenAccess( + override.get("metadata_driven_access", self.metadata_driven_access) + ), + metadata_driven_size=MetadataDrivenSize(override.get("metadata_driven_size", self.metadata_driven_size)), + metadata_driven_lookup_tool=MetadataDrivenLookupTool( + override.get("metadata_driven_lookup_tool", self.metadata_driven_lookup_tool) + ), + per_task=self.per_task, + ) + + +DEFAULT_PREFERENCES: Final[TranslationPreferences] = TranslationPreferences() + + +@dataclass(frozen=True, slots=True, kw_only=True) +class QuestionOption: + """One allowed answer to a :class:`TranslationQuestion`. + + Attributes: + value: Machine-readable identifier matching the backing enum member. + label: Short human-readable label suitable for a prompt button. + description: One-sentence explanation of the trade-off this option + implies for the migrated bundle. + """ + + value: str + label: str + description: str + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TranslationQuestion: + """A single just-in-time question raised by the IR inspector. + + Attributes: + question_id: Stable identifier matching the preferences field. + prompt: Human-readable question text. + rationale: One- or two-sentence explanation of why the question + is being raised. + options: Allowed answers; the first option is the conservative + default and is also exposed via ``default``. + affected_task_keys: Activity task keys impacted by the answer. + default: Default value applied when the caller skips the question. + conditions: Tuples of ``(question_id, expected_value)`` that must + already be answered with the expected value before this + question surfaces. An empty tuple means the question is + evaluated solely on its IR/motif preconditions. + """ + + question_id: str + prompt: str + rationale: str + options: tuple[QuestionOption, ...] + affected_task_keys: tuple[str, ...] + default: str + conditions: tuple[tuple[str, str], ...] = () + + +@dataclass(slots=True, kw_only=True) +class PendingQuestions: + """Outstanding questions for a single pipeline translation. + + Attributes: + pipeline_name: Name of the pipeline these questions belong to. + questions: Ordered list of questions still awaiting an answer. + """ + + pipeline_name: str + questions: list[TranslationQuestion] = field(default_factory=list) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class MigrationInputQuestion: + """A free-text input gathered before an flowx phase runs. + + Attributes: + question_id: Stable identifier the skill uses to key the answer. + prompt: Human-readable question text. + description: One-sentence explanation of what the value is used + for and what shape is expected (path, URL, identifier). + default: Default value applied when the caller skips the + question; ``None`` when the field is required and has no + sensible default. + required: When ``True`` the skill must collect a value; when + ``False`` the default (which may be ``None``) is permitted. + """ + + question_id: str + prompt: str + description: str + default: str | None = None + required: bool = True + + +@dataclass(slots=True, kw_only=True) +class PendingMigrationInputs: + """Outstanding migration-phase input questions for a single phase. + + Attributes: + phase: The migration phase name (``"ingest"``, ``"translate"``, + ``"prepare"``). + questions: Ordered list of questions still awaiting an answer. + """ + + phase: str + questions: list[MigrationInputQuestion] = field(default_factory=list) diff --git a/src/orchestra/adapter/operations.py b/src/orchestra/adapter/operations.py new file mode 100644 index 0000000..cfc526c --- /dev/null +++ b/src/orchestra/adapter/operations.py @@ -0,0 +1,1125 @@ +"""Standalone operations: question gathering, validation, and IR modification. + +The agent adapter and the CLI bridge call into these functions; nothing +here is stateful. Preference dataclasses, StrEnums, and question shapes +live in :mod:`flowx.adapter.models`. +""" + +from __future__ import annotations + +import dataclasses +import json +from enum import StrEnum +from pathlib import Path +from typing import Any + +from flowx.adapter.constants import ( + COMPUTE_MODE_CLASSIC_MULTI_NODE, + COMPUTE_MODE_CLASSIC_SINGLE_NODE, + COMPUTE_MODE_INHERIT, + COMPUTE_MODE_SERVERLESS, + DATABASE_SOURCE_TYPE_HINT, + LAKEFLOW_CONNECT_MOTIF_REPLACEMENTS, + LAKEFLOW_CONNECT_REPLACEMENT, + LAKEFLOW_CONNECTOR_TYPE_QUERY_BASED, + METADATA_DRIVEN_MOTIF_ID, + QUESTION_COPY_ACTIVITY_PARADIGM, + QUESTION_DATABRICKS_TASK_COMPUTE, + QUESTION_METADATA_DRIVEN_ACCESS, + QUESTION_METADATA_DRIVEN_CONSOLIDATE, + QUESTION_METADATA_DRIVEN_LOOKUP_TOOL, + QUESTION_METADATA_DRIVEN_SIZE, + QUESTION_NON_DATABRICKS_TASK_COMPUTE, + QUESTION_USE_LAKEFLOW_CONNECTORS, +) +from flowx.adapter.models import ( + FIELD_TO_ENUM, + CopyActivityParadigm, + DatabricksTaskCompute, + LakeflowConnectorType, + MetadataDrivenAccess, + MetadataDrivenConsolidate, + MetadataDrivenLookupTool, + MetadataDrivenSize, + NonDatabricksTaskCompute, + PendingQuestions, + QuestionOption, + TranslationPreferences, + TranslationQuestion, + UseLakeflowConnectors, +) +from flowx.adapter.predicates import ( + copy_eligible_for_any_lfc_connector, + copy_eligible_for_lfc_query_based, + copy_query_unfit_for_lfc, + copy_targets_delta, + is_non_databricks_task, + walk_activities, +) +from flowx.models.ir import ( + Activity, + CopyActivity, + ForEachActivity, + IfConditionActivity, + MotifActivity, + NotebookActivity, + Pipeline, + SparkPythonActivity, + SwitchActivity, + SwitchCase, +) +from flowx.models.motifs import MOTIF_LAKEFLOW_CONNECT_DATABASE + + +def enum_for(question_id: str) -> type[StrEnum] | None: + """Returns the enum class backing a preference field. + + Args: + question_id: Field name (e.g. ``"copy_activity_paradigm"``). + + Returns: + The :class:`StrEnum` subclass that defines the allowed values, or + ``None`` when the field is unknown. + """ + return FIELD_TO_ENUM.get(question_id) + + +def allowed_values_for(question_id: str) -> tuple[str, ...]: + """Returns the allowed string values for a preference field. + + Args: + question_id: Field name (e.g. ``"copy_activity_paradigm"``). + + Returns: + Tuple of allowed string values in declaration order. Empty when + the field is unknown. + """ + enum_cls = enum_for(question_id) + return tuple(member.value for member in enum_cls) if enum_cls else () + + +def validate_answer(question_id: str, value: str) -> str: + """Returns *value* when it is an allowed answer for *question_id*. + + Args: + question_id: Stable question identifier. + value: Caller-supplied answer string. + + Returns: + The validated value, unchanged. + + Raises: + ValueError: When *question_id* is not known or *value* is not in + the allowed set for the question. + """ + allowed = allowed_values_for(question_id) + if not allowed: + raise ValueError(f"Unknown question_id {question_id!r}") + if value not in allowed: + raise ValueError(f"Invalid answer {value!r} for {question_id!r}; allowed: {sorted(allowed)}") + return value + + +def collect_workspace_artifact_paths(report_path: Path) -> list[str]: + """Returns absolute workspace and DBFS paths referenced by a translation report. + + Args: + report_path: Path to a translation report (single pipeline IR or + an aggregated translation report). + + Returns: + List of paths the bundle would need to download to be + self-contained: notebook paths starting with ``/``, SparkPython + files under DBFS or absolute workspace paths, and SparkJar + library JARs under DBFS or absolute workspace paths. Returns + an empty list when the report cannot be read or contains no + such paths. + """ + try: + with open(report_path, encoding="utf-8") as report_file: + report = json.load(report_file) + except (OSError, json.JSONDecodeError): + return [] + candidates: list[str] = [] + if "tasks" in report: + _walk_workspace_paths(report.get("tasks"), candidates) + for translation in report.get("translations") or []: + ir = translation.get("ir") or {} + _walk_workspace_paths(ir.get("tasks"), candidates) + return candidates + + +def detect_databricks_hosts(source_dir: Path) -> list[str]: + """Returns unique Databricks workspace hosts referenced by an ADF export. + + Args: + source_dir: Root directory of the ADF JSON export. Expected to + contain a ``linked_services/`` subdirectory. + + Returns: + Sorted, unique list of workspace hosts pulled from every + ``AzureDatabricks``-typed linked service whose ``domain`` field + is populated. Returns an empty list when no such linked + services are present or the directory does not exist. + """ + linked_services_dir = source_dir / "linked_services" + if not linked_services_dir.exists(): + return [] + hosts: set[str] = set() + for path in sorted(linked_services_dir.glob("*.json")): + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + properties = data.get("properties") if isinstance(data.get("properties"), dict) else data + if not isinstance(properties, dict): + continue + if properties.get("type") not in {"AzureDatabricks", "Databricks"}: + continue + domain = properties.get("domain") or properties.get("workspaceUrl") + if isinstance(domain, str) and domain.strip(): + hosts.add(domain.strip().rstrip("/")) + return sorted(hosts) + + +def _walk_workspace_paths(tasks: list[dict[str, Any]] | None, candidates: list[str]) -> None: + """Appends workspace-resident artifact paths from *tasks* into *candidates*. + + Args: + tasks: List of task dicts (top-level or nested in control flow). + candidates: List that the caller mutates with discovered paths. + """ + for task in tasks or []: + task_type = task.get("type") + if task_type == "NotebookActivity": + path = task.get("notebook_path") or "" + if isinstance(path, str) and path.startswith("/") and not path.startswith("../"): + candidates.append(path) + elif task_type == "SparkPythonActivity": + path = task.get("python_file") or "" + if isinstance(path, str) and (path.startswith("dbfs:") or path.startswith("/")): + candidates.append(path) + elif task_type == "SparkJarActivity": + for lib in task.get("libraries") or []: + jar = lib.get("jar") if isinstance(lib, dict) else None + if isinstance(jar, str) and (jar.startswith("dbfs:") or jar.startswith("/")): + candidates.append(jar) + _walk_workspace_paths(task.get("inner_activities"), candidates) + _walk_workspace_paths(task.get("if_true_activities"), candidates) + _walk_workspace_paths(task.get("if_false_activities"), candidates) + for case in task.get("cases") or []: + _walk_workspace_paths(case.get("activities"), candidates) + _walk_workspace_paths(task.get("default_activities"), candidates) + + +def gather_questions( + pipeline: Pipeline, + motifs: list | None = None, + *, + answers: dict[str, str] | None = None, +) -> PendingQuestions: + """Walks the IR and returns the questions that apply to *pipeline*. + + Args: + pipeline: Translated pipeline IR after motif collapsing. + motifs: Detected motifs, used to surface the Lakeflow Connect + question for multi-step database ingestion patterns. + answers: Answers the caller has already collected. Questions + whose ``question_id`` is in this mapping are filtered out, + and questions whose ``conditions`` reference earlier answers + are evaluated against this mapping. + + Returns: + A :class:`PendingQuestions` instance carrying the questions whose + IR preconditions and answer-dependent conditions are met but + whose ``question_id`` has not yet been answered. + """ + motif_list = motifs or [] + answer_map = answers or {} + builders = ( + _build_use_lakeflow_connectors_question, + _build_lakeflow_connector_type_question, + _build_copy_activity_paradigm_question, + _build_non_databricks_task_compute_question, + _build_databricks_task_compute_question, + _build_metadata_driven_consolidate_question, + _build_metadata_driven_access_question, + _build_metadata_driven_size_question, + _build_metadata_driven_lookup_tool_question, + ) + candidates = (builder(pipeline, motif_list, answers=answer_map) for builder in builders) + pending = [ + question + for question in candidates + if question is not None + and question.question_id not in answer_map + and _conditions_met(question.conditions, answer_map) + ] + return PendingQuestions(pipeline_name=pipeline.name, questions=pending) + + +def _conditions_met(conditions: tuple[tuple[str, str], ...], answers: dict[str, str]) -> bool: + """Returns True when every condition is satisfied by *answers*. + + Args: + conditions: Tuples of ``(question_id, expected_value)`` from a + :class:`TranslationQuestion`. + answers: Mapping of question_id to the caller-supplied answer. + + Returns: + ``True`` when every condition's question has been answered with + the expected value (or when *conditions* is empty); ``False`` + otherwise. + """ + return all(answers.get(qid) == expected for qid, expected in conditions) + + +def apply_preferences(pipeline: Pipeline, pipeline_preferences: TranslationPreferences) -> Pipeline: + """Returns a copy of *pipeline* with preferences stamped onto each activity. + + Args: + pipeline: Translated pipeline IR after motif collapsing. + pipeline_preferences: Validated pipeline-wide preferences. + + Returns: + A new :class:`Pipeline` whose activities carry concrete decisions + about compute, target format, and Lakeflow Connect replacement. + The input pipeline is not mutated. + """ + stamped_tasks = [_stamp_activity(activity, pipeline_preferences) for activity in pipeline.tasks] + return dataclasses.replace( + pipeline, + tasks=stamped_tasks, + translation_preferences=pipeline_preferences, + ) + + +def _build_copy_activity_paradigm_question( + pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None +) -> TranslationQuestion | None: + """Builds the SDP-vs-notebook question for Copy activities targeting Delta. + + Args: + pipeline: Translated pipeline IR. + motifs: Detected motifs (unused; accepted for builder uniformity). + answers: Answers already supplied for prior prompts. When the + user opted into Lakeflow Connect, this question only fires + for Copy activities that are *not* LFC-eligible -- the + paradigm choice is moot for Copies that will become + managed LFC pipelines. + + Returns: + The constructed :class:`TranslationQuestion`, or ``None`` when no + Copy activity needs a paradigm choice. Copies whose source + query is unfit for both LFC and SDP (joins, aggregates, etc.) + are forced to PySpark notebook and excluded from the affected + set. + """ + answers = answers or {} + going_to_lfc = answers.get(QUESTION_USE_LAKEFLOW_CONNECTORS) == UseLakeflowConnectors.LAKEFLOW_CONNECT.value + affected = tuple( + activity.task_key + for activity in walk_activities(pipeline.tasks) + if isinstance(activity, CopyActivity) + and copy_targets_delta(activity) + and not _copy_paradigm_decided_by_lfc(activity, going_to_lfc) + ) + if not affected: + return None + return TranslationQuestion( + question_id=QUESTION_COPY_ACTIVITY_PARADIGM, + prompt="How should Copy Data activities targeting Delta be implemented?", + rationale=( + "One or more Copy Data activities write to a Delta table. " + "Lakeflow Spark Declarative Pipelines define tables declaratively; " + "a PySpark notebook stays closer to the original ADF activity shape." + ), + options=( + QuestionOption( + value=CopyActivityParadigm.NOTEBOOK.value, + label="PySpark notebook", + description="Generates a notebook task that reads the source and writes Delta directly.", + ), + QuestionOption( + value=CopyActivityParadigm.SDP.value, + label="Lakeflow Spark Declarative Pipeline", + description="Emits an SDP pipeline resource with declarative table definitions.", + ), + ), + affected_task_keys=affected, + default=CopyActivityParadigm.NOTEBOOK.value, + ) + + +def _build_non_databricks_task_compute_question( + pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None +) -> TranslationQuestion | None: + """Builds the serverless-vs-classic question for non-Databricks tasks. + + Args: + pipeline: Translated pipeline IR. + motifs: Detected motifs (unused; accepted for builder uniformity). + answers: Answers already supplied for prior prompts. Copies + that will become managed LFC pipelines are excluded + because LFC pipelines always use serverless compute. + + Returns: + The constructed :class:`TranslationQuestion`, or ``None`` when + every non-Databricks task in the pipeline is going to LFC (no + compute choice to make). + """ + answers = answers or {} + going_to_lfc = answers.get(QUESTION_USE_LAKEFLOW_CONNECTORS) == UseLakeflowConnectors.LAKEFLOW_CONNECT.value + affected = tuple( + activity.task_key + for activity in walk_activities(pipeline.tasks) + if is_non_databricks_task(activity) and not _task_compute_decided_by_lfc(activity, going_to_lfc) + ) + if not affected: + return None + return TranslationQuestion( + question_id=QUESTION_NON_DATABRICKS_TASK_COMPUTE, + prompt="What compute should the non-Databricks tasks use?", + rationale=( + "Tasks such as Copy Data, Web, Lookup, and Wait can run on serverless " + "or classic compute. Classic provisions a single-node cluster for most " + "tasks and a larger fixed-size cluster for Copy Data." + ), + options=( + QuestionOption( + value=NonDatabricksTaskCompute.SERVERLESS.value, + label="Serverless", + description="Runs every non-Databricks task on serverless compute.", + ), + QuestionOption( + value=NonDatabricksTaskCompute.CLASSIC.value, + label="Classic job_cluster", + description="Provisions classic job_clusters sized per task type.", + ), + ), + affected_task_keys=affected, + default=NonDatabricksTaskCompute.SERVERLESS.value, + ) + + +def _build_use_lakeflow_connectors_question( + pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None +) -> TranslationQuestion | None: + """Builds the Lakeflow Connect question for eligible database ingestions. + + Args: + pipeline: Translated pipeline IR. + motifs: Detected motifs, scanned for database-source ingestion patterns. + + Returns: + The constructed :class:`TranslationQuestion`, or ``None`` when no + Copy activity or motif qualifies for Lakeflow Connect. + """ + affected = _affected_task_keys_for_lakeflow_connect(pipeline, motifs) + if not affected: + return None + return TranslationQuestion( + question_id=QUESTION_USE_LAKEFLOW_CONNECTORS, + prompt="Migrate eligible SQL Server, MySQL, and PostgreSQL ingestions to Lakeflow Connect?", + rationale=( + "One or more Copy Data activities ingest from SQL Server, MySQL, or " + "PostgreSQL into Delta. Managed Lakeflow Connect replaces the bespoke " + "ingestion with a declarative pipeline; the existing translation keeps " + "the ADF-shaped activity intact." + ), + options=( + QuestionOption( + value=UseLakeflowConnectors.EXISTING.value, + label="Keep existing translation", + description="Preserves the Copy Data activity as a notebook or SDP task.", + ), + QuestionOption( + value=UseLakeflowConnectors.LAKEFLOW_CONNECT.value, + label="Use Lakeflow Connect", + description="Replaces eligible ingestions with a managed Lakeflow Connect pipeline.", + ), + ), + affected_task_keys=affected, + default=UseLakeflowConnectors.EXISTING.value, + ) + + +def _build_lakeflow_connector_type_question( + pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None +) -> TranslationQuestion | None: + """Builds the CDC-vs-query connector question, suppressed when not actionable. + + Args: + pipeline: Translated pipeline IR. + motifs: Detected motifs (unused; accepted for builder uniformity). + answers: Answers already supplied for prior prompts (unused). + + Returns: + Currently always ``None``. The per-Copy LFC eligibility rules + ensure each Copy can only be routed through one connector type + (table-based reads → CDC because the query-based connector + requires a cursor column; queries with a cursor → query-based + because CDC requires direct table access). A pipeline-wide + preference between CDC and query-based therefore has no + actionable effect; the modifier picks the eligible connector + per Copy. + """ + del pipeline, motifs, answers + return None + + +def _build_databricks_task_compute_question( + pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None +) -> TranslationQuestion | None: + """Builds the serverless-vs-existing question for ADF Databricks-* tasks. + + Args: + pipeline: Translated pipeline IR. + motifs: Detected motifs (unused; accepted for builder uniformity). + + Returns: + The constructed :class:`TranslationQuestion`, or ``None`` when + no Databricks notebook or Python task is present. + """ + affected = tuple( + activity.task_key + for activity in walk_activities(pipeline.tasks) + if isinstance(activity, (NotebookActivity, SparkPythonActivity)) + ) + if not affected: + return None + return TranslationQuestion( + question_id=QUESTION_DATABRICKS_TASK_COMPUTE, + prompt="Migrate existing Databricks notebook and Python tasks to serverless?", + rationale=( + "ADF DatabricksNotebook and DatabricksSparkPython tasks bind to a " + "classic cluster derived from the source linked service. Serverless " + "drops that binding; keeping the existing compute preserves init " + "scripts or DBR-specific features." + ), + options=( + QuestionOption( + value=DatabricksTaskCompute.EXISTING.value, + label="Keep linked-service compute", + description="Binds the task to the cluster derived from the ADF linked service.", + ), + QuestionOption( + value=DatabricksTaskCompute.SERVERLESS.value, + label="Serverless", + description="Removes the cluster binding so the task runs on serverless compute.", + ), + ), + affected_task_keys=affected, + default=DatabricksTaskCompute.EXISTING.value, + ) + + +def _build_metadata_driven_consolidate_question( + pipeline: Pipeline, + motifs: list, + answers: dict[str, str] | None = None, +) -> TranslationQuestion | None: + """Builds the consolidate-or-keep question for metadata-driven motifs. + + Args: + pipeline: Translated pipeline IR. + motifs: Detected motifs; the question only surfaces when at + least one matches the metadata-driven bulk copy pattern. + + Returns: + The constructed :class:`TranslationQuestion`, or ``None`` when + the pipeline contains no metadata-driven motif. + """ + affected = _metadata_driven_motif_task_keys(pipeline, motifs) + if not affected: + return None + return TranslationQuestion( + question_id=QUESTION_METADATA_DRIVEN_CONSOLIDATE, + prompt="Consolidate the metadata-driven ingestions into one managed pipeline?", + rationale=( + "A Lookup feeds a ForEach that copies each row's table. Consolidating " + "replaces this loop with a single Lakeflow Connect or Lakeflow Spark " + "Declarative Pipeline whose objects list materialises each source as " + "its own streaming table. Keeping the loop preserves the existing " + "per-row Copy translation." + ), + options=( + QuestionOption( + value=MetadataDrivenConsolidate.KEEP.value, + label="Keep the per-row loop", + description="Preserves the ForEach + Copy translation as a motif scaffold.", + ), + QuestionOption( + value=MetadataDrivenConsolidate.CONSOLIDATE.value, + label="Consolidate into one pipeline", + description="Emits one pipeline resource that ingests every source from the lookup.", + ), + ), + affected_task_keys=affected, + default=MetadataDrivenConsolidate.KEEP.value, + ) + + +def _build_metadata_driven_access_question( + pipeline: Pipeline, + motifs: list, + answers: dict[str, str] | None = None, +) -> TranslationQuestion | None: + """Builds the metadata-source access question, gated on consolidate=yes. + + Args: + pipeline: Translated pipeline IR. + motifs: Detected motifs. + + Returns: + The constructed :class:`TranslationQuestion`, or ``None`` when no + metadata-driven motif applies. + """ + affected = _metadata_driven_motif_task_keys(pipeline, motifs) + if not affected: + return None + return TranslationQuestion( + question_id=QUESTION_METADATA_DRIVEN_ACCESS, + prompt="Do you have access to query the metadata source and approve doing so?", + rationale=( + "Consolidating a metadata-driven ingestion requires materialising the " + "lookup query at translation time so each row becomes a pipeline object. " + "Answering yes confirms the metadata source can be queried during this " + "translation pass; answering no falls back to the per-row scaffold." + ), + options=( + QuestionOption( + value=MetadataDrivenAccess.YES.value, + label="Yes, query is allowed", + description="The metadata source is reachable and approved for read during translation.", + ), + QuestionOption( + value=MetadataDrivenAccess.NO.value, + label="No, skip materialising the lookup", + description="Keeps the per-row motif scaffold without inlining the configuration.", + ), + ), + affected_task_keys=affected, + default=MetadataDrivenAccess.NO.value, + conditions=((QUESTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value),), + ) + + +def _build_metadata_driven_size_question( + pipeline: Pipeline, + motifs: list, + answers: dict[str, str] | None = None, +) -> TranslationQuestion | None: + """Builds the t-shirt sizing question, gated on consolidate=yes. + + Args: + pipeline: Translated pipeline IR. + motifs: Detected motifs. + + Returns: + The constructed :class:`TranslationQuestion`, or ``None`` when no + metadata-driven motif applies. + """ + affected = _metadata_driven_motif_task_keys(pipeline, motifs) + if not affected: + return None + return TranslationQuestion( + question_id=QUESTION_METADATA_DRIVEN_SIZE, + prompt="Roughly how many configuration rows feed the metadata-driven ingestion?", + rationale=( + "The size determines whether the modifier inlines every lookup row into " + "one consolidated pipeline. Small and medium-sized configurations are " + "expanded inline; large configurations keep the per-row scaffold to " + "avoid generating an unwieldy pipeline definition." + ), + options=( + QuestionOption( + value=MetadataDrivenSize.SMALL.value, + label="S (under 50 rows)", + description="Lookup feeds fewer than 50 ingestion targets.", + ), + QuestionOption( + value=MetadataDrivenSize.MEDIUM.value, + label="M (under 250 rows)", + description="Lookup feeds 50 to 249 ingestion targets.", + ), + QuestionOption( + value=MetadataDrivenSize.LARGE.value, + label="L (250 or more rows)", + description="Lookup feeds 250+ targets; skip inline consolidation.", + ), + ), + affected_task_keys=affected, + default=MetadataDrivenSize.LARGE.value, + conditions=((QUESTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value),), + ) + + +def _build_metadata_driven_lookup_tool_question( + pipeline: Pipeline, + motifs: list, + answers: dict[str, str] | None = None, +) -> TranslationQuestion | None: + """Builds the agent-tool question for the lookup query, gated on size != L. + + Args: + pipeline: Translated pipeline IR. + motifs: Detected motifs. + + Returns: + The constructed :class:`TranslationQuestion`, or ``None`` when no + metadata-driven motif applies. + """ + affected = _metadata_driven_motif_task_keys(pipeline, motifs) + if not affected: + return None + return TranslationQuestion( + question_id=QUESTION_METADATA_DRIVEN_LOOKUP_TOOL, + prompt="Does the agent have a tool that can run the lookup query?", + rationale=( + "When the agent has a Genie skill, an MCP database tool, or a SQL " + "warehouse it can call, the modifier asks the agent to execute the " + "lookup query directly and reuses the rows. When no tool is " + "available the agent prompts the user for a CSV file or " + "comma-separated string of values and the modifier ingests that." + ), + options=( + QuestionOption( + value=MetadataDrivenLookupTool.HAVE.value, + label="Yes, the agent can run the lookup", + description="Agent executes the lookup query via its own tool.", + ), + QuestionOption( + value=MetadataDrivenLookupTool.NONE.value, + label="No, ask the user for the values", + description="Agent prompts the user for a CSV file or string of values.", + ), + ), + affected_task_keys=affected, + default=MetadataDrivenLookupTool.NONE.value, + conditions=( + (QUESTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value), + (QUESTION_METADATA_DRIVEN_ACCESS, MetadataDrivenAccess.YES.value), + ), + ) + + +def _metadata_driven_motif_task_keys( + pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None +) -> tuple[str, ...]: + """Returns the task keys of metadata-driven bulk-copy motifs in *pipeline*. + + Args: + pipeline: Translated pipeline IR after motif collapsing. + motifs: Detected motifs for the pipeline. + + Returns: + Tuple of motif activity task keys whose source motif is the + metadata-driven bulk-copy pattern. Empty when no such motif is + present. + """ + del motifs + return tuple( + activity.task_key + for activity in pipeline.tasks + if isinstance(activity, MotifActivity) and activity.motif_id == METADATA_DRIVEN_MOTIF_ID + ) + + +def _affected_task_keys_for_lakeflow_connect( + pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None +) -> tuple[str, ...]: + """Returns the task keys eligible for Lakeflow Connect replacement. + + Args: + pipeline: Translated pipeline IR after motif collapsing. + motifs: Detected motifs for the pipeline. + + Returns: + Unique tuple combining standalone Copy activities with database + sources targeting Delta and motif activities representing + database ingestion patterns. + """ + copy_keys = [ + activity.task_key + for activity in walk_activities(pipeline.tasks) + if isinstance(activity, CopyActivity) and copy_eligible_for_any_lfc_connector(activity) + ] + motif_keys = _motif_task_keys_for_lakeflow_connect(pipeline, motifs) + return tuple(dict.fromkeys(copy_keys + motif_keys)) + + +def _copy_paradigm_decided_by_lfc(activity: CopyActivity, going_to_lfc: bool) -> bool: + """Reports whether a Copy's paradigm is already decided without a prompt. + + Args: + activity: Copy activity to inspect. + going_to_lfc: ``True`` when the caller answered the LFC question + with ``lakeflow_connect``. + + Returns: + ``True`` when the Copy is going to LFC (paradigm = managed + ingestion pipeline), or when its source query is unfit for both + LFC and SDP (paradigm forced to PySpark notebook). ``False`` + when the user still needs to pick between SDP and notebook. + """ + if going_to_lfc and copy_eligible_for_any_lfc_connector(activity): + return True + return copy_query_unfit_for_lfc(activity) + + +def _task_compute_decided_by_lfc(activity, going_to_lfc: bool) -> bool: + """Reports whether a task's compute mode is already decided by an LFC routing. + + Args: + activity: Activity to inspect. + going_to_lfc: ``True`` when the caller answered the LFC question + with ``lakeflow_connect``. + + Returns: + ``True`` for Copy activities that will become LFC pipelines + (LFC manages its own serverless compute). ``False`` for every + other non-Databricks task -- those still need a compute choice. + """ + if not going_to_lfc: + return False + return isinstance(activity, CopyActivity) and copy_eligible_for_any_lfc_connector(activity) + + +def _motif_task_keys_for_lakeflow_connect( + pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None +) -> list[str]: + """Returns motif task keys eligible for Lakeflow Connect replacement. + + When the caller supplies the original :class:`DetectedMotif` list + (in-process translator usage) the eligibility check uses the motif + definition. When the list is empty (CLI usage that only sees the + serialised IR), eligibility is derived directly from each + :class:`MotifActivity`'s ``source_type_hint`` and + ``databricks_replacement`` fields. + + Args: + pipeline: Translated pipeline IR after motif collapsing. + motifs: Detected motifs for the pipeline. May be empty. + + Returns: + Task keys of motif activities whose source hint is ``database`` + and whose Databricks replacement is a known ingestion pattern + that Lakeflow Connect can take over. + """ + motif_tasks_by_id = { + activity.motif_id: activity for activity in pipeline.tasks if isinstance(activity, MotifActivity) + } + if motifs: + return [ + motif_tasks_by_id[detected.definition.motif_id].task_key + for detected in motifs + if detected.source_type_hint == DATABASE_SOURCE_TYPE_HINT + and detected.definition.databricks_replacement in LAKEFLOW_CONNECT_MOTIF_REPLACEMENTS + and detected.definition.motif_id in motif_tasks_by_id + ] + return [ + activity.task_key + for activity in motif_tasks_by_id.values() + if activity.source_type_hint == DATABASE_SOURCE_TYPE_HINT + and activity.databricks_replacement in LAKEFLOW_CONNECT_MOTIF_REPLACEMENTS + ] + + +def _stamp_activity(activity: Activity, pipeline_preferences: TranslationPreferences) -> Activity: + """Stamps preference-derived decisions onto an activity. + + Args: + activity: Source activity from the IR. + pipeline_preferences: Pipeline-wide preferences; per-task overrides + apply via :meth:`TranslationPreferences.effective_for`. + + Returns: + A new activity instance with ``compute_mode``, ``target_format``, + and motif-replacement updates applied as appropriate. Control- + flow activities are recursed into so their inner bodies are + stamped too. + """ + activity_preferences = pipeline_preferences.effective_for(activity.task_key) + if isinstance(activity, ForEachActivity): + return _stamp_for_each_activity(activity, pipeline_preferences, activity_preferences) + if isinstance(activity, IfConditionActivity): + return _stamp_if_condition_activity(activity, pipeline_preferences, activity_preferences) + if isinstance(activity, SwitchActivity): + return _stamp_switch_activity(activity, pipeline_preferences, activity_preferences) + if isinstance(activity, CopyActivity): + return _stamp_copy_activity(activity, activity_preferences) + if isinstance(activity, MotifActivity): + return _stamp_motif_activity(activity, activity_preferences) + if isinstance(activity, (NotebookActivity, SparkPythonActivity)): + return dataclasses.replace( + activity, + compute_mode=_resolve_databricks_task_compute_mode(activity_preferences), + ) + return dataclasses.replace(activity, compute_mode=_resolve_compute_mode(activity, activity_preferences)) + + +def _stamp_for_each_activity( + activity: ForEachActivity, + pipeline_preferences: TranslationPreferences, + activity_preferences: TranslationPreferences, +) -> ForEachActivity: + """Stamps a ForEach activity and recurses into its inner body. + + Args: + activity: Source ForEach activity. + pipeline_preferences: Pipeline-wide preferences threaded into + inner activities so they re-resolve their own overrides. + activity_preferences: Preferences after per-task overrides for + *activity*. + + Returns: + A new :class:`ForEachActivity` with inner activities stamped. + """ + return dataclasses.replace( + activity, + inner_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.inner_activities], + compute_mode=_resolve_compute_mode(activity, activity_preferences), + ) + + +def _stamp_if_condition_activity( + activity: IfConditionActivity, + pipeline_preferences: TranslationPreferences, + activity_preferences: TranslationPreferences, +) -> IfConditionActivity: + """Stamps an IfCondition activity and recurses into both branches. + + Args: + activity: Source IfCondition activity. + pipeline_preferences: Pipeline-wide preferences threaded into + inner activities so they re-resolve their own overrides. + activity_preferences: Preferences after per-task overrides for + *activity*. + + Returns: + A new :class:`IfConditionActivity` with both branches stamped. + """ + return dataclasses.replace( + activity, + if_true_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.if_true_activities], + if_false_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.if_false_activities], + compute_mode=_resolve_compute_mode(activity, activity_preferences), + ) + + +def _stamp_switch_activity( + activity: SwitchActivity, + pipeline_preferences: TranslationPreferences, + activity_preferences: TranslationPreferences, +) -> SwitchActivity: + """Stamps a Switch activity and recurses into every case and the default. + + Args: + activity: Source Switch activity. + pipeline_preferences: Pipeline-wide preferences threaded into + inner activities so they re-resolve their own overrides. + activity_preferences: Preferences after per-task overrides for + *activity*. + + Returns: + A new :class:`SwitchActivity` with every case body stamped. + """ + stamped_cases = [ + SwitchCase( + value=case.value, + activities=[_stamp_activity(inner, pipeline_preferences) for inner in case.activities], + ) + for case in activity.cases + ] + return dataclasses.replace( + activity, + cases=stamped_cases, + default_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.default_activities], + compute_mode=_resolve_compute_mode(activity, activity_preferences), + ) + + +def _stamp_copy_activity( + activity: CopyActivity, + activity_preferences: TranslationPreferences, +) -> CopyActivity: + """Stamps a Copy activity with paradigm, compute, and Lakeflow Connect flags. + + Args: + activity: Source Copy activity. + activity_preferences: Effective preferences for this activity. + + Returns: + A new :class:`CopyActivity` whose ``target_format``, + ``compute_mode``, and ``use_lakeflow_connector`` fields reflect + the user's choices. Copies whose query is unfit for LFC and + SDP (joins, aggregates, etc.) are forced to the notebook + paradigm regardless of preference because the alternative + paradigms cannot represent arbitrary SQL. + """ + user_picked_lfc = activity_preferences.use_lakeflow_connectors is UseLakeflowConnectors.LAKEFLOW_CONNECT + use_lakeflow_connector = user_picked_lfc and copy_eligible_for_any_lfc_connector(activity) + paradigm = _resolve_paradigm(activity, activity_preferences, use_lakeflow_connector) + connector_type = ( + _resolve_lakeflow_connector_type(activity, activity_preferences) if use_lakeflow_connector else None + ) + return dataclasses.replace( + activity, + target_format=paradigm.value, + compute_mode=_resolve_compute_mode(activity, activity_preferences), + use_lakeflow_connector=use_lakeflow_connector, + lakeflow_connector_type=connector_type, + ) + + +def _resolve_paradigm( + activity: CopyActivity, + activity_preferences: TranslationPreferences, + use_lakeflow_connector: bool, +) -> CopyActivityParadigm: + """Resolves the paradigm (notebook vs SDP) for a Copy that won't go to LFC. + + Args: + activity: Source Copy activity. + activity_preferences: Effective preferences for this activity. + use_lakeflow_connector: ``True`` when the modifier already + routed the Copy to a managed LFC pipeline; the paradigm is + informational in that case. + + Returns: + ``CopyActivityParadigm.NOTEBOOK`` when the activity's source + query is unfit for SDP (joins, aggregates, etc.) so the + notebook PySpark + JDBC path is the only viable alternative. + Otherwise the user's preferred paradigm when the Copy targets + Delta, falling back to ``NOTEBOOK`` for non-Delta sinks. + """ + if copy_query_unfit_for_lfc(activity): + return CopyActivityParadigm.NOTEBOOK + if not copy_targets_delta(activity): + return CopyActivityParadigm.NOTEBOOK + return activity_preferences.copy_activity_paradigm + + +def _resolve_lakeflow_connector_type(activity: CopyActivity, activity_preferences: TranslationPreferences) -> str: + """Resolves which Lakeflow Connect connector to use for an eligible Copy. + + Args: + activity: Source Copy activity (already known to be LFC-eligible). + activity_preferences: Effective preferences for this activity. + + Returns: + Always the connector flavour the Copy is actually eligible for. + Query-based eligibility (parseable query + cursor column) wins + over the user's CDC preference because no cursor candidate + exists for a CDC connector to use on a query-only Copy. + Table-based Copies route to CDC because the query-based + connector requires a cursor column and there is none. + """ + if copy_eligible_for_lfc_query_based(activity): + return LAKEFLOW_CONNECTOR_TYPE_QUERY_BASED + return LakeflowConnectorType.CDC.value + + +def _stamp_motif_activity( + activity: MotifActivity, + activity_preferences: TranslationPreferences, +) -> MotifActivity: + """Stamps a Motif activity, swapping in Lakeflow Connect when eligible. + + Args: + activity: Source motif activity. + activity_preferences: Effective preferences for this activity. + + Returns: + A new :class:`MotifActivity` whose ``databricks_replacement`` is + set to ``lakeflow_connect_database`` when the motif represents a + database ingestion and the user opted into Lakeflow Connect. + Metadata-driven motifs also pick up the + ``consolidate_metadata_driven`` flag when the user approved + consolidation, granted access, and the size bucket is S or M. + """ + qualifies_for_lakeflow_connect = ( + activity_preferences.use_lakeflow_connectors is UseLakeflowConnectors.LAKEFLOW_CONNECT + and activity.source_type_hint == DATABASE_SOURCE_TYPE_HINT + ) + replacement = LAKEFLOW_CONNECT_REPLACEMENT if qualifies_for_lakeflow_connect else activity.databricks_replacement + notebook_template = ( + MOTIF_LAKEFLOW_CONNECT_DATABASE.notebook_template + if qualifies_for_lakeflow_connect + else activity.notebook_template + ) + consolidate = _should_consolidate_metadata_driven(activity, activity_preferences) + return dataclasses.replace( + activity, + databricks_replacement=replacement, + notebook_template=notebook_template, + compute_mode=_resolve_compute_mode(activity, activity_preferences), + consolidate_metadata_driven=consolidate, + ) + + +def _should_consolidate_metadata_driven( + activity: MotifActivity, + activity_preferences: TranslationPreferences, +) -> bool: + """Returns True when the modifier should consolidate a metadata-driven motif. + + Args: + activity: Source motif activity. + activity_preferences: Effective preferences for this activity. + + Returns: + ``True`` when the motif matches the metadata-driven bulk-copy + pattern, the user opted to consolidate, granted access to the + metadata source, and the configuration size is S or M. + ``False`` otherwise -- including when the IR was not stamped by + the metadata-driven prompts. + """ + if activity.motif_id != "metadata_driven_bulk_copy": + return False + if activity_preferences.metadata_driven_consolidate is not MetadataDrivenConsolidate.CONSOLIDATE: + return False + if activity_preferences.metadata_driven_access is not MetadataDrivenAccess.YES: + return False + return activity_preferences.metadata_driven_size is not MetadataDrivenSize.LARGE + + +def _resolve_compute_mode(activity: Activity, activity_preferences: TranslationPreferences) -> str: + """Resolves the compute mode an activity should run on. + + Args: + activity: Source activity. + activity_preferences: Effective preferences for this activity. + + Returns: + One of :data:`COMPUTE_MODE_SERVERLESS`, + :data:`COMPUTE_MODE_CLASSIC_SINGLE_NODE`, + :data:`COMPUTE_MODE_CLASSIC_MULTI_NODE`, or + :data:`COMPUTE_MODE_INHERIT`. + """ + if isinstance(activity, (NotebookActivity, SparkPythonActivity)): + return _resolve_databricks_task_compute_mode(activity_preferences) + if not is_non_databricks_task(activity): + return COMPUTE_MODE_INHERIT + if activity_preferences.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS: + return COMPUTE_MODE_SERVERLESS + if isinstance(activity, CopyActivity): + return COMPUTE_MODE_CLASSIC_MULTI_NODE + return COMPUTE_MODE_CLASSIC_SINGLE_NODE + + +def _resolve_databricks_task_compute_mode(activity_preferences: TranslationPreferences) -> str: + """Resolves the compute mode for an ADF Databricks-* task. + + Args: + activity_preferences: Effective preferences for the task. + + Returns: + :data:`COMPUTE_MODE_SERVERLESS` when the caller opted into + serverless; otherwise :data:`COMPUTE_MODE_INHERIT`, which leaves + the linked-service-derived binding in place. + """ + if activity_preferences.databricks_task_compute is DatabricksTaskCompute.SERVERLESS: + return COMPUTE_MODE_SERVERLESS + return COMPUTE_MODE_INHERIT diff --git a/src/orchestra/adapter/predicates.py b/src/orchestra/adapter/predicates.py new file mode 100644 index 0000000..aa211e9 --- /dev/null +++ b/src/orchestra/adapter/predicates.py @@ -0,0 +1,243 @@ +"""Pure IR predicates shared by the agent adapter and the pipeline modifier.""" + +from __future__ import annotations + +from typing import Final + +from flowx.adapter.constants import COPY_SOURCE_QUERY_KEYS, DATABASE_SOURCE_TOKENS, DELTA_SINK_TOKENS +from flowx.models.ir import ( + Activity, + AppendVariableActivity, + CopyActivity, + DeleteActivity, + FilterActivity, + ForEachActivity, + IfConditionActivity, + LookupActivity, + MotifActivity, + NotebookActivity, + SetVariableActivity, + SparkPythonActivity, + SwitchActivity, + WaitActivity, + WebActivity, +) + +_NON_DATABRICKS_ACTIVITY_TYPES: Final[tuple[type[Activity], ...]] = ( + CopyActivity, + LookupActivity, + WebActivity, + DeleteActivity, + WaitActivity, + FilterActivity, + SetVariableActivity, + AppendVariableActivity, + MotifActivity, +) + + +def walk_activities(activities: list[Activity]) -> list[Activity]: + """Flattens an activity tree by descending into every control-flow body. + + Args: + activities: Top-level activity list from a :class:`Pipeline`. + + Returns: + Flat list of every activity, including those nested inside + ForEach, IfCondition, and Switch bodies. + """ + flattened: list[Activity] = [] + for activity in activities: + flattened.append(activity) + flattened.extend(_child_activities(activity)) + return flattened + + +def copy_targets_delta(activity: CopyActivity) -> bool: + """Reports whether a Copy activity's sink resolves to a Delta table. + + Args: + activity: Copy activity to inspect. + + Returns: + ``True`` when the activity's sink format, dataset type, or + properties indicate a Delta sink; ``False`` otherwise. + """ + if activity.sink_format and activity.sink_format.lower() in DELTA_SINK_TOKENS: + return True + sink_dataset_type = (activity.sink_dataset_type or "").lower() + if any(token in sink_dataset_type for token in DELTA_SINK_TOKENS): + return True + sink_properties = activity.sink_properties or {} + return bool(sink_properties.get("table")) + + +def copy_has_source_query(activity: CopyActivity) -> bool: + """Reports whether a Copy activity reads its source via an explicit SQL query. + + Args: + activity: Copy activity to inspect. + + Returns: + ``True`` when one of the well-known query fields + (``query``, ``sqlReaderQuery``, ``sql_query``) is populated on + ``source_properties``; ``False`` otherwise (table-based read). + """ + source_properties = activity.source_properties or {} + return any(source_properties.get(key) for key in COPY_SOURCE_QUERY_KEYS) + + +def copy_query_is_parseable_for_lfc(activity: CopyActivity) -> bool: + """Reports whether a Copy's source query decomposes into LFC fields. + + Args: + activity: Copy activity to inspect. + + Returns: + ``True`` when the translator's query analyzer concluded the + query can be expressed as the LFC query-based connector's + structured fields (cursor / row_filter / include_columns / + exclude_columns). ``False`` for table-based Copies or Copies + whose query contains constructs LFC cannot represent. + """ + source_properties = activity.source_properties or {} + return bool(source_properties.get("query_parseable_for_lfc")) + + +def copy_query_has_cursor_column(activity: CopyActivity) -> bool: + """Reports whether the Copy's query analysis found a cursor column candidate. + + Args: + activity: Copy activity to inspect. + + Returns: + ``True`` when ``source_properties.query_cursor_column`` is set + (the translator detected a range or BETWEEN predicate suitable + for the LFC connector's cursor field). + """ + source_properties = activity.source_properties or {} + return bool(source_properties.get("query_cursor_column")) + + +def copy_eligible_for_lfc_cdc(activity: CopyActivity) -> bool: + """Reports whether a Copy can be migrated to the LFC CDC connector. + + Args: + activity: Copy activity to inspect. + + Returns: + ``True`` when the Copy reads a database table directly (no + ``sqlReaderQuery``) and has both a database source and a Delta + sink. The CDC connector reads change events without needing a + cursor column, so table-based reads are always eligible. + """ + return not copy_has_source_query(activity) and has_database_source(activity) and copy_targets_delta(activity) + + +def copy_eligible_for_lfc_query_based(activity: CopyActivity) -> bool: + """Reports whether a Copy can be migrated to the LFC query-based connector. + + Args: + activity: Copy activity to inspect. + + Returns: + ``True`` when the Copy has a database source, targets Delta, + the source query parses into the LFC query-based connector + fields, and the query contains a range predicate that can + drive the connector's cursor column. + """ + if not copy_has_source_query(activity): + return False + if not has_database_source(activity): + return False + if not copy_targets_delta(activity): + return False + if not copy_query_is_parseable_for_lfc(activity): + return False + return copy_query_has_cursor_column(activity) + + +def copy_eligible_for_any_lfc_connector(activity: CopyActivity) -> bool: + """Reports whether a Copy is eligible for at least one LFC connector flavour. + + Args: + activity: Copy activity to inspect. + + Returns: + ``True`` when either :func:`copy_eligible_for_lfc_cdc` or + :func:`copy_eligible_for_lfc_query_based` returns ``True``. + """ + return copy_eligible_for_lfc_cdc(activity) or copy_eligible_for_lfc_query_based(activity) + + +def copy_query_unfit_for_lfc(activity: CopyActivity) -> bool: + """Reports whether a Copy carries a query that LFC cannot represent. + + Args: + activity: Copy activity to inspect. + + Returns: + ``True`` when the activity has a source query but the + translator's analyzer marked it as not parseable (the query + contains JOIN, GROUP BY, aggregates, UNION, window functions, + subqueries, or column expressions). Such Copies should be + translated through PySpark notebooks regardless of paradigm + preference because LFC's query-based connector and SDP's + declarative table form both reject the query. + """ + if not copy_has_source_query(activity): + return False + return not copy_query_is_parseable_for_lfc(activity) + + +def has_database_source(activity: CopyActivity) -> bool: + """Reports whether a Copy activity's source is SQL Server, MySQL, or PostgreSQL. + + Args: + activity: Copy activity to inspect. + + Returns: + ``True`` when the source type contains a known database token; + ``False`` otherwise. + """ + source_type = (activity.source_type or "").lower() + return any(token in source_type for token in DATABASE_SOURCE_TOKENS) + + +def is_non_databricks_task(activity: Activity) -> bool: + """Reports whether *activity* runs outside the Databricks notebook surface. + + Args: + activity: Activity to classify. + + Returns: + ``True`` for Copy, Lookup, Web, Delete, Wait, Filter, variable, + and motif activities; ``False`` for ADF DatabricksNotebook and + DatabricksSparkPython tasks. + """ + if isinstance(activity, (NotebookActivity, SparkPythonActivity)): + return False + return isinstance(activity, _NON_DATABRICKS_ACTIVITY_TYPES) + + +def _child_activities(activity: Activity) -> list[Activity]: + """Returns the activities nested inside a control-flow activity. + + Args: + activity: Activity to descend into. + + Returns: + Flattened list of child activities, or an empty list when + *activity* is a leaf type. + """ + if isinstance(activity, ForEachActivity): + return walk_activities(activity.inner_activities) + if isinstance(activity, IfConditionActivity): + return walk_activities(activity.if_true_activities + activity.if_false_activities) + if isinstance(activity, SwitchActivity): + nested: list[Activity] = [] + for case in activity.cases: + nested.extend(walk_activities(case.activities)) + nested.extend(walk_activities(activity.default_activities)) + return nested + return [] diff --git a/src/orchestra/adapter/session.py b/src/orchestra/adapter/session.py new file mode 100644 index 0000000..a312438 --- /dev/null +++ b/src/orchestra/adapter/session.py @@ -0,0 +1,431 @@ +"""Agent adapter that drives the ask-validate-resume loop. + +:class:`TranslationSession` is the entry point an agent uses to +translate tool-call arguments into validated preferences. When the IR +raises questions the agent cannot answer from context alone, the +session surfaces them as structured :class:`TranslationQuestion` +objects (and, via :exc:`TranslationInputRequired`, as exceptions) so +the agent can route them back to the user. The pipeline modifier is +invoked only once every question has an answer. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from flowx.adapter.constants import ( + INPUT_ADF_RESOURCE_URL, + INPUT_ADF_SOURCE_PATH, + INPUT_BUNDLE_NAME, + INPUT_CATALOG, + INPUT_DATABRICKS_PROFILE, + INPUT_INVENTORY_PATH, + INPUT_OUTPUT_BUNDLE_PATH, + INPUT_OUTPUT_DIR, + INPUT_SCHEMA, + INPUT_TRANSLATION_REPORT_PATH, + PHASE_INGEST, + PHASE_PREPARE, + PHASE_TRANSLATE, +) +from flowx.adapter.models import ( + DEFAULT_PREFERENCES, + CopyActivityParadigm, + DatabricksTaskCompute, + LakeflowConnectorType, + MetadataDrivenAccess, + MetadataDrivenConsolidate, + MetadataDrivenLookupTool, + MetadataDrivenSize, + MigrationInputQuestion, + NonDatabricksTaskCompute, + PendingMigrationInputs, + PendingQuestions, + TranslationPreferences, + TranslationQuestion, + UseLakeflowConnectors, +) +from flowx.adapter.operations import ( + apply_preferences, + gather_questions, + validate_answer, +) +from flowx.models.ir import Pipeline +from flowx.models.motifs import DetectedMotif + + +class TranslationInputRequired(Exception): + """Raised by :meth:`TranslationSession.run` when answers are still missing. + + Attributes: + pending: The outstanding questions the agent should route to the + user before retrying :meth:`TranslationSession.run`. + """ + + def __init__(self, pending: PendingQuestions) -> None: + """Stores the pending questions on the exception. + + Args: + pending: Outstanding questions surfaced by the session. + """ + super().__init__( + f"{len(pending.questions)} translation question(s) require user input " + f"for pipeline {pending.pipeline_name!r}" + ) + self.pending = pending + + +@dataclass(slots=True, kw_only=True) +class TranslationSession: + """Coordinates the ask-validate-resume loop for one translated pipeline. + + A session is single-use: the caller drives it by either polling via + :meth:`pending` and :meth:`answer`, or calling :meth:`run` and + handling :exc:`TranslationInputRequired`. When every question is + answered, :meth:`run` (or :meth:`resume`) returns the + preference-stamped pipeline. + + Attributes: + pipeline: Translated pipeline IR after motif collapsing. + motifs: Detected motifs for the pipeline. Optional; only used to + decide whether the Lakeflow Connect question applies. + defaults: Baseline preferences applied when the caller skips a + question. Per-task overrides on this object are preserved + verbatim when :meth:`build_preferences` composes the final + snapshot. + """ + + pipeline: Pipeline + motifs: list[DetectedMotif] = field(default_factory=list) + defaults: TranslationPreferences = DEFAULT_PREFERENCES + _answers: dict[str, str] = field(default_factory=dict) + + def pending(self) -> PendingQuestions: + """Returns the questions still awaiting an answer. + + Returns: + A :class:`PendingQuestions` instance containing only the + questions whose preconditions are met by the IR and whose + IDs are not yet in the answer set. + """ + return gather_questions( + self.pipeline, + self.motifs, + answers=self._answers, + ) + + def answer(self, question_id: str, value: str) -> None: + """Validates and records a single answer. + + Args: + question_id: Stable question identifier from + :class:`TranslationQuestion`. + value: Caller-supplied answer string. + + Raises: + ValueError: When *question_id* is unknown or *value* is not + in the allowed set for the question. + """ + self._answers[question_id] = validate_answer(question_id, value) + + def answer_many(self, answers: dict[str, str]) -> None: + """Validates and records multiple answers atomically. + + Args: + answers: Mapping of question_id to the caller-supplied answer. + + Raises: + ValueError: When any pair fails validation. No answers from + the batch are recorded when the call raises. + """ + validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} + self._answers.update(validated) + + def find_question(self, question_id: str) -> TranslationQuestion | None: + """Looks up a pending question by its identifier. + + Args: + question_id: Stable question identifier. + + Returns: + The matching :class:`TranslationQuestion` if it is still + pending, otherwise ``None``. + """ + return next( + (question for question in self.pending().questions if question.question_id == question_id), + None, + ) + + def build_preferences(self) -> TranslationPreferences: + """Composes the validated preferences snapshot from collected answers. + + Returns: + A :class:`TranslationPreferences` where every answered field + takes the caller-supplied value and every unanswered field + falls back to the corresponding value on ``defaults``. + """ + return TranslationPreferences( + copy_activity_paradigm=CopyActivityParadigm( + self._answers.get("copy_activity_paradigm", self.defaults.copy_activity_paradigm) + ), + non_databricks_task_compute=NonDatabricksTaskCompute( + self._answers.get("non_databricks_task_compute", self.defaults.non_databricks_task_compute) + ), + use_lakeflow_connectors=UseLakeflowConnectors( + self._answers.get("use_lakeflow_connectors", self.defaults.use_lakeflow_connectors) + ), + databricks_task_compute=DatabricksTaskCompute( + self._answers.get("databricks_task_compute", self.defaults.databricks_task_compute) + ), + lakeflow_connector_type=LakeflowConnectorType( + self._answers.get("lakeflow_connector_type", self.defaults.lakeflow_connector_type) + ), + metadata_driven_consolidate=MetadataDrivenConsolidate( + self._answers.get("metadata_driven_consolidate", self.defaults.metadata_driven_consolidate) + ), + metadata_driven_access=MetadataDrivenAccess( + self._answers.get("metadata_driven_access", self.defaults.metadata_driven_access) + ), + metadata_driven_size=MetadataDrivenSize( + self._answers.get("metadata_driven_size", self.defaults.metadata_driven_size) + ), + metadata_driven_lookup_tool=MetadataDrivenLookupTool( + self._answers.get("metadata_driven_lookup_tool", self.defaults.metadata_driven_lookup_tool) + ), + per_task=self.defaults.per_task, + ) + + def resume(self) -> Pipeline: + """Returns the preference-stamped pipeline IR. + + Returns: + A new :class:`Pipeline` produced by applying the composed + preferences to ``self.pipeline``. The input pipeline is not + mutated. + """ + return apply_preferences(self.pipeline, self.build_preferences()) + + def run(self) -> Pipeline: + """Returns the modified pipeline, raising when input is still required. + + Returns: + The preference-stamped pipeline IR when every applicable + question has an answer. + + Raises: + TranslationInputRequired: When one or more questions are + still outstanding. The exception carries the pending + questions so the agent can route them to the user. + """ + pending = self.pending() + if pending.questions: + raise TranslationInputRequired(pending) + return self.resume() + + +_INGEST_QUESTIONS: tuple[MigrationInputQuestion, ...] = ( + MigrationInputQuestion( + question_id=INPUT_ADF_SOURCE_PATH, + prompt="Where are the ADF JSON exports?", + description=( + "Unity Catalog volume path (``/Volumes///``) " + "or a local directory containing the ADF ARM/JSON export." + ), + required=True, + ), + MigrationInputQuestion( + question_id=INPUT_ADF_RESOURCE_URL, + prompt="ADF resource URL?", + description=( + "Azure portal URL of the source Data Factory. Captured for " + "traceability and surfaced in the generated bundle README; " + "leave blank when the source is exported from a local copy." + ), + default="", + required=False, + ), + MigrationInputQuestion( + question_id=INPUT_OUTPUT_DIR, + prompt="Where should flowx write the ingest output?", + description="Directory the ingest phase writes ``inventory.json`` and ``ast/`` into.", + default="./orchestra_output/ingest", + required=False, + ), +) + +_TRANSLATE_QUESTIONS: tuple[MigrationInputQuestion, ...] = ( + MigrationInputQuestion( + question_id=INPUT_INVENTORY_PATH, + prompt="Path to the inventory.json from the ingest phase?", + description="Inventory produced by the ingest phase that the translator consumes.", + default="./orchestra_output/ingest/inventory.json", + required=False, + ), + MigrationInputQuestion( + question_id=INPUT_ADF_SOURCE_PATH, + prompt="Path to the ADF JSON exports?", + description="Same source directory the ingest phase consumed; needed for cross-references.", + required=True, + ), + MigrationInputQuestion( + question_id=INPUT_OUTPUT_DIR, + prompt="Where should flowx write the translate output?", + description="Directory the translate phase writes the report and IR into.", + default="./orchestra_output/translate", + required=False, + ), +) + +_PREPARE_QUESTIONS: tuple[MigrationInputQuestion, ...] = ( + MigrationInputQuestion( + question_id=INPUT_TRANSLATION_REPORT_PATH, + prompt="Path to the translation report?", + description=( + "Preference-stamped report from `python -m flowx.adapter modify`, " + "or the raw translate-phase report when no preferences were applied." + ), + default="./orchestra_output/translate/translation_report.stamped.json", + required=False, + ), + MigrationInputQuestion( + question_id=INPUT_OUTPUT_BUNDLE_PATH, + prompt="Where should the generated DAB bundle be written?", + description="Root directory for the emitted Databricks Declarative Automation Bundle.", + default="./dab_output", + required=False, + ), + MigrationInputQuestion( + question_id=INPUT_CATALOG, + prompt="Target Unity Catalog catalog?", + description="Default ``catalog`` bundle variable used by emitted notebooks and pipelines.", + default="main", + required=False, + ), + MigrationInputQuestion( + question_id=INPUT_SCHEMA, + prompt="Target Unity Catalog schema?", + description="Default ``schema`` bundle variable used by emitted notebooks and pipelines.", + default="default", + required=False, + ), + MigrationInputQuestion( + question_id=INPUT_BUNDLE_NAME, + prompt="Bundle name override?", + description="Defaults to the first translated pipeline's resource key when blank.", + default="", + required=False, + ), + MigrationInputQuestion( + question_id=INPUT_DATABRICKS_PROFILE, + prompt="Databricks CLI profile?", + description=( + "Profile used to download workspace-resident notebooks during the " + "prepare phase. Leave blank to use the default profile from " + "``~/.databrickscfg`` or the active ``DATABRICKS_*`` env vars." + ), + default="", + required=False, + ), +) + +_QUESTIONS_BY_PHASE: dict[str, tuple[MigrationInputQuestion, ...]] = { + PHASE_INGEST: _INGEST_QUESTIONS, + PHASE_TRANSLATE: _TRANSLATE_QUESTIONS, + PHASE_PREPARE: _PREPARE_QUESTIONS, +} + + +class UnknownMigrationPhaseError(ValueError): + """Raised when a MigrationInputSession is constructed with an unrecognised phase.""" + + +@dataclass(slots=True, kw_only=True) +class MigrationInputSession: + """Coordinates the free-text input prompts at the top of an flowx phase. + + A session is single-use: the caller drives it by polling + :meth:`pending` and recording answers via :meth:`answer`, then reads + them out with :meth:`collected` once every required input has a + value. The session is intentionally distinct from + :class:`TranslationSession` because the inputs it gathers are + free-text paths and identifiers rather than enum-backed choices. + + Attributes: + phase: One of ``"ingest"``, ``"translate"``, ``"prepare"``. + """ + + phase: str + _answers: dict[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Validates that *phase* is one of the supported migration phases. + + Raises: + UnknownMigrationPhaseError: When *phase* is not registered in + :data:`_QUESTIONS_BY_PHASE`. + """ + if self.phase not in _QUESTIONS_BY_PHASE: + raise UnknownMigrationPhaseError( + f"Unknown migration phase {self.phase!r}; expected one of {sorted(_QUESTIONS_BY_PHASE)}" + ) + + def pending(self) -> PendingMigrationInputs: + """Returns the input questions still awaiting an answer. + + Returns: + A :class:`PendingMigrationInputs` with the unanswered + questions for ``self.phase`` in registration order. + """ + questions = [ + question for question in _QUESTIONS_BY_PHASE[self.phase] if question.question_id not in self._answers + ] + return PendingMigrationInputs(phase=self.phase, questions=questions) + + def answer(self, question_id: str, value: str) -> None: + """Records an answer to one input question. + + Args: + question_id: Stable identifier of the question. + value: Caller-supplied string value. + + Raises: + ValueError: When *question_id* is not a known input for the + session's phase. + """ + if not any(question.question_id == question_id for question in _QUESTIONS_BY_PHASE[self.phase]): + raise ValueError(f"Unknown input question {question_id!r} for phase {self.phase!r}") + self._answers[question_id] = value + + def answer_many(self, answers: dict[str, str]) -> None: + """Records multiple input answers atomically. + + Args: + answers: Mapping of question_id to the caller-supplied value. + + Raises: + ValueError: When any pair references an unknown question. + No answers are recorded when the call raises. + """ + known_ids = {question.question_id for question in _QUESTIONS_BY_PHASE[self.phase]} + unknown = set(answers) - known_ids + if unknown: + raise ValueError(f"Unknown input questions for phase {self.phase!r}: {sorted(unknown)}") + self._answers.update(answers) + + def collected(self) -> dict[str, str]: + """Returns the collected answers merged with each question's default. + + Returns: + A dict keyed by question_id covering every question for the + phase: caller-supplied answers take precedence; otherwise + the question's ``default`` value (which may be the empty + string) is used. Required questions whose answers are + missing are omitted so the caller can detect them. + """ + collected: dict[str, str] = {} + for question in _QUESTIONS_BY_PHASE[self.phase]: + if question.question_id in self._answers: + collected[question.question_id] = self._answers[question.question_id] + elif question.default is not None: + collected[question.question_id] = question.default + return collected diff --git a/src/orchestra/bundler/constants.py b/src/orchestra/bundler/constants.py new file mode 100644 index 0000000..0d3780d --- /dev/null +++ b/src/orchestra/bundler/constants.py @@ -0,0 +1,21 @@ +"""String constants for DAB cluster definitions produced by the bundler.""" + +from __future__ import annotations + +from typing import Final + +from flowx.adapter.constants import ( + COMPUTE_MODE_CLASSIC_MULTI_NODE, + COMPUTE_MODE_CLASSIC_SINGLE_NODE, +) + +DEFAULT_JOB_CLUSTER_KEY: Final[str] = "default_cluster" +SINGLE_NODE_JOB_CLUSTER_KEY: Final[str] = "single_node_cluster" +MULTI_NODE_JOB_CLUSTER_KEY: Final[str] = "multi_node_cluster" + +MULTI_NODE_CLUSTER_NODE_TYPE_ID: Final[str] = "Standard_D8ds_v5" + +COMPUTE_MODE_TO_CLUSTER_KEY: Final[dict[str, str]] = { + COMPUTE_MODE_CLASSIC_SINGLE_NODE: SINGLE_NODE_JOB_CLUSTER_KEY, + COMPUTE_MODE_CLASSIC_MULTI_NODE: MULTI_NODE_JOB_CLUSTER_KEY, +} diff --git a/src/orchestra/bundler/dab_writer.py b/src/orchestra/bundler/dab_writer.py index 22d7524..fb4ddfb 100644 --- a/src/orchestra/bundler/dab_writer.py +++ b/src/orchestra/bundler/dab_writer.py @@ -12,6 +12,14 @@ import yaml +from flowx.adapter.operations import collect_workspace_artifact_paths +from flowx.bundler.constants import ( + COMPUTE_MODE_TO_CLUSTER_KEY, + DEFAULT_JOB_CLUSTER_KEY, + MULTI_NODE_CLUSTER_NODE_TYPE_ID, + MULTI_NODE_JOB_CLUSTER_KEY, + SINGLE_NODE_JOB_CLUSTER_KEY, +) from flowx.bundler.inner_job_params import normalize_value from flowx.bundler.notebook_writer import write_notebooks from flowx.bundler.prereqs_writer import ManualParameter, build_prereqs, render_setup_md @@ -176,6 +184,26 @@ def write_bundle( ) created_files.append(inner_yml_path.resolve()) + # 2b. Write Lakeflow pipeline resources (Lakeflow Connect ingestion + # definitions emitted by the Copy preparer's LFC branch). Each + # resource lives in its own YAML so the bundle parser merges them + # alongside the job resources via the ``include`` glob. + pipelines_dir = resources_dir / "pipelines" + for resource in _collect_pipeline_resources(workflow): + pipelines_dir.mkdir(parents=True, exist_ok=True) + resource_yml_path = pipelines_dir / f"{resource['resource_key']}.yml" + resource_yml_path.write_text( + yaml.dump( + _wrap_pipeline_resource(resource), + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + Dumper=_BundleYamlDumper, + ), + encoding="utf-8", + ) + created_files.append(resource_yml_path.resolve()) + # 3. Write generated notebooks src_dir = output_dir / "src" if workflow.notebooks: @@ -306,7 +334,7 @@ def main() -> None: set_profile(args.profile) if not args.no_vendor_workspace_files: - workspace_paths = _collect_workspace_artifact_paths(args.report) + workspace_paths = collect_workspace_artifact_paths(args.report) if workspace_paths: if not prompt_for_auth_if_missing(workspace_paths): print( @@ -459,22 +487,131 @@ def _build_databricks_yml( } -_DEFAULT_JOB_CLUSTER_KEY = "default_cluster" +def _build_default_job_clusters(needed_keys: set[str]) -> list[dict[str, Any]]: + """Builds the job_clusters stanza, emitting only the clusters in use. + Args: + needed_keys: Set of job_cluster_key strings referenced by any task + in the workflow. -def _build_default_job_clusters() -> list[dict[str, Any]]: - """Return a job_clusters stanza that binds notebook tasks to a real cluster.""" - return [ - { - "job_cluster_key": _DEFAULT_JOB_CLUSTER_KEY, - "new_cluster": { - "spark_version": "${var.spark_version}", - "node_type_id": "${var.node_type_id}", - "num_workers": 1, - "data_security_mode": "SINGLE_USER", - }, - } - ] + Returns: + Ordered list of cluster definitions for inclusion under the job's + ``job_clusters`` block. + """ + builders = ( + (DEFAULT_JOB_CLUSTER_KEY, _build_default_cluster), + (SINGLE_NODE_JOB_CLUSTER_KEY, _build_single_node_cluster), + (MULTI_NODE_JOB_CLUSTER_KEY, _build_multi_node_cluster), + ) + return [builder() for key, builder in builders if key in needed_keys] + + +def _build_default_cluster() -> dict[str, Any]: + """Builds the multi-purpose default job_cluster used for legacy bindings. + + Returns: + Cluster definition with one worker and bundle-variable knobs for + spark_version and node_type_id. + """ + return { + "job_cluster_key": DEFAULT_JOB_CLUSTER_KEY, + "new_cluster": { + "spark_version": "${var.spark_version}", + "node_type_id": "${var.node_type_id}", + "num_workers": 1, + "data_security_mode": "SINGLE_USER", + }, + } + + +def _build_single_node_cluster() -> dict[str, Any]: + """Builds the single-node job_cluster used for non-Databricks tasks under classic compute. + + Returns: + Cluster definition using ``is_single_node`` so Databricks + configures the cluster for single-node execution without + requiring ``num_workers``, custom Spark conf, or tags. + """ + return { + "job_cluster_key": SINGLE_NODE_JOB_CLUSTER_KEY, + "new_cluster": { + "spark_version": "${var.spark_version}", + "node_type_id": "${var.node_type_id}", + "is_single_node": True, + "data_security_mode": "SINGLE_USER", + }, + } + + +def _build_multi_node_cluster() -> dict[str, Any]: + """Builds the fixed two-node job_cluster used for Copy Data tasks under classic compute. + + Returns: + Cluster definition with two workers on the Copy Data instance + type and the bundle-variable spark_version knob. + """ + return { + "job_cluster_key": MULTI_NODE_JOB_CLUSTER_KEY, + "new_cluster": { + "spark_version": "${var.spark_version}", + "node_type_id": MULTI_NODE_CLUSTER_NODE_TYPE_ID, + "num_workers": 2, + "data_security_mode": "SINGLE_USER", + }, + } + + +def _collect_pipeline_resources(workflow: PreparedWorkflow) -> list[dict[str, Any]]: + """Returns every Lakeflow pipeline resource carried by *workflow* and its inner jobs. + + Args: + workflow: The prepared workflow being written. + + Returns: + Flat list of pipeline-resource dicts (each with ``resource_key`` + and ``definition``), including entries from inner workflows. + """ + resources = list(workflow.pipeline_resources) + for inner in workflow.inner_workflows: + resources.extend(inner.pipeline_resources) + return resources + + +def _wrap_pipeline_resource(resource: dict[str, Any]) -> dict[str, Any]: + """Wraps a pipeline definition in the DAB ``resources.pipelines`` envelope. + + Args: + resource: Dict with ``resource_key`` and ``definition`` keys as + produced by the Copy preparer's Lakeflow Connect branch. + + Returns: + A dict shaped for direct YAML serialisation under a bundle + resource file. + """ + return {"resources": {"pipelines": {resource["resource_key"]: resource["definition"]}}} + + +def _collect_required_cluster_keys(tasks: list[dict[str, Any]]) -> set[str]: + """Walks every task and returns the set of job_cluster keys actually bound. + + Args: + tasks: Top-level task dicts after cluster binding has run. + + Returns: + Set of ``job_cluster_key`` values present anywhere in the task + tree (including bodies under ``for_each_task.task``). + """ + return {task["job_cluster_key"] for task in _iter_tasks_recursively(tasks) if task.get("job_cluster_key")} + + +def _strip_compute_mode_markers(tasks: list[dict[str, Any]]) -> None: + """Removes the private ``_compute_mode`` marker from every task before YAML output. + + Args: + tasks: Top-level task dicts (mutated in place). + """ + for task in _iter_tasks_recursively(tasks): + task.pop("_compute_mode", None) # Patterns that signal a base_parameter value couldn't be evaluated cleanly. @@ -547,17 +684,35 @@ def _any_task_uses_classic_cluster(tasks: list[dict[str, Any]]) -> bool: def _bind_cluster_to_notebook_tasks(tasks: list[dict[str, Any]]) -> None: - """Attaches the default job_cluster_key to existing-notebook tasks.""" + """Binds notebook tasks to the cluster their compute_mode marker dictates. + + Tasks that the pipeline modifier marked ``serverless`` are left + unbound so they run on serverless compute. Tasks marked + ``classic_single_node`` or ``classic_multi_node`` bind to the + matching job_cluster. Tasks without a marker fall back to the + legacy behaviour: existing-workspace notebooks bind to + ``default_cluster`` and flowx-generated notebooks stay unbound. + + Args: + tasks: Top-level task dicts (mutated in place). + """ for task in _iter_tasks_recursively(tasks): notebook_task = task.get("notebook_task") if notebook_task is None: continue + if any(key in task for key in _CLUSTER_BINDING_KEYS): + continue + compute_mode = task.get("_compute_mode") + if compute_mode == "serverless": + continue + cluster_key = COMPUTE_MODE_TO_CLUSTER_KEY.get(compute_mode or "") + if cluster_key is not None: + task["job_cluster_key"] = cluster_key + continue notebook_path = notebook_task.get("notebook_path", "") if notebook_path.startswith("../src/"): continue - if any(key in task for key in _CLUSTER_BINDING_KEYS): - continue - task["job_cluster_key"] = _DEFAULT_JOB_CLUSTER_KEY + task["job_cluster_key"] = DEFAULT_JOB_CLUSTER_KEY def _rewrite_post_branch_dependencies(tasks: list[dict[str, Any]]) -> None: @@ -728,12 +883,11 @@ def _build_job_resource( if attach_clusters: _bind_cluster_to_notebook_tasks(workflow.tasks) - # Only emit the ``job_clusters`` block when at least one task is - # actually bound to it. When every task runs on serverless (the - # generated-notebook case), the job stays cluster-free and inherits - # the workspace's serverless defaults. - if _any_task_uses_classic_cluster(workflow.tasks): - job_def["job_clusters"] = _build_default_job_clusters() + needed_keys = _collect_required_cluster_keys(workflow.tasks) + if needed_keys: + job_def["job_clusters"] = _build_default_job_clusters(needed_keys) + + _strip_compute_mode_markers(workflow.tasks) if workflow.parameters: job_def["parameters"] = workflow.parameters @@ -776,54 +930,6 @@ def _normalize_base_parameters( return resolved -def _collect_workspace_artifact_paths(report_path: Path) -> list[str]: - """Return workspace-resident artifact paths the bundler would try to download. - - Used as a pre-flight before invoking the preparers: when the report - contains any absolute workspace paths (``/Shared/...``, ``/Workspace/...``) - or DBFS / Volume URIs, we want to surface them to the user so they can - authenticate before the prepare pass. - """ - try: - with open(report_path, encoding="utf-8") as report_file: - report = json.load(report_file) - except (OSError, json.JSONDecodeError): - return [] - - candidates: list[str] = [] - - def _walk_tasks(tasks: list[dict[str, Any]] | None) -> None: - for task in tasks or []: - task_type = task.get("type") - if task_type == "NotebookActivity": - path = task.get("notebook_path") or "" - if isinstance(path, str) and path.startswith("/") and not path.startswith("../"): - candidates.append(path) - elif task_type == "SparkPythonActivity": - path = task.get("python_file") or "" - if isinstance(path, str) and (path.startswith("dbfs:") or path.startswith("/")): - candidates.append(path) - elif task_type == "SparkJarActivity": - for lib in task.get("libraries") or []: - jar = lib.get("jar") if isinstance(lib, dict) else None - if isinstance(jar, str) and (jar.startswith("dbfs:") or jar.startswith("/")): - candidates.append(jar) - _walk_tasks(task.get("inner_activities")) - _walk_tasks(task.get("if_true_activities")) - _walk_tasks(task.get("if_false_activities")) - for case in task.get("cases") or []: - _walk_tasks(case.get("activities")) - _walk_tasks(task.get("default_activities")) - - if "tasks" in report: - _walk_tasks(report.get("tasks")) - for translation in report.get("translations") or []: - ir = translation.get("ir") or {} - _walk_tasks(ir.get("tasks")) - - return candidates - - def _load_report(report_path: Path) -> list[PreparedWorkflow]: """Loads a translation report and reconstruct PreparedWorkflow objects. @@ -879,25 +985,65 @@ def _pipeline_dict_to_workflow(pipeline_dict: dict[str, Any]) -> PreparedWorkflo expression resolution, and motif handling without duplicating the per-activity preparer logic. """ - activities = [_reconstruct_ir(task_ir) for task_ir in pipeline_dict.get("tasks", [])] + pipeline, parameters = pipeline_dict_to_ir(pipeline_dict) + workflow = prepare_workflow(pipeline) + if parameters: + workflow.parameters.extend(parameters) + return workflow + +def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[dict[str, Any]]]: + """Rehydrates a serialised pipeline IR dict into a typed :class:`Pipeline`. + + Args: + pipeline_dict: Dict produced by ``engine._pipeline_to_dict`` (or + the equivalent shape emitted by the adapter CLI bridge). + + Returns: + Tuple of ``(pipeline, parameters)`` where ``pipeline`` is the + rehydrated :class:`Pipeline` and ``parameters`` is the normalised + list of pipeline-level parameter definitions (empty when the + report carries no parameters). + """ + activities = [_reconstruct_ir(task_ir) for task_ir in pipeline_dict.get("tasks", [])] parameters: list[dict[str, Any]] = [] for param in pipeline_dict.get("parameters") or []: entry: dict[str, Any] = {"name": param["name"]} if "default" in param and param["default"] is not None: entry["default"] = normalize_value(str(param["default"])) parameters.append(entry) - pipeline = Pipeline( name=pipeline_dict.get("name", "unknown"), tasks=activities, parameters=parameters or None, + translation_preferences=_reconstruct_preferences(pipeline_dict.get("translation_preferences")), ) + return pipeline, parameters - workflow = prepare_workflow(pipeline) - if parameters: - workflow.parameters.extend(parameters) - return workflow + +def _reconstruct_preferences(raw: dict[str, Any] | None) -> Any: + """Rebuilds a :class:`TranslationPreferences` from its serialised form. + + Args: + raw: Dict emitted by ``engine._preferences_to_dict``, or ``None`` + when the report carries no preferences. + + Returns: + A :class:`TranslationPreferences` instance, or ``None`` when + *raw* is falsy. + """ + if not raw: + return None + from flowx.adapter.models import TranslationPreferences + + return TranslationPreferences( + copy_activity_paradigm=raw.get("copy_activity_paradigm", "notebook"), + non_databricks_task_compute=raw.get("non_databricks_task_compute", "serverless"), + use_lakeflow_connectors=raw.get("use_lakeflow_connectors", "existing"), + databricks_task_compute=raw.get("databricks_task_compute", "existing"), + lakeflow_connector_type=raw.get("lakeflow_connector_type", "cdc"), + per_task=dict(raw.get("per_task") or {}), + ) def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: @@ -929,6 +1075,9 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: sink_format=task_ir.get("sink_format"), sink_resolved_path=task_ir.get("sink_resolved_path"), column_mapping=task_ir.get("column_mapping"), + target_format=task_ir.get("target_format"), + use_lakeflow_connector=bool(task_ir.get("use_lakeflow_connector", False)), + lakeflow_connector_type=task_ir.get("lakeflow_connector_type"), ) if task_type == "WebActivity": return WebActivity( @@ -1051,6 +1200,8 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: original_activities=[], notebook_template=task_ir.get("notebook_template"), motif_config=task_ir.get("motif_config") or {}, + consolidate_metadata_driven=bool(task_ir.get("consolidate_metadata_driven", False)), + lookup_values=list(task_ir.get("lookup_values") or []), ) if task_type == "UnsupportedActivity": return UnsupportedActivity( @@ -1085,6 +1236,7 @@ def _common_activity_kwargs(task_ir: dict[str, Any]) -> dict[str, Any]: "depends_on": _reconstruct_dependencies(task_ir.get("depends_on")), "cluster": task_ir.get("cluster"), "required_parameters": dict(task_ir.get("required_parameters") or {}), + "compute_mode": task_ir.get("compute_mode"), } diff --git a/src/orchestra/models/ir.py b/src/orchestra/models/ir.py index c4e1cf4..3bb7fa7 100644 --- a/src/orchestra/models/ir.py +++ b/src/orchestra/models/ir.py @@ -4,7 +4,10 @@ from dataclasses import dataclass, field from types import MappingProxyType -from typing import Any, TypeAlias +from typing import TYPE_CHECKING, Any, TypeAlias + +if TYPE_CHECKING: + from flowx.adapter.models import TranslationPreferences @dataclass(slots=True, kw_only=True) @@ -58,6 +61,10 @@ class Activity: # activity. Preparers thread these into ``base_parameters`` so DAB # resolves the refs at job runtime. required_parameters: dict[str, str] = field(default_factory=dict) + # Compute mode stamped by the pipeline modifier in response to user + # preferences. One of "serverless", "classic_single_node", + # "classic_multi_node", "inherit", or None when no preferences were applied. + compute_mode: str | None = None @dataclass(slots=True, kw_only=True) @@ -104,6 +111,16 @@ class CopyActivity(Activity): sink_format: str | None = None sink_resolved_path: str | None = None column_mapping: list[dict[str, str]] | None = None + # Code paradigm chosen by the pipeline modifier: "notebook" (default + # PySpark output) or "sdp" (Lakeflow Spark Declarative Pipeline). + target_format: str | None = None + # True when the modifier selected Lakeflow Connect for an eligible + # database-source Copy → Delta ingestion. + use_lakeflow_connector: bool = False + # Lakeflow Connect connector flavour resolved by the modifier when + # use_lakeflow_connector is True: "query_based" or "cdc". None when + # the modifier did not stamp a connector type. + lakeflow_connector_type: str | None = None @dataclass(slots=True, kw_only=True) @@ -407,6 +424,17 @@ class MotifActivity(Activity): confidence_notes: list[str] = field(default_factory=list) original_activities: list[Activity] = field(default_factory=list) notebook_template: str | None = None + # Set by the pipeline modifier when the user opts into metadata-driven + # consolidation, has access to query the lookup source, and the + # configuration size is S or M. When True the preparer should emit + # a single consolidated pipeline whose objects come from lookup_values. + consolidate_metadata_driven: bool = False + # Concrete lookup rows materialised at translation time (CLI + # ``materialize-lookup`` subcommand or agent-supplied JSON). Each + # element is a dict mirroring a row from the original ADF Lookup + # query. Empty when consolidation is requested but values have not + # been resolved yet. + lookup_values: list[dict[str, Any]] = field(default_factory=list) # Small dict of motif-specific settings extracted from the collapsed # activities — e.g. ``{"lookup_query": ..., "lookup_scope": ...}`` for # ``for_each_ingestion``. Used by the notebook generator so the motif @@ -434,6 +462,7 @@ class Pipeline: tasks: list[Activity] = field(default_factory=list) tags: dict[str, str] = field(default_factory=dict) not_translatable: list[dict[str, Any]] = field(default_factory=list) + translation_preferences: TranslationPreferences | None = None @dataclass(frozen=True, slots=True) diff --git a/src/orchestra/models/motifs.py b/src/orchestra/models/motifs.py index 2e998b8..1bc85fa 100644 --- a/src/orchestra/models/motifs.py +++ b/src/orchestra/models/motifs.py @@ -182,6 +182,21 @@ class DetectedMotif: notebook_template="copy_and_notify.py", ) +MOTIF_LAKEFLOW_CONNECT_DATABASE = MotifDefinition( + motif_id="lakeflow_connect_database", + display_name="Lakeflow Connect Database Ingestion", + description=( + "A database ingestion (SQL Server, MySQL, or PostgreSQL writing " + "into Delta) that the pipeline modifier replaced with a managed " + "Lakeflow Connect pipeline. Emitted in place of a bespoke Copy " + "activity or a watermarked file-landing motif when the user opts " + "into Lakeflow Connect." + ), + expected_activity_types=("Copy", "Lookup", "SqlServerStoredProcedure"), + databricks_replacement="lakeflow_connect_database", + notebook_template="lakeflow_connect_database.py", +) + ALL_MOTIFS: tuple[MotifDefinition, ...] = ( MOTIF_INCREMENTAL_LOAD_WATERMARK, MOTIF_CDC_CHANGE_TRACKING, @@ -193,4 +208,5 @@ class DetectedMotif: MOTIF_SCD_TYPE_2, MOTIF_STAGED_LOAD_SYNAPSE, MOTIF_COPY_AND_NOTIFY, + MOTIF_LAKEFLOW_CONNECT_DATABASE, ) diff --git a/src/orchestra/preparer/activity_preparers/copy.py b/src/orchestra/preparer/activity_preparers/copy.py index fb648ca..c21cf58 100644 --- a/src/orchestra/preparer/activity_preparers/copy.py +++ b/src/orchestra/preparer/activity_preparers/copy.py @@ -5,6 +5,7 @@ import dataclasses import re from dataclasses import dataclass +from typing import Any from flowx.models.dab import SecretInstruction, SetupTask from flowx.models.ir import CopyActivity @@ -15,7 +16,17 @@ ) from flowx.preparer.activity_preparers.naming import notebook_filename from flowx.preparer.code_generator import generate_copy_notebook -from flowx.preparer.workflow_preparer import PreparedActivity +from flowx.preparer.workflow_preparer import PreparedActivity, build_common_task_fields + +_LAKEFLOW_SOURCE_TYPE_TO_CONNECTION_TYPE: dict[str, str] = { + "SqlServerSource": "SQLSERVER", + "AzureSqlSource": "SQLSERVER", + "MicrosoftSqlServerSource": "SQLSERVER", + "MySqlSource": "MYSQL", + "AzureMySqlSource": "MYSQL", + "PostgreSqlSource": "POSTGRESQL", + "AzurePostgreSqlSource": "POSTGRESQL", +} _ABFSS_URL_RE = re.compile(r"abfss://([^@]+)@([^/]+)/?(.*)") _VOLUME_NAME_SANITIZE_RE = re.compile(r"[^a-zA-Z0-9_]") @@ -63,7 +74,22 @@ def _augment_with_volume_paths(activity: CopyActivity, binding: _VolumeBinding) def prepare(activity: CopyActivity, *, scope: str = "") -> PreparedActivity: - """Converts a CopyActivity into a notebook_task with a generated copy notebook.""" + """Converts a CopyActivity into a notebook_task with a generated copy notebook. + + Args: + activity: The Copy activity IR node, already stamped by the + pipeline modifier with ``target_format`` and + ``use_lakeflow_connector``. + scope: Secret scope name (defaults to ``activity.task_key`` when empty). + + Returns: + A :class:`PreparedActivity` whose shape depends on the modifier's + choice: a ``pipeline_task`` plus pipeline resource and connection + setup notebook when ``use_lakeflow_connector`` is true, otherwise + a ``notebook_task`` with the legacy generated copy notebook. + """ + if activity.use_lakeflow_connector: + return _prepare_lakeflow_connect_copy(activity, scope=scope) source_type = activity.source_type or "" volume_binding: _VolumeBinding | None = None if source_type in FILE_SOURCE_TYPES: @@ -98,6 +124,37 @@ def prepare(activity: CopyActivity, *, scope: str = "") -> PreparedActivity: return PreparedActivity(task=task, notebooks=notebooks, secrets=secrets, setup_tasks=setup_tasks) +def _prepare_lakeflow_connect_copy(activity: CopyActivity, *, scope: str) -> PreparedActivity: + """Returns a PreparedActivity that materialises a Lakeflow Connect ingestion. + + Args: + activity: Copy activity stamped with ``use_lakeflow_connector=True``. + scope: Secret scope name (unused for the LFC path; accepted for + uniformity with the legacy preparer). + + Returns: + A :class:`PreparedActivity` whose ``task`` is a ``pipeline_task`` + referencing the emitted Lakeflow Connect pipeline resource and + whose ``setup_tasks`` create the matching Unity Catalog + connection. The connection is keyed by the source linked + service name so multiple Copies sharing a source emit one + shared connection. ``notebooks`` is empty: Lakeflow Connect is + driven purely by the pipeline resource and the connection setup + script. + """ + del scope + resource_key = _lakeflow_pipeline_resource_key(activity.task_key) + connection_name = _lakeflow_connection_name_for_activity(activity) + pipeline_definition = _build_lakeflow_pipeline_definition(activity, connection_name) + task = _build_common_pipeline_task(activity, resource_key) + setup_tasks = [_build_lakeflow_connection_setup_task(activity, connection_name)] + return PreparedActivity( + task=task, + setup_tasks=setup_tasks, + pipeline_resources=[{"resource_key": resource_key, "definition": pipeline_definition}], + ) + + def _build_secrets(activity: CopyActivity, source_type: str, scope_name: str) -> list[SecretInstruction]: """Returns the SecretInstructions a Copy activity needs to deploy.""" if source_type in JDBC_SOURCE_TYPES: @@ -147,6 +204,247 @@ def _build_setup_tasks( return setup_tasks +def _lakeflow_pipeline_resource_key(task_key: str) -> str: + """Returns the DAB resource key for a Copy activity's Lakeflow Connect pipeline. + + Args: + task_key: Sanitised task key of the source Copy activity. + + Returns: + A resource key suffixed with ``_lfc`` so it does not collide with + any other pipeline or job resource emitted by the bundle. + """ + return f"{task_key}_lfc" + + +def _lakeflow_connection_name_for_activity(activity: CopyActivity) -> str: + """Returns the Unity Catalog connection name for a Lakeflow Connect Copy. + + Args: + activity: Source Copy activity carrying source-side metadata. + + Returns: + A connection name namespaced under ``orchestra_`` and derived + from the source linked service name when available, so multiple + Copies that share a source linked service emit one connection. + Falls back to the activity task key when the IR does not record + a linked service name. + """ + source_properties = activity.source_properties or {} + linked_service_name = source_properties.get("linked_service_name") or activity.task_key + return f"orchestra_{_sanitize_identifier(linked_service_name)}_connection" + + +def _sanitize_identifier(value: str) -> str: + """Coerces a value into a valid Unity Catalog identifier. + + Args: + value: Source string (typically a linked service name). + + Returns: + The input with non-alphanumeric characters replaced by + underscores and adjacent underscores collapsed. + """ + cleaned = re.sub(r"[^A-Za-z0-9_]", "_", value) + cleaned = re.sub(r"_+", "_", cleaned) + return cleaned.strip("_") or "connection" + + +def _build_common_pipeline_task(activity: CopyActivity, resource_key: str) -> dict[str, Any]: + """Builds the ``pipeline_task`` dict referencing a Lakeflow Connect pipeline. + + Args: + activity: Copy activity whose IR carries dependency metadata. + resource_key: Resource key the pipeline will be emitted under. + + Returns: + A task dict with the base activity fields plus a + ``pipeline_task`` pointing at ``${resources.pipelines..id}``. + """ + task = build_common_task_fields(activity) + task["pipeline_task"] = {"pipeline_id": f"${{resources.pipelines.{resource_key}.id}}"} + return task + + +def _build_lakeflow_pipeline_definition(activity: CopyActivity, connection_name: str) -> dict[str, Any]: + """Builds the Lakeflow Connect pipeline resource definition. + + Args: + activity: Source Copy activity carrying source and sink metadata + plus the connector type the modifier resolved. + connection_name: Name of the Unity Catalog connection the + pipeline will read through. + + Returns: + A dict matching the DAB ``resources.pipelines`` schema. Emits a + ``query_based_connector_config`` ingestion definition for Copy + activities marked as query-based; otherwise emits the + table-based CDC ingestion definition. + """ + return { + "name": _lakeflow_pipeline_resource_key(activity.task_key), + "catalog": "${var.catalog}", + "target": "${var.schema}", + "ingestion_definition": { + "connection_name": connection_name, + "objects": [_build_lakeflow_ingestion_object(activity)], + }, + } + + +def _build_lakeflow_ingestion_object(activity: CopyActivity) -> dict[str, Any]: + """Builds the per-object entry under ``ingestion_definition.objects``. + + Args: + activity: Source Copy activity stamped with + ``lakeflow_connector_type``. + + Returns: + Either a ``table_configuration`` object that drives the + query-based connector or a ``table`` object that drives the CDC + connector, depending on the stamped connector type. + """ + if activity.lakeflow_connector_type == "query_based": + return {"table_configuration": _build_query_based_table_configuration(activity)} + return {"table": _build_cdc_table_configuration(activity)} + + +def _build_cdc_table_configuration(activity: CopyActivity) -> dict[str, Any]: + """Builds the CDC-connector ``table`` ingestion entry. + + Args: + activity: Source Copy activity supplying table metadata. + + Returns: + A dict suitable for direct YAML serialisation under + ``objects[].table``. Source catalog/schema/table values come + from the translator-resolved ``source_properties`` keys; when + the IR does not carry literal values, bundle variables stand in + so the user can fill them at deploy time. + """ + source_properties = activity.source_properties or {} + sink_properties = activity.sink_properties or {} + source_schema = source_properties.get("source_schema") or "${var.source_schema}" + source_table = source_properties.get("source_table") or sink_properties.get("table") or activity.task_key + source_catalog = ( + ( + source_properties.get("connection", {}).get("database") + if isinstance(source_properties.get("connection"), dict) + else None + ) + or source_properties.get("sourceCatalog") + or "${var.source_catalog}" + ) + destination_table = sink_properties.get("table") or source_table + return { + "source_catalog": source_catalog, + "source_schema": source_schema, + "source_table": source_table, + "destination_catalog": "${var.catalog}", + "destination_schema": "${var.schema}", + "destination_table": destination_table, + } + + +def _build_query_based_table_configuration(activity: CopyActivity) -> dict[str, Any]: + """Builds the ``table_configuration`` entry for the query-based connector. + + Args: + activity: Source Copy activity stamped by the translator's + query analyzer with ``query_cursor_column``, + ``query_row_filter``, and ``query_include_columns``. + + Returns: + A dict suitable for direct YAML serialisation under + ``objects[].table_configuration``. Emits the structured fields + the Lakeflow Connect query-based connector accepts (``cursor``, + ``row_filter``, ``include_columns``, ``exclude_columns``) + rather than an arbitrary ``query`` string. Source table info + (``source_catalog``, ``source_schema``, ``source_table``) is + carried alongside so the connector knows which physical table + to scan. + """ + source_properties = activity.source_properties or {} + sink_properties = activity.sink_properties or {} + raw_connection = source_properties.get("connection") + connection: dict[str, Any] = raw_connection if isinstance(raw_connection, dict) else {} + destination_table = sink_properties.get("table") or activity.task_key + source_schema = source_properties.get("source_schema") or "${var.source_schema}" + source_table = source_properties.get("source_table") or destination_table + source_catalog = connection.get("database") or source_properties.get("sourceCatalog") or "${var.source_catalog}" + + config: dict[str, Any] = {} + cursor_column = source_properties.get("query_cursor_column") + if cursor_column: + config["cursor"] = cursor_column + row_filter = source_properties.get("query_row_filter") + if row_filter: + config["row_filter"] = row_filter + include_columns = source_properties.get("query_include_columns") + if include_columns: + config["include_columns"] = list(include_columns) + exclude_columns = source_properties.get("query_exclude_columns") + if exclude_columns: + config["exclude_columns"] = list(exclude_columns) + + return { + "source_catalog": source_catalog, + "source_schema": source_schema, + "source_table": source_table, + "destination_catalog": "${var.catalog}", + "destination_schema": "${var.schema}", + "destination_table": destination_table, + "query_based_connector_config": config, + } + + +def _build_lakeflow_connection_setup_task(activity: CopyActivity, connection_name: str) -> SetupTask: + """Builds the Unity Catalog connection :class:`SetupTask` for a Lakeflow Copy. + + Args: + activity: Source Copy activity supplying the connector type hint + and any host/port the translator pulled from the linked + service's connection string. + connection_name: Name the setup notebook should create. + + Returns: + A :class:`SetupTask` of type ``connection`` whose config is + consumed by the existing connection setup notebook generator. + Host and port come from ``source_properties.connection`` when + the translator resolved them; otherwise placeholders force the + user to fill in real values before running setup. + """ + source_type = activity.source_type or "" + connection_type = _LAKEFLOW_SOURCE_TYPE_TO_CONNECTION_TYPE.get(source_type, "SQLSERVER") + source_properties = activity.source_properties or {} + raw_connection = source_properties.get("connection") + connection: dict[str, Any] = raw_connection if isinstance(raw_connection, dict) else {} + host = connection.get("host", "PLACEHOLDER_HOST") + port = str(connection.get("port", _default_port_for(connection_type))) + return SetupTask( + type="connection", + config={ + "connection_name": connection_name, + "connection_type": connection_type, + "host": host, + "port": port, + }, + ) + + +def _default_port_for(connection_type: str) -> str: + """Returns a sensible default port for a Lakeflow Connect source type. + + Args: + connection_type: One of ``SQLSERVER``, ``MYSQL``, ``POSTGRESQL``. + + Returns: + The canonical default port for the protocol as a string, falling + back to ``"1433"`` (SQL Server) for unknown types. + """ + return {"SQLSERVER": "1433", "MYSQL": "3306", "POSTGRESQL": "5432"}.get(connection_type, "1433") + + def _build_sink_volume_setup_task(activity: CopyActivity) -> SetupTask | None: """Returns a UC volume SetupTask for the sink side of *activity*, or None. diff --git a/src/orchestra/preparer/activity_preparers/motif.py b/src/orchestra/preparer/activity_preparers/motif.py index a6ade86..cf25589 100644 --- a/src/orchestra/preparer/activity_preparers/motif.py +++ b/src/orchestra/preparer/activity_preparers/motif.py @@ -1,22 +1,153 @@ -"""Preparer for MotifActivity -> notebook_task with motif scaffold notebook.""" +"""Preparer for MotifActivity -> notebook_task or consolidated pipeline_task.""" from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any +from flowx.models.dab import SetupTask from flowx.preparer.activity_preparers.helpers import build_notebook_activity_task from flowx.preparer.code_generator import generate_motif_notebook -from flowx.preparer.workflow_preparer import PreparedActivity +from flowx.preparer.workflow_preparer import PreparedActivity, build_common_task_fields if TYPE_CHECKING: from flowx.models.ir import MotifActivity def prepare(activity: MotifActivity, *, scope: str = "") -> PreparedActivity: - """Converts a MotifActivity into a notebook_task with the motif scaffold.""" + """Converts a MotifActivity into a DAB task. + + Args: + activity: The motif activity stamped by the pipeline modifier. + scope: Secret scope name (defaults to the activity task_key). + + Returns: + A :class:`PreparedActivity` whose shape depends on the motif: + metadata-driven motifs marked for consolidation emit a + consolidated Lakeflow Connect pipeline resource and a + ``pipeline_task``; every other motif keeps the legacy scaffold + notebook task. + """ + if activity.consolidate_metadata_driven and activity.lookup_values: + return _prepare_consolidated_metadata_driven(activity) task, notebooks = build_notebook_activity_task( activity, notebook_relative_path=f"notebooks/{activity.task_key}.py", notebook_content=generate_motif_notebook(activity), ) return PreparedActivity(task=task, notebooks=notebooks) + + +def _prepare_consolidated_metadata_driven(activity: MotifActivity) -> PreparedActivity: + """Returns a PreparedActivity that materialises a consolidated ingestion pipeline. + + Args: + activity: Motif activity carrying ``lookup_values`` and the + ``consolidate_metadata_driven`` flag. + + Returns: + A :class:`PreparedActivity` whose task is a ``pipeline_task`` + referencing a single Lakeflow Connect pipeline resource. The + pipeline's ``objects`` list contains one entry per lookup row. + """ + resource_key = _consolidated_resource_key(activity.task_key) + connection_name = _consolidated_connection_name(activity.task_key) + pipeline_definition = _build_consolidated_pipeline_definition(activity, connection_name, resource_key) + task = build_common_task_fields(activity) + task["pipeline_task"] = {"pipeline_id": f"${{resources.pipelines.{resource_key}.id}}"} + setup_tasks = [ + SetupTask( + type="connection", + config={ + "connection_name": connection_name, + "connection_type": "SQLSERVER", + "host": "PLACEHOLDER_HOST", + "port": "1433", + }, + ) + ] + return PreparedActivity( + task=task, + setup_tasks=setup_tasks, + pipeline_resources=[{"resource_key": resource_key, "definition": pipeline_definition}], + ) + + +def _consolidated_resource_key(task_key: str) -> str: + """Returns the DAB resource key for a consolidated metadata-driven pipeline. + + Args: + task_key: Sanitised task key of the source motif activity. + + Returns: + A resource key suffixed with ``_consolidated`` so it does not + collide with other pipeline resources in the bundle. + """ + return f"{task_key}_consolidated" + + +def _consolidated_connection_name(task_key: str) -> str: + """Returns the Unity Catalog connection name for a consolidated pipeline. + + Args: + task_key: Sanitised task key of the source motif activity. + + Returns: + A connection name namespaced under ``orchestra_`` so the setup + notebook can recreate it idempotently. + """ + return f"orchestra_{task_key}_connection" + + +def _build_consolidated_pipeline_definition( + activity: MotifActivity, + connection_name: str, + resource_key: str, +) -> dict[str, Any]: + """Builds the consolidated Lakeflow Connect pipeline definition. + + Args: + activity: Motif activity carrying the lookup rows. + connection_name: Name of the Unity Catalog connection the + pipeline will read through. + resource_key: Resource key the pipeline will be emitted under. + + Returns: + A dict matching the DAB ``resources.pipelines`` schema with one + ingestion object per row in ``activity.lookup_values``. + """ + return { + "name": resource_key, + "catalog": "${var.catalog}", + "target": "${var.schema}", + "ingestion_definition": { + "connection_name": connection_name, + "objects": [_build_object_from_lookup_row(row) for row in activity.lookup_values], + }, + } + + +def _build_object_from_lookup_row(row: dict[str, Any]) -> dict[str, Any]: + """Builds a single ``objects[]`` entry from one lookup row. + + Args: + row: Dict with optional ``source_catalog``, ``source_schema``, + ``source_table``, and ``destination_table`` keys. Common + ADF aliases (``schema_name``, ``table_name``) are also + accepted. + + Returns: + A dict suitable for direct YAML serialisation under + ``objects[].table``. Bundle variables back-fill any field the + lookup row did not supply. + """ + source_table = row.get("source_table") or row.get("table_name") or row.get("table") or "${var.source_table}" + return { + "table": { + "source_catalog": row.get("source_catalog") or row.get("catalog_name") or "${var.source_catalog}", + "source_schema": row.get("source_schema") or row.get("schema_name") or "${var.source_schema}", + "source_table": source_table, + "destination_catalog": "${var.catalog}", + "destination_schema": "${var.schema}", + "destination_table": row.get("destination_table") or source_table, + } + } diff --git a/src/orchestra/preparer/activity_preparers/notebook.py b/src/orchestra/preparer/activity_preparers/notebook.py index 79f0227..a756cec 100644 --- a/src/orchestra/preparer/activity_preparers/notebook.py +++ b/src/orchestra/preparer/activity_preparers/notebook.py @@ -116,20 +116,13 @@ def prepare( if is_existing_notebook: downloaded = download_notebook(resolved_path) if workspace_downloads_enabled() else None if downloaded is not None: - # Preserve the workspace basename so the bundle file mirrors the - # source notebook name; fall back to the activity-derived snake - # case when the workspace path is unusable (empty trailing - # segment, all special chars, etc.). filename = workspace_notebook_filename(resolved_path) or notebook_filename(activity.task_key, activity.name) notebook_relative_path = f"notebooks/{filename}" task["notebook_task"] = {"notebook_path": f"../src/{notebook_relative_path}"} if base_parameters is not None: task["notebook_task"]["base_parameters"] = base_parameters - # Downloaded notebooks were authored for classic compute and may - # use init scripts or DBR-only features that serverless can't run. - # _bind_cluster_to_notebook_tasks skips "../src/" paths because - # flowx-generated notebooks target serverless, so bind here. - task["job_cluster_key"] = "default_cluster" + if activity.compute_mode != "serverless": + task["job_cluster_key"] = "default_cluster" notebooks = [DabNotebook(relative_path=notebook_relative_path, content=downloaded)] return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/code_generator.py b/src/orchestra/preparer/code_generator.py index 89e04f9..4bb12fa 100644 --- a/src/orchestra/preparer/code_generator.py +++ b/src/orchestra/preparer/code_generator.py @@ -359,8 +359,15 @@ def generate_copy_notebook(activity: CopyActivity, *, scope: str = "") -> str: scope: Secret scope name (defaults to task_key if empty). Returns: - Complete notebook source code as a string. + Complete notebook source code as a string. When the IR is + stamped with ``use_lakeflow_connector=True`` the body is a + Lakeflow Connect scaffold; when ``target_format='sdp'`` it is a + Lakeflow Spark Declarative Pipeline scaffold; otherwise it is + the PySpark notebook the legacy translator emits. """ + if activity.target_format == "sdp": + return _generate_sdp_copy_notebook(activity, scope=scope) + header = _notebook_header(f"Copy: {activity.name}") source_type = activity.source_type or "" @@ -384,6 +391,102 @@ def generate_copy_notebook(activity: CopyActivity, *, scope: str = "") -> str: return header + _command_separator() + body +def _generate_sdp_copy_notebook(activity: CopyActivity, *, scope: str = "") -> str: + """Generates an SDP scaffold notebook for a Copy activity targeting Delta. + + Args: + activity: The CopyActivity IR node. + scope: Secret scope name (defaults to task_key if empty). + + Returns: + Notebook source code that defines a Lakeflow Spark Declarative + Pipeline table reading from the resolved source and materialising + into the resolved sink table. + """ + header = _notebook_header(f"SDP Copy: {activity.name}") + sink_table = _resolve_sink_table_reference(activity) + source_descriptor = _resolve_source_descriptor(activity, scope=scope) + body = textwrap.dedent(f"""\ + from pyspark import pipelines as sdp + from pyspark.sql import functions as F + + + @sdp.table( + name="{sink_table}", + comment="Generated by flowx from ADF Copy activity '{activity.name}'.", + ) + def {_safe_identifier(activity.task_key)}(): + return {source_descriptor} + """) + return header + _command_separator() + body + + +def _resolve_sink_table_reference(activity: CopyActivity) -> str: + """Resolves the fully-qualified Delta table reference for a Copy sink. + + Args: + activity: Copy activity to inspect. + + Returns: + Catalog/schema/table reference using DAB bundle variables when + only the bare table name is known, or a literal reference when + the IR already carries one. + """ + sink_properties = activity.sink_properties or {} + table = sink_properties.get("table") or activity.task_key + if "." in table: + return table + return f"${{var.catalog}}.${{var.schema}}.{table}" + + +def _resolve_source_descriptor(activity: CopyActivity, *, scope: str) -> str: + """Resolves a Spark read expression for an SDP scaffold body. + + Args: + activity: Copy activity whose source the scaffold reads. + scope: Secret scope name used for JDBC-style sources. + + Returns: + Python expression that returns a Spark DataFrame for the source. + Falls back to a placeholder ``spark.read.table(...)`` when the + source type is unrecognised so the scaffold still parses. + """ + source_properties = activity.source_properties or {} + source_path = source_properties.get("resolved_path") + source_type = activity.source_type or "" + if source_type in FILE_SOURCE_TYPES and source_path: + return f'spark.readStream.format("cloudFiles").load("{source_path}")' + if source_type in JDBC_SOURCE_TYPES: + secret_scope = scope or activity.task_key + return ( + 'spark.read.format("jdbc")\n' + f' .option("url", dbutils.secrets.get(scope="{secret_scope}", key="jdbc-url"))\n' + f' .option("user", dbutils.secrets.get(scope="{secret_scope}", key="jdbc-user"))\n' + f' .option("password", dbutils.secrets.get(scope="{secret_scope}", key="jdbc-password"))\n' + " .load()" + ) + if source_path: + return f'spark.read.load("{source_path}")' + return 'spark.read.table("source_placeholder")' + + +def _safe_identifier(value: str) -> str: + """Coerces a string into a Python identifier safe for use as a function name. + + Args: + value: Source string, typically a sanitised task key. + + Returns: + The input with non-identifier characters replaced by underscores, + falling back to ``ingest`` when the result would be empty. + """ + cleaned = "".join(ch if ch.isalnum() or ch == "_" else "_" for ch in value) + cleaned = cleaned.strip("_") + if not cleaned or cleaned[0].isdigit(): + cleaned = f"ingest_{cleaned}" if cleaned else "ingest" + return cleaned + + def _detect_imports(body: str) -> list[str]: """Return import lines required by code references found in *body*.""" needed: list[str] = [] diff --git a/src/orchestra/preparer/workflow_preparer.py b/src/orchestra/preparer/workflow_preparer.py index e3f4034..815ae4c 100644 --- a/src/orchestra/preparer/workflow_preparer.py +++ b/src/orchestra/preparer/workflow_preparer.py @@ -45,6 +45,11 @@ class PreparedActivity: # ``_case_``; ``prepare_workflow`` reads this map to # rewrite ``depends_on`` edges that referenced the original key. task_key_remap: dict[str, str] = field(default_factory=dict) + # Lakeflow pipeline resources (e.g. Lakeflow Connect managed + # ingestion pipelines) the bundle writer emits under + # ``resources/pipelines/.yml``. Each entry is a dict + # with ``resource_key`` and ``definition`` keys. + pipeline_resources: list[dict[str, Any]] = field(default_factory=list) @dataclass(slots=True, kw_only=True) @@ -59,6 +64,7 @@ class PreparedWorkflow: inner_workflows: list[PreparedWorkflow] = field(default_factory=list) parameters: list[dict[str, Any]] = field(default_factory=list) cluster_hints: list[dict[str, Any]] = field(default_factory=list) + pipeline_resources: list[dict[str, Any]] = field(default_factory=list) def run_if_from_adf_outcomes(outcomes: list[str | None]) -> str | None: @@ -153,18 +159,38 @@ def prepare_activity( preparer_fn = dispatch.get(type(activity)) if preparer_fn is None: if isinstance(activity, (PlaceholderActivity, UnsupportedActivity)): - return _prepare_placeholder(activity) - raise ValueError( - f"No preparer registered for activity type {type(activity).__name__} (task_key={activity.task_key!r})" - ) + prepared = _prepare_placeholder(activity) + else: + raise ValueError( + f"No preparer registered for activity type {type(activity).__name__} (task_key={activity.task_key!r})" + ) + elif type(activity) is NotebookActivity: + prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) + elif type(activity) is AppendVariableActivity: + prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) + else: + prepared = preparer_fn(activity, scope=scope) + + prepared.task = _stamp_compute_mode(prepared.task, activity.compute_mode) + return prepared + - # NotebookActivity rewrites ``@variables()`` references; AppendVariable - # reads the prior writer's task_key to find the value to append to. - if type(activity) is NotebookActivity: - return preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) - if type(activity) is AppendVariableActivity: - return preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) - return preparer_fn(activity, scope=scope) +def _stamp_compute_mode(task: dict[str, Any], compute_mode: str | None) -> dict[str, Any]: + """Returns a new task dict carrying a private compute-mode marker. + + Args: + task: Task dict produced by a per-type preparer. Not mutated. + compute_mode: Value from ``Activity.compute_mode``. When ``None`` + the marker is omitted and the original *task* is returned + unchanged. + + Returns: + A new dict shallow-copied from *task* with the ``_compute_mode`` + marker attached, or *task* itself when no marker applies. + """ + if not compute_mode: + return task + return {**task, "_compute_mode": compute_mode} def _prepare_placeholder(activity: Activity) -> PreparedActivity: @@ -210,12 +236,13 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: @dataclass(frozen=True, slots=True) class PreparedArtifacts: - """Immutable accumulator for the four artifact lists a workflow collects.""" + """Immutable accumulator for the artifact lists a workflow collects.""" notebooks: tuple[DabNotebook, ...] = () secrets: tuple[SecretInstruction, ...] = () setup_tasks: tuple[SetupTask, ...] = () inner_workflows: tuple[PreparedWorkflow, ...] = () + pipeline_resources: tuple[dict[str, Any], ...] = () def merge_prepared_artifacts( @@ -228,6 +255,7 @@ def merge_prepared_artifacts( secrets=artifacts.secrets + tuple(prepared.secrets), setup_tasks=artifacts.setup_tasks + tuple(prepared.setup_tasks), inner_workflows=artifacts.inner_workflows + tuple(prepared.inner_workflows), + pipeline_resources=artifacts.pipeline_resources + tuple(prepared.pipeline_resources), ) @@ -275,7 +303,57 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: tasks=all_tasks, notebooks=list(artifacts.notebooks), secrets=unique_secrets, - setup_tasks=list(artifacts.setup_tasks), + setup_tasks=_dedupe_setup_tasks(artifacts.setup_tasks), inner_workflows=list(artifacts.inner_workflows), cluster_hints=cluster_hints, + pipeline_resources=list(artifacts.pipeline_resources), ) + + +def _dedupe_setup_tasks(setup_tasks: tuple[SetupTask, ...]) -> list[SetupTask]: + """Returns the setup-task list with duplicates collapsed by identifying config. + + Args: + setup_tasks: Setup tasks aggregated across every prepared + activity in the workflow. + + Returns: + List with at most one entry per logical resource: connection + tasks dedupe by ``connection_name``; volume tasks dedupe by + ``volume_name``. Other task types are kept as-is so the + downstream setup-notebook generator still sees them. + """ + seen: set[tuple[str, str]] = set() + unique: list[SetupTask] = [] + for task in setup_tasks: + key = _setup_task_dedupe_key(task) + if key is None: + unique.append(task) + continue + if key in seen: + continue + seen.add(key) + unique.append(task) + return unique + + +def _setup_task_dedupe_key(task: SetupTask) -> tuple[str, str] | None: + """Returns the identity tuple used to dedupe a :class:`SetupTask`. + + Args: + task: A setup task collected from a prepared activity. + + Returns: + A ``(type, identifier)`` tuple for known setup task types, or + ``None`` when the task type does not have a stable identity + (which preserves the original behaviour of emitting each + occurrence). + """ + config = task.config or {} + if task.type == "connection": + name = config.get("connection_name") + return (task.type, str(name)) if name else None + if task.type == "volume": + name = config.get("volume_name") + return (task.type, str(name)) if name else None + return None diff --git a/src/orchestra/translator/activity_translators/copy.py b/src/orchestra/translator/activity_translators/copy.py index d8a519b..4fdf14d 100644 --- a/src/orchestra/translator/activity_translators/copy.py +++ b/src/orchestra/translator/activity_translators/copy.py @@ -13,6 +13,7 @@ resolve_interpolated_string, resolve_interpolated_string_for_notebook, ) +from flowx.translator.query_analysis import analyze_copy_query, dialect_for_source_type _DATASET_TYPE_TO_SPARK_FORMAT: dict[str, str] = { "DelimitedText": "csv", @@ -41,6 +42,29 @@ _ACCOUNT_NAME_RE = re.compile(r"AccountName=([A-Za-z0-9]+)", re.IGNORECASE) _DATASET_PARAM_RE = re.compile(r"^@dataset\(\)\.([A-Za-z_][A-Za-z0-9_]*)$") +# Database connection-string parsers: each picks up the canonical +# host/port/database fields from an ADF linked service's +# ``connectionString``. The patterns are intentionally tolerant of +# casing and surrounding whitespace because ADF accepts both +# ``Server=`` and ``server=`` etc. +_AZURE_SQL_SERVER_RE = re.compile(r"\bServer=(?:tcp:)?([^,;]+?)(?:,(\d+))?(?:;|$)", re.IGNORECASE) +_AZURE_SQL_DATABASE_RE = re.compile(r"\b(?:Initial Catalog|Database)=([^;]+)", re.IGNORECASE) +_MYSQL_SERVER_RE = re.compile(r"\b(?:Server|Host)=([^;]+)", re.IGNORECASE) +_MYSQL_PORT_RE = re.compile(r"\bPort=(\d+)", re.IGNORECASE) +_POSTGRES_SERVER_RE = re.compile(r"\b(?:Server|Host)=([^;]+)", re.IGNORECASE) +_POSTGRES_PORT_RE = re.compile(r"\bPort=(\d+)", re.IGNORECASE) + +_DATABASE_DEFAULT_PORTS: dict[str, int] = { + "AzureSqlDatabase": 1433, + "AzureSqlMI": 1433, + "SqlServer": 1433, + "AzureMySql": 3306, + "MySql": 3306, + "AzurePostgreSql": 5432, + "PostgreSql": 5432, + "Oracle": 1521, +} + @dataclass(slots=True) class SinkPathInfo: @@ -100,6 +124,8 @@ def _resolve_param_value( return "" if isinstance(raw, dict) and raw.get("type") == "Expression": raw = raw.get("value", "") + if isinstance(raw, (list, dict)): + return "" if not isinstance(raw, str): return str(raw) text = raw @@ -253,6 +279,248 @@ def _resolve_path_info( ) +def _extract_source_query_text(source_properties: dict[str, Any]) -> str | None: + """Returns the SQL query a Copy source executes, when one is supplied. + + Args: + source_properties: ``source_properties`` dict on the Copy IR + (still carrying raw ADF field names). + + Returns: + First non-empty value across the well-known query keys + (``sqlReaderQuery``, ``query``, ``sql_query``). ADF expression + wrappers (``{type: "Expression", value: "..."}``) are unwrapped + to their inner string. ``None`` when the source reads a table + directly. + """ + for key in ("sqlReaderQuery", "query", "sql_query"): + value = source_properties.get(key) + if value is None: + continue + if isinstance(value, dict) and value.get("type") == "Expression": + value = value.get("value") + if isinstance(value, str) and value.strip(): + return value + return None + + +def _effective_dataset_params(dataset_ref: Any, dataset_props: dict[str, Any]) -> dict[str, Any]: + """Returns the effective parameter map for a dataset reference. + + Args: + dataset_ref: Activity-side dataset reference (carries parameter + overrides supplied at the call site). + dataset_props: Full properties dict of the referenced dataset. + + Returns: + Mapping of parameter name to resolved value: dataset declared + defaults first, then activity-side overrides win. + """ + declared = dataset_props.get("parameters") or {} + effective: dict[str, Any] = {} + for name, spec in declared.items(): + if isinstance(spec, dict) and "defaultValue" in spec: + effective[name] = spec["defaultValue"] + if dataset_ref is not None and getattr(dataset_ref, "parameters", None): + effective.update(dict(dataset_ref.parameters)) + return effective + + +def _resolve_table_reference( + dataset_ref: Any, + dataset_props: dict[str, Any] | None, + context: TranslationContext, +) -> tuple[str | None, str | None]: + """Resolves the schema and table name from a dataset reference. + + Args: + dataset_ref: Activity-side dataset reference. + dataset_props: Full properties dict of the referenced dataset. + context: Translation context for expression resolution. + + Returns: + Tuple of ``(schema, table)`` strings. Either may be ``None`` + when the dataset does not carry that field. ADF parameter + expressions are resolved against the dataset reference's + effective parameter map. Handles both the nested + ``typeProperties`` shape and the ``schemaTypePropertiesSchema`` + flattened form ``az datafactory dataset show`` emits. + """ + if not dataset_props: + return None, None + type_props = dataset_props.get("typeProperties") if isinstance(dataset_props.get("typeProperties"), dict) else None + effective_params = _effective_dataset_params(dataset_ref, dataset_props) + schema_raw = _pick_dataset_field( + type_props, + dataset_props, + ("schema", "database"), + ("schemaTypePropertiesSchema", "database"), + ) + table_raw = _pick_dataset_field( + type_props, + dataset_props, + ("table", "tableName"), + ("table", "tableName"), + ) + schema = _resolve_param_value(schema_raw, effective_params, context) if schema_raw is not None else None + table = _resolve_param_value(table_raw, effective_params, context) if table_raw is not None else None + return (schema or None), (table or None) + + +def _pick_dataset_field( + type_props: dict[str, Any] | None, + dataset_props: dict[str, Any], + nested_keys: tuple[str, ...], + flat_keys: tuple[str, ...], +) -> Any: + """Returns the first populated dataset field across nested and flat shapes. + + Args: + type_props: ``typeProperties`` dict when present, ``None`` + when the dataset is in the az-flattened shape. + dataset_props: Top-level dataset properties dict. + nested_keys: Keys to try inside ``type_props`` (nested ADF shape). + flat_keys: Keys to try at the top level (az flattened shape). + + Returns: + The first non-empty value found. Empty strings, empty lists, + and ``None`` are skipped so column-schema artifacts like + ``schema: []`` don't shadow the actual database schema stored + under a flattened key. + """ + candidates: list[Any] = [] + if type_props is not None: + candidates.extend(type_props.get(key) for key in nested_keys) + candidates.extend(dataset_props.get(key) for key in flat_keys) + for value in candidates: + if value is None: + continue + if isinstance(value, (list, dict)) and not value: + continue + if isinstance(value, str) and not value.strip(): + continue + return value + return None + + +def _resolve_dataset_linked_service_name(dataset_props: dict[str, Any] | None) -> str | None: + """Returns the linked service name a dataset references. + + Args: + dataset_props: Full properties dict of the referenced dataset. + + Returns: + Linked service name string, or ``None`` when not present. + """ + if not dataset_props: + return None + raw = dataset_props.get("linkedServiceName") or {} + if isinstance(raw, dict): + return raw.get("referenceName") or None + return str(raw) or None + + +def _resolve_database_connection( + linked_service_name: str, + definitions: AdfDefinitions, +) -> dict[str, Any]: + """Pulls host / port / database from a database linked service. + + Args: + linked_service_name: Linked service name (referenced by a dataset). + definitions: Full ADF definitions for lookups. + + Returns: + Dict with optional keys ``host``, ``port``, ``database``, and + ``type``. Empty dict when the linked service is missing or has + no parseable connection string. + """ + linked_service = definitions.linked_services.get(linked_service_name) if linked_service_name else None + if linked_service is None: + return {} + properties = linked_service.properties or {} + type_props = properties.get("typeProperties") or properties + ls_type = properties.get("type") or "" + connection_string = type_props.get("connectionString") + if isinstance(connection_string, dict): + connection_string = connection_string.get("value", "") + if not isinstance(connection_string, str): + connection_string = "" + host: str | None = type_props.get("server") or type_props.get("host") + port: int | None = type_props.get("port") + database: str | None = type_props.get("database") or type_props.get("databaseName") or type_props.get("catalog") + if connection_string: + host = host or _extract_connection_host(ls_type, connection_string) + port = port or _extract_connection_port(ls_type, connection_string) + database = database or _extract_connection_database(ls_type, connection_string) + default_port = _DATABASE_DEFAULT_PORTS.get(ls_type) + if port is None and default_port is not None: + port = default_port + out: dict[str, Any] = {} + if host: + out["host"] = str(host).strip() + if port: + out["port"] = int(port) + if database: + out["database"] = str(database).strip() + if ls_type: + out["type"] = ls_type + return out + + +def _extract_connection_host(ls_type: str, connection_string: str) -> str | None: + """Returns the host substring from a database connection string. + + Args: + ls_type: Linked service type (e.g. ``AzureSqlDatabase``). + connection_string: Raw connection-string value. + + Returns: + Host string with surrounding whitespace stripped, or ``None`` + when the pattern for this database family does not match. + """ + pattern = ( + _AZURE_SQL_SERVER_RE if "Sql" in ls_type else _MYSQL_SERVER_RE if "MySql" in ls_type else _POSTGRES_SERVER_RE + ) + match = pattern.search(connection_string) + return match.group(1).strip() if match else None + + +def _extract_connection_port(ls_type: str, connection_string: str) -> int | None: + """Returns the port number from a database connection string. + + Args: + ls_type: Linked service type. + connection_string: Raw connection-string value. + + Returns: + Integer port, or ``None`` when absent. + """ + if "Sql" in ls_type: + match = _AZURE_SQL_SERVER_RE.search(connection_string) + if match and match.group(2): + return int(match.group(2)) + return None + pattern = _MYSQL_PORT_RE if "MySql" in ls_type else _POSTGRES_PORT_RE + match = pattern.search(connection_string) + return int(match.group(1)) if match else None + + +def _extract_connection_database(ls_type: str, connection_string: str) -> str | None: + """Returns the database name from a database connection string. + + Args: + ls_type: Linked service type. + connection_string: Raw connection-string value. + + Returns: + Database name with surrounding whitespace stripped, or ``None``. + """ + del ls_type + match = _AZURE_SQL_DATABASE_RE.search(connection_string) + return match.group(1).strip() if match else None + + def _resolve_source_path(activity: AdfActivity, definitions: AdfDefinitions) -> str | None: """Resolves the full storage path from the activity's input dataset.""" if not activity.inputs: @@ -290,6 +558,37 @@ def translate( if resolved_path: source_properties["resolved_path"] = resolved_path + if activity.inputs: + source_dataset_ref = activity.inputs[0] + source_dataset_props = _dataset_props(source_dataset_ref, definitions) + source_schema, source_table = _resolve_table_reference(source_dataset_ref, source_dataset_props, context) + source_ls_name = _resolve_dataset_linked_service_name(source_dataset_props) + if source_schema: + source_properties["source_schema"] = source_schema + if source_table: + source_properties["source_table"] = source_table + if source_ls_name: + source_properties["linked_service_name"] = source_ls_name + connection = _resolve_database_connection(source_ls_name, definitions) + if connection: + source_properties["connection"] = connection + + raw_query = _extract_source_query_text(source_properties) + if raw_query: + dialect = dialect_for_source_type(source_type) + analysis = analyze_copy_query(raw_query, dialect=dialect) + if dialect: + source_properties["query_dialect"] = dialect + source_properties["query_parseable_for_lfc"] = analysis.parseable + if analysis.cursor_column: + source_properties["query_cursor_column"] = analysis.cursor_column + if analysis.row_filter: + source_properties["query_row_filter"] = analysis.row_filter + if analysis.include_columns: + source_properties["query_include_columns"] = list(analysis.include_columns) + if analysis.rejection_reasons: + source_properties["query_rejection_reasons"] = list(analysis.rejection_reasons) + sink_raw = type_properties.get("sink", {}) sink_type = sink_raw.get("type") sink_properties = {k: v for k, v in sink_raw.items() if k != "type"} if sink_raw else {} @@ -345,10 +644,12 @@ def translate( # an abfss:// path or None for tables). sink_resolved_path = _resolve_dataset_path(sink_dataset_props, definitions) - type_props = sink_dataset_props.get("typeProperties") or sink_dataset_props - sink_table_name = ( - type_props.get("tableName") or type_props.get("table") or sink_dataset_props.get("tableName") - ) + sink_schema, sink_table_name = _resolve_table_reference(sink_dataset_ref, sink_dataset_props, context) + sink_ls_name = _resolve_dataset_linked_service_name(sink_dataset_props) + if sink_schema: + sink_properties["schema"] = sink_schema + if sink_ls_name: + sink_properties["linked_service_name"] = sink_ls_name if sink_table_name: sink_properties = {**sink_properties, "table": sink_table_name} diff --git a/src/orchestra/translator/engine.py b/src/orchestra/translator/engine.py index a9fa8b6..038e2d9 100644 --- a/src/orchestra/translator/engine.py +++ b/src/orchestra/translator/engine.py @@ -476,13 +476,36 @@ def _pipeline_to_dict(pipeline: Pipeline) -> dict[str, Any]: Returns: Dictionary suitable for ``json.dumps``. """ - return { + result: dict[str, Any] = { "name": pipeline.name, "parameters": pipeline.parameters, "schedule": pipeline.schedule, "tags": pipeline.tags, "tasks": [_activity_to_dict(task) for task in pipeline.tasks], } + if pipeline.translation_preferences is not None: + result["translation_preferences"] = _preferences_to_dict(pipeline.translation_preferences) + return result + + +def _preferences_to_dict(preferences: Any) -> dict[str, Any]: + """Serialise a TranslationPreferences instance to a JSON-friendly dictionary. + + Args: + preferences: The :class:`TranslationPreferences` snapshot to serialise. + + Returns: + Dictionary with the four StrEnum fields rendered as their string + values and per-task overrides preserved verbatim. + """ + return { + "copy_activity_paradigm": str(preferences.copy_activity_paradigm), + "non_databricks_task_compute": str(preferences.non_databricks_task_compute), + "use_lakeflow_connectors": str(preferences.use_lakeflow_connectors), + "databricks_task_compute": str(preferences.databricks_task_compute), + "lakeflow_connector_type": str(preferences.lakeflow_connector_type), + "per_task": dict(preferences.per_task), + } def _activity_to_dict(task: Activity) -> dict[str, Any]: @@ -513,6 +536,8 @@ def _activity_to_dict(task: Activity) -> dict[str, Any]: ] if task.cluster: task_dict["cluster"] = task.cluster + if task.compute_mode: + task_dict["compute_mode"] = task.compute_mode extra = _activity_extra_fields(task) task_dict.update(extra) @@ -550,6 +575,12 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["sink_resolved_path"] = activity.sink_resolved_path if activity.column_mapping: extra["column_mapping"] = activity.column_mapping + if activity.target_format: + extra["target_format"] = activity.target_format + if activity.use_lakeflow_connector: + extra["use_lakeflow_connector"] = activity.use_lakeflow_connector + if activity.lakeflow_connector_type: + extra["lakeflow_connector_type"] = activity.lakeflow_connector_type case ForEachActivity(): extra["items_expression"] = activity.items_expression extra["concurrency"] = activity.concurrency @@ -651,6 +682,10 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["notebook_template"] = activity.notebook_template if activity.motif_config: extra["motif_config"] = activity.motif_config + if activity.consolidate_metadata_driven: + extra["consolidate_metadata_driven"] = activity.consolidate_metadata_driven + if activity.lookup_values: + extra["lookup_values"] = activity.lookup_values case PlaceholderActivity(): extra["original_type"] = activity.original_type extra["comment"] = activity.comment diff --git a/src/orchestra/translator/query_analysis.py b/src/orchestra/translator/query_analysis.py new file mode 100644 index 0000000..33f8ec7 --- /dev/null +++ b/src/orchestra/translator/query_analysis.py @@ -0,0 +1,287 @@ +"""SQL analysis that drives Lakeflow Connect eligibility for Copy queries. + +The Lakeflow Connect query-based connector accepts a structured set of +fields under ``query_based_connector_config`` -- ``cursor``, +``row_filter``, ``include_columns``, ``exclude_columns`` -- not a raw +SQL string. When flowx is asked to migrate a Copy activity whose +source carries a ``sqlReaderQuery``, the translator runs the query +through :func:`analyze_copy_query` to decide whether the query +decomposes cleanly into those fields and whether it contains a range +predicate the connector can adopt as its cursor column. + +Parsing is delegated to ``sqlglot`` so the analyzer can inspect a real +AST instead of guessing at structure with regex. ADF Copy queries +target many source databases, so :func:`analyze_copy_query` accepts an +optional ``dialect`` (matching ``sqlglot``'s dialect names: ``tsql``, +``mysql``, ``postgres``, ``oracle``, ``snowflake``, ...). The +translator picks the dialect from the ADF source type via +:func:`dialect_for_source_type`; when no dialect maps, sqlglot's +generic parser is used. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Final + +import sqlglot +from sqlglot import exp +from sqlglot.errors import ParseError + +_DIALECT_BY_SOURCE_TYPE: Final[tuple[tuple[str, str], ...]] = ( + ("sqlserver", "tsql"), + ("azuresql", "tsql"), + ("synapsesql", "tsql"), + ("sqlmi", "tsql"), + ("mysql", "mysql"), + ("azuremysql", "mysql"), + ("postgre", "postgres"), + ("azurepostgre", "postgres"), + ("oracle", "oracle"), + ("snowflake", "snowflake"), + ("teradata", "teradata"), + ("db2", "db2"), +) + +_RANGE_COMPARISONS: Final[tuple[type[exp.Expression], ...]] = (exp.GT, exp.GTE, exp.LT, exp.LTE) +_TOP_LEVEL_SET_OPS: Final[tuple[type[exp.Expression], ...]] = (exp.Union, exp.Intersect, exp.Except) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class QueryAnalysis: + """Structured result of inspecting a Copy activity's source SQL. + + Attributes: + parseable: ``True`` when the query elements decompose into the + Lakeflow Connect query-based connector's structured fields + (``cursor``, ``row_filter``, ``include_columns``, + ``exclude_columns``). ``False`` when the query contains + constructs the connector cannot represent. + cursor_column: Name of the column the connector should use as + its cursor for incremental ingestion. ``None`` when the + query has no range predicate. + row_filter: Static predicate the connector should apply on + every run (everything from the WHERE clause that does not + participate in the cursor predicate). ``None`` when the + query has no static predicates. + include_columns: Explicit column list from the SELECT clause, + or ``None`` when the query selects all columns. + exclude_columns: Reserved for future use; ADF queries do not + express exclusions directly so this remains ``None`` for + now. + rejection_reasons: Human-readable reasons the analyzer rejected + the query when ``parseable`` is ``False``. Empty when the + query parses cleanly. + """ + + parseable: bool + cursor_column: str | None = None + row_filter: str | None = None + include_columns: list[str] | None = None + exclude_columns: list[str] | None = None + rejection_reasons: list[str] = field(default_factory=list) + + +def dialect_for_source_type(source_type: str | None) -> str | None: + """Maps an ADF source-side activity type to a ``sqlglot`` dialect name. + + Args: + source_type: The Copy activity's source ``type`` field as it + appears in the ADF JSON (e.g. ``"AzureSqlSource"``, + ``"MySqlSource"``, ``"PostgreSqlV2Source"``). + + Returns: + A ``sqlglot`` dialect name when the source type contains a + recognised database token; ``None`` otherwise. ``None`` falls + through to sqlglot's generic parser, which is permissive enough + to handle the simple ``SELECT ... FROM ... WHERE`` queries ADF + typically emits even when the dialect is unknown. + """ + if not source_type: + return None + lowered = source_type.lower() + for token, dialect in _DIALECT_BY_SOURCE_TYPE: + if token in lowered: + return dialect + return None + + +def analyze_copy_query(query: str | None, *, dialect: str | None = None) -> QueryAnalysis: + """Analyses a Copy activity's source SQL for Lakeflow Connect fit. + + Args: + query: SQL text the Copy activity will execute against the + source database. ``None`` and empty strings short-circuit + to ``parseable=False`` with a single rejection reason. + dialect: Optional ``sqlglot`` dialect name describing the + source database (e.g. ``"tsql"``, ``"mysql"``, + ``"postgres"``). Used both to parse the query and to + render the ``row_filter`` string back out. ``None`` falls + through to sqlglot's generic parser. + + Returns: + A :class:`QueryAnalysis` describing whether the query decomposes + into the LFC query-based connector fields and what cursor / + filter / column extraction the connector should configure. + """ + if not query or not query.strip(): + return QueryAnalysis(parseable=False, rejection_reasons=["empty query"]) + + try: + tree = sqlglot.parse_one(query, dialect=dialect) + except ParseError as error: + return QueryAnalysis(parseable=False, rejection_reasons=[f"unparseable SQL: {error}"]) + + if isinstance(tree, _TOP_LEVEL_SET_OPS): + return QueryAnalysis(parseable=False, rejection_reasons=["contains UNION / INTERSECT / EXCEPT"]) + if not isinstance(tree, exp.Select): + return QueryAnalysis(parseable=False, rejection_reasons=["not a SELECT statement"]) + + rejection_reasons = _disqualifying_constructs(tree) + if rejection_reasons: + return QueryAnalysis(parseable=False, rejection_reasons=rejection_reasons) + + include_columns, column_rejection = _extract_select_columns(tree) + if column_rejection: + return QueryAnalysis(parseable=False, rejection_reasons=[column_rejection]) + + where = tree.args.get("where") + if not isinstance(where, exp.Where): + return QueryAnalysis(parseable=True, include_columns=include_columns) + + cursor_column, row_filter = _analyze_where(where, dialect) + return QueryAnalysis( + parseable=True, + cursor_column=cursor_column, + row_filter=row_filter, + include_columns=include_columns, + ) + + +def _disqualifying_constructs(tree: exp.Select) -> list[str]: + """Returns rejection reasons for any unsupported SQL construct in *tree*. + + Args: + tree: A parsed top-level ``SELECT`` expression. + + Returns: + List of human-readable rejection reasons. Empty when the + query is free of disqualifying constructs. + """ + reasons: list[str] = [] + if tree.find(exp.Join): + reasons.append("contains JOIN") + if tree.args.get("group"): + reasons.append("contains GROUP BY / HAVING") + if tree.args.get("having"): + reasons.append("contains GROUP BY / HAVING") + if tree.find(exp.AggFunc): + reasons.append("contains aggregate function") + if tree.find(exp.Window): + reasons.append("contains window function (OVER)") + if tree.args.get("distinct"): + reasons.append("contains DISTINCT") + if any(node is not tree for node in tree.find_all(exp.Select)): + reasons.append("contains subquery") + return reasons + + +def _extract_select_columns(tree: exp.Select) -> tuple[list[str] | None, str | None]: + """Returns the explicit column list from a SELECT clause. + + Args: + tree: A parsed top-level ``SELECT`` expression. + + Returns: + Tuple of ``(include_columns, rejection_reason)``. When the + query selects all columns with ``*``, ``include_columns`` is + ``None`` and ``rejection_reason`` is ``None``. When the column + list contains expressions, aliases, or qualified names the + connector cannot accept, ``include_columns`` is ``None`` and + ``rejection_reason`` carries a short explanation. + """ + items = tree.expressions + if not items: + return None, None + if len(items) == 1 and isinstance(items[0], exp.Star): + return None, None + columns: list[str] = [] + for item in items: + if isinstance(item, exp.Star): + return None, "SELECT clause mixes * with explicit columns LFC cannot represent" + if not isinstance(item, exp.Column): + return None, "SELECT clause contains expressions or aliases LFC cannot represent" + if item.table: + return None, "SELECT clause contains qualified columns LFC cannot represent" + columns.append(item.name) + return columns, None + + +def _analyze_where(where: exp.Where, dialect: str | None) -> tuple[str | None, str | None]: + """Splits a WHERE clause into a cursor column and a row filter. + + Args: + where: The ``exp.Where`` node off the parsed SELECT. + dialect: ``sqlglot`` dialect to use when rendering the + ``row_filter`` back to SQL text. + + Returns: + Tuple of ``(cursor_column, row_filter)``. ``cursor_column`` is + the leftmost column participating in a range (``>``, ``<``, + ``>=``, ``<=``) or ``BETWEEN`` predicate, or ``None`` when no + such predicate is present. ``row_filter`` is the remaining + AND-clauses rendered back to SQL and joined by ``AND``, or + ``None`` when no other predicates remain. + """ + body = where.this + if body is None: + return None, None + clauses = list(_flatten_and(body)) + cursor_column: str | None = None + remaining: list[exp.Expr] = [] + for clause in clauses: + column = _cursor_candidate(clause) + if column and cursor_column is None: + cursor_column = column + continue + remaining.append(clause) + if not remaining: + return cursor_column, None + row_filter = " AND ".join(clause.sql(dialect=dialect) for clause in remaining) + return cursor_column, row_filter + + +def _cursor_candidate(clause: exp.Expr) -> str | None: + """Returns the cursor column name if *clause* is a range or BETWEEN predicate. + + Args: + clause: One predicate fragment from a flattened WHERE clause. + + Returns: + The bare column name on the LHS when *clause* is a range + comparison or ``BETWEEN`` against a single column. ``None`` + otherwise (including for equality predicates, ``IN`` lists, and + any predicate whose LHS is not a bare column). + """ + if isinstance(clause, _RANGE_COMPARISONS) or isinstance(clause, exp.Between): + lhs = clause.this + if isinstance(lhs, exp.Column) and not lhs.table: + return lhs.name + return None + + +def _flatten_and(node: exp.Expr) -> list[exp.Expr]: + """Flattens an ``AND`` tree into a list of leaf predicates. + + Args: + node: The expression rooted at the WHERE clause body. + + Returns: + List of predicates in left-to-right order, treating nested + ``AND`` nodes as transparent. ``OR`` and other operators are + returned as single leaves so the caller treats them as part of + the row filter. + """ + if isinstance(node, exp.And): + return _flatten_and(node.left) + _flatten_and(node.right) + return [node] diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py new file mode 100644 index 0000000..0247e30 --- /dev/null +++ b/tests/unit/test_adapter.py @@ -0,0 +1,1158 @@ +"""Unit tests for the flowx agent adapter and the pipeline modifier.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from flowx.adapter import ( + CopyActivityParadigm, + DatabricksTaskCompute, + NonDatabricksTaskCompute, + TranslationInputRequired, + TranslationPreferences, + TranslationQuestion, + TranslationSession, + UseLakeflowConnectors, + apply_preferences, + gather_questions, + validate_answer, +) +from flowx.adapter.__main__ import main as adapter_cli_main +from flowx.adapter.constants import ( + COMPUTE_MODE_CLASSIC_MULTI_NODE, + COMPUTE_MODE_CLASSIC_SINGLE_NODE, + COMPUTE_MODE_INHERIT, + COMPUTE_MODE_SERVERLESS, + LAKEFLOW_CONNECT_REPLACEMENT, + QUESTION_COPY_ACTIVITY_PARADIGM, + QUESTION_DATABRICKS_TASK_COMPUTE, + QUESTION_LAKEFLOW_CONNECTOR_TYPE, + QUESTION_METADATA_DRIVEN_ACCESS, + QUESTION_METADATA_DRIVEN_CONSOLIDATE, + QUESTION_METADATA_DRIVEN_LOOKUP_TOOL, + QUESTION_METADATA_DRIVEN_SIZE, + QUESTION_NON_DATABRICKS_TASK_COMPUTE, + QUESTION_USE_LAKEFLOW_CONNECTORS, +) +from flowx.adapter.operations import allowed_values_for, enum_for +from flowx.models.ir import ( + CopyActivity, + ForEachActivity, + MotifActivity, + NotebookActivity, + Pipeline, + SparkPythonActivity, + WaitActivity, +) +from flowx.models.motifs import ( + MOTIF_INCREMENTAL_LOAD_WATERMARK, + DetectedMotif, +) + + +def _make_base(name: str = "task", task_key: str | None = None) -> dict[str, Any]: + """Builds the common Activity kwargs used by these tests.""" + return { + "name": name, + "task_key": task_key or name, + "description": None, + "timeout_seconds": None, + "max_retries": None, + "min_retry_interval_millis": None, + "depends_on": None, + "cluster": None, + } + + +def _delta_copy(name: str = "copy_to_delta") -> CopyActivity: + """Builds a Copy activity whose sink resolves to a Delta table.""" + return CopyActivity( + **_make_base(name), + source_type="AzureSqlSource", + sink_type="DeltaSink", + sink_format="delta", + sink_properties={"table": "raw.events"}, + ) + + +def _metadata_driven_motif(task_key: str = "motif_metadata_driven_bulk_copy") -> MotifActivity: + """Builds a metadata-driven bulk-copy motif activity for tests.""" + return MotifActivity( + **_make_base(task_key, task_key), + motif_id="metadata_driven_bulk_copy", + display_name="Metadata-Driven Bulk Copy", + databricks_replacement="for_each_ingestion", + matched_activity_names=["GetTables", "ForEachTable", "CopyTable"], + source_type_hint="database", + ) + + +def _query_delta_copy(name: str = "copy_query") -> CopyActivity: + """Builds a Copy activity that reads via a SQL query and writes to Delta. + + The query analysis fields the translator normally stamps are + included here so the IR is shaped exactly as it would be after + ``flowx.translator.engine`` runs against this Copy. + """ + return CopyActivity( + **_make_base(name), + source_type="AzureSqlSource", + sink_type="DeltaSink", + sink_format="delta", + sink_properties={"table": "raw.events"}, + source_properties={ + "sqlReaderQuery": "SELECT id, name FROM dbo.events WHERE updated_at > '2024-01-01'", + "query_parseable_for_lfc": True, + "query_cursor_column": "updated_at", + "query_include_columns": ["id", "name"], + }, + ) + + +def _file_copy(name: str = "copy_files") -> CopyActivity: + """Builds a Copy activity that does not target Delta.""" + return CopyActivity( + **_make_base(name), + source_type="BlobSource", + sink_type="ParquetSink", + sink_format="parquet", + ) + + +class TestPreferences: + def test_default_preferences_are_conservative(self): + prefs = TranslationPreferences() + assert prefs.copy_activity_paradigm is CopyActivityParadigm.NOTEBOOK + assert prefs.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS + assert prefs.use_lakeflow_connectors is UseLakeflowConnectors.EXISTING + assert prefs.databricks_task_compute is DatabricksTaskCompute.EXISTING + + def test_string_values_coerce_to_enums(self): + prefs = TranslationPreferences( + copy_activity_paradigm="sdp", + non_databricks_task_compute="classic", + ) + assert prefs.copy_activity_paradigm is CopyActivityParadigm.SDP + assert prefs.non_databricks_task_compute is NonDatabricksTaskCompute.CLASSIC + + def test_invalid_value_raises(self): + with pytest.raises(ValueError, match="not a valid CopyActivityParadigm"): + TranslationPreferences(copy_activity_paradigm="bogus") + + def test_per_task_override_takes_precedence(self): + base = TranslationPreferences( + copy_activity_paradigm="notebook", + per_task={"copy_a": {"copy_activity_paradigm": "sdp"}}, + ) + scoped = base.effective_for("copy_a") + assert scoped.copy_activity_paradigm is CopyActivityParadigm.SDP + other = base.effective_for("copy_b") + assert other.copy_activity_paradigm is CopyActivityParadigm.NOTEBOOK + + def test_effective_for_returns_self_when_no_override(self): + prefs = TranslationPreferences() + assert prefs.effective_for("missing") is prefs + + def test_enum_for_and_allowed_values_for(self): + assert enum_for("copy_activity_paradigm") is CopyActivityParadigm + assert enum_for("unknown") is None + assert set(allowed_values_for("copy_activity_paradigm")) == {"notebook", "sdp"} + assert allowed_values_for("unknown") == () + + +class TestGatherQuestions: + def test_no_questions_for_empty_pipeline(self): + pipeline = Pipeline(name="empty", tasks=[]) + pending = gather_questions(pipeline) + assert pending.pipeline_name == "empty" + assert pending.questions == [] + + def test_copy_paradigm_question_only_when_delta_sink_present(self): + delta_pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + question_ids = {q.question_id for q in gather_questions(delta_pipeline).questions} + assert QUESTION_COPY_ACTIVITY_PARADIGM in question_ids + + non_delta = Pipeline(name="p", tasks=[_file_copy()]) + question_ids = {q.question_id for q in gather_questions(non_delta).questions} + assert QUESTION_COPY_ACTIVITY_PARADIGM not in question_ids + + def test_non_databricks_compute_question_when_any_non_db_task(self): + pipeline = Pipeline(name="p", tasks=[WaitActivity(**_make_base("w"), wait_time_seconds=1)]) + ids = {q.question_id for q in gather_questions(pipeline).questions} + assert QUESTION_NON_DATABRICKS_TASK_COMPUTE in ids + + def test_lakeflow_connect_question_only_for_db_to_delta(self): + with_db = Pipeline(name="p", tasks=[_delta_copy()]) + ids = {q.question_id for q in gather_questions(with_db).questions} + assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids + + without_db = Pipeline(name="p", tasks=[_file_copy()]) + ids = {q.question_id for q in gather_questions(without_db).questions} + assert QUESTION_USE_LAKEFLOW_CONNECTORS not in ids + + def test_lakeflow_connect_question_surfaces_for_database_motif_without_detected_motifs(self): + """CLI callers don't have DetectedMotif objects; eligibility should derive from the IR alone.""" + motif_activity = MotifActivity( + **_make_base("motif_incremental_load_watermark", "motif_incremental_load_watermark"), + motif_id="incremental_load_watermark", + display_name="Incremental Load (Watermark)", + databricks_replacement="auto_loader", + matched_activity_names=["Lookup1", "Lookup2", "Copy", "UpdateWatermark"], + source_type_hint="database", + ) + pipeline = Pipeline(name="p", tasks=[motif_activity]) + pending = gather_questions(pipeline) + ids = {q.question_id for q in pending.questions} + assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids + question = next(q for q in pending.questions if q.question_id == QUESTION_USE_LAKEFLOW_CONNECTORS) + assert "motif_incremental_load_watermark" in question.affected_task_keys + + def test_lakeflow_connect_question_surfaces_for_database_motif(self): + motif_activity = MotifActivity( + **_make_base("motif_incremental_load_watermark", "motif_incremental_load_watermark"), + motif_id="incremental_load_watermark", + display_name="Incremental Load (Watermark)", + databricks_replacement="auto_loader", + matched_activity_names=["Lookup1", "Lookup2", "Copy", "UpdateWatermark"], + source_type_hint="database", + ) + pipeline = Pipeline(name="p", tasks=[motif_activity]) + motifs = [ + DetectedMotif( + definition=MOTIF_INCREMENTAL_LOAD_WATERMARK, + matched_activities=motif_activity.matched_activity_names, + source_type_hint="database", + confidence_notes=[], + ) + ] + pending = gather_questions(pipeline, motifs) + ids = {q.question_id for q in pending.questions} + assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids + lfc_question = next(q for q in pending.questions if q.question_id == QUESTION_USE_LAKEFLOW_CONNECTORS) + assert "motif_incremental_load_watermark" in lfc_question.affected_task_keys + + def test_databricks_task_compute_question_when_notebook_present(self): + pipeline = Pipeline( + name="p", + tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")], + ) + ids = {q.question_id for q in gather_questions(pipeline).questions} + assert QUESTION_DATABRICKS_TASK_COMPUTE in ids + + def test_databricks_task_compute_question_for_spark_python(self): + pipeline = Pipeline( + name="p", + tasks=[SparkPythonActivity(**_make_base("py"), python_file="dbfs:/scripts/etl.py")], + ) + ids = {q.question_id for q in gather_questions(pipeline).questions} + assert QUESTION_DATABRICKS_TASK_COMPUTE in ids + + def test_already_answered_filters_pending(self): + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + pending = gather_questions(pipeline, answers={QUESTION_COPY_ACTIVITY_PARADIGM: "sdp"}) + ids = {q.question_id for q in pending.questions} + assert QUESTION_COPY_ACTIVITY_PARADIGM not in ids + + def test_walks_into_for_each_inner_activities(self): + inner_copy = _delta_copy("inner_copy") + for_each = ForEachActivity( + **_make_base("loop"), + items_expression="@activity('lookup').output.value", + inner_activities=[inner_copy], + ) + pipeline = Pipeline(name="p", tasks=[for_each]) + question = next( + (q for q in gather_questions(pipeline).questions if q.question_id == QUESTION_COPY_ACTIVITY_PARADIGM), + None, + ) + assert question is not None + assert "inner_copy" in question.affected_task_keys + + +class TestValidateAnswer: + def test_accepts_allowed_value(self): + assert validate_answer("copy_activity_paradigm", "sdp") == "sdp" + + def test_rejects_unknown_question(self): + with pytest.raises(ValueError, match="Unknown question_id"): + validate_answer("not_a_question", "x") + + def test_rejects_invalid_value(self): + with pytest.raises(ValueError, match="Invalid answer"): + validate_answer("copy_activity_paradigm", "yaml") + + +class TestApplyPreferences: + def test_serverless_default_leaves_activities_on_serverless_compute(self): + pipeline = Pipeline(name="p", tasks=[_delta_copy(), WaitActivity(**_make_base("w"), wait_time_seconds=1)]) + modified = apply_preferences(pipeline, TranslationPreferences()) + copy_task = modified.tasks[0] + wait_task = modified.tasks[1] + assert copy_task.compute_mode == COMPUTE_MODE_SERVERLESS + assert wait_task.compute_mode == COMPUTE_MODE_SERVERLESS + + def test_classic_compute_routes_copy_to_multi_node_cluster(self): + pipeline = Pipeline(name="p", tasks=[_delta_copy(), WaitActivity(**_make_base("w"), wait_time_seconds=1)]) + prefs = TranslationPreferences(non_databricks_task_compute="classic") + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].compute_mode == COMPUTE_MODE_CLASSIC_MULTI_NODE + assert modified.tasks[1].compute_mode == COMPUTE_MODE_CLASSIC_SINGLE_NODE + + def test_databricks_task_serverless_stamps_serverless(self): + pipeline = Pipeline(name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")]) + prefs = TranslationPreferences(databricks_task_compute="serverless") + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].compute_mode == COMPUTE_MODE_SERVERLESS + + def test_databricks_task_existing_stamps_inherit(self): + pipeline = Pipeline(name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")]) + modified = apply_preferences(pipeline, TranslationPreferences()) + assert modified.tasks[0].compute_mode == COMPUTE_MODE_INHERIT + + def test_copy_paradigm_sdp_stamps_target_format(self): + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + prefs = TranslationPreferences(copy_activity_paradigm="sdp") + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].target_format == "sdp" + + def test_copy_paradigm_does_not_apply_to_non_delta_copy(self): + pipeline = Pipeline(name="p", tasks=[_file_copy()]) + prefs = TranslationPreferences(copy_activity_paradigm="sdp") + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].target_format == "notebook" + + def test_lakeflow_connect_flag_set_for_eligible_copy(self): + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].use_lakeflow_connector is True + + def test_lakeflow_connect_skipped_for_non_database_copy(self): + pipeline = Pipeline(name="p", tasks=[_file_copy()]) + prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].use_lakeflow_connector is False + + def test_motif_replacement_swapped_for_lakeflow_connect_when_database(self): + motif = MotifActivity( + **_make_base("motif_incremental_load_watermark", "motif_incremental_load_watermark"), + motif_id="incremental_load_watermark", + display_name="Incremental Load (Watermark)", + databricks_replacement="auto_loader", + matched_activity_names=["L1", "L2", "C", "U"], + source_type_hint="database", + ) + pipeline = Pipeline(name="p", tasks=[motif]) + prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].databricks_replacement == LAKEFLOW_CONNECT_REPLACEMENT + + def test_motif_replacement_unchanged_for_file_source(self): + motif = MotifActivity( + **_make_base("motif_file_landing"), + motif_id="file_landing_zone_processing", + display_name="File Landing Zone", + databricks_replacement="auto_loader_file_notification", + matched_activity_names=["GetMeta", "ForEach", "Copy"], + source_type_hint="files", + ) + pipeline = Pipeline(name="p", tasks=[motif]) + prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].databricks_replacement == "auto_loader_file_notification" + + def test_per_task_override_wins(self): + pipeline = Pipeline(name="p", tasks=[_delta_copy("c1"), _delta_copy("c2")]) + prefs = TranslationPreferences( + copy_activity_paradigm="notebook", + per_task={"c1": {"copy_activity_paradigm": "sdp"}}, + ) + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].target_format == "sdp" + assert modified.tasks[1].target_format == "notebook" + + def test_preferences_attached_to_pipeline(self): + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + prefs = TranslationPreferences(copy_activity_paradigm="sdp") + modified = apply_preferences(pipeline, prefs) + assert modified.translation_preferences is prefs + + def test_apply_preferences_does_not_mutate_input(self): + original = Pipeline(name="p", tasks=[_delta_copy()]) + apply_preferences(original, TranslationPreferences(copy_activity_paradigm="sdp")) + assert original.tasks[0].target_format is None + assert original.translation_preferences is None + + def test_recurses_into_for_each_inner_activities(self): + inner = _delta_copy("inner") + for_each = ForEachActivity( + **_make_base("loop"), + items_expression="@activity('l').output.value", + inner_activities=[inner], + ) + pipeline = Pipeline(name="p", tasks=[for_each]) + prefs = TranslationPreferences(copy_activity_paradigm="sdp") + modified = apply_preferences(pipeline, prefs) + inner_after = modified.tasks[0].inner_activities[0] + assert inner_after.target_format == "sdp" + + +class TestTranslationSession: + def test_pending_returns_only_outstanding_questions(self): + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + session = TranslationSession(pipeline=pipeline) + first = session.pending() + assert len(first.questions) > 0 + session.answer(QUESTION_COPY_ACTIVITY_PARADIGM, "sdp") + ids_after = {q.question_id for q in session.pending().questions} + assert QUESTION_COPY_ACTIVITY_PARADIGM not in ids_after + + def test_run_raises_when_questions_outstanding(self): + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + session = TranslationSession(pipeline=pipeline) + with pytest.raises(TranslationInputRequired) as info: + session.run() + assert info.value.pending.pipeline_name == "p" + assert any(q.question_id == QUESTION_COPY_ACTIVITY_PARADIGM for q in info.value.pending.questions) + + def test_run_returns_modified_pipeline_when_complete(self): + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + session = TranslationSession(pipeline=pipeline) + pending = session.pending() + answers = {q.question_id: q.default for q in pending.questions} + session.answer_many(answers) + modified = session.run() + assert modified.translation_preferences is not None + + def test_answer_validates(self): + session = TranslationSession(pipeline=Pipeline(name="p", tasks=[_delta_copy()])) + with pytest.raises(ValueError): + session.answer(QUESTION_COPY_ACTIVITY_PARADIGM, "yaml") + + def test_answer_many_is_atomic(self): + session = TranslationSession(pipeline=Pipeline(name="p", tasks=[_delta_copy()])) + with pytest.raises(ValueError): + session.answer_many({QUESTION_COPY_ACTIVITY_PARADIGM: "sdp", "bogus": "x"}) + assert QUESTION_COPY_ACTIVITY_PARADIGM not in session._answers + + def test_find_question_returns_pending_question(self): + session = TranslationSession(pipeline=Pipeline(name="p", tasks=[_delta_copy()])) + found = session.find_question(QUESTION_COPY_ACTIVITY_PARADIGM) + assert isinstance(found, TranslationQuestion) + session.answer(QUESTION_COPY_ACTIVITY_PARADIGM, "sdp") + assert session.find_question(QUESTION_COPY_ACTIVITY_PARADIGM) is None + + +class TestSerializationRoundtrip: + def test_preferences_survive_json_roundtrip(self): + from flowx.bundler.dab_writer import pipeline_dict_to_ir + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline( + name="p", tasks=[_delta_copy(), NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")] + ) + prefs = TranslationPreferences( + copy_activity_paradigm="sdp", + non_databricks_task_compute="classic", + use_lakeflow_connectors="lakeflow_connect", + databricks_task_compute="serverless", + ) + stamped = apply_preferences(pipeline, prefs) + roundtripped, _ = pipeline_dict_to_ir(json.loads(json.dumps(_pipeline_to_dict(stamped), default=str))) + assert roundtripped.translation_preferences.copy_activity_paradigm is CopyActivityParadigm.SDP + assert roundtripped.tasks[0].target_format == "sdp" + assert roundtripped.tasks[0].use_lakeflow_connector is True + assert roundtripped.tasks[0].compute_mode == COMPUTE_MODE_CLASSIC_MULTI_NODE + assert roundtripped.tasks[1].compute_mode == COMPUTE_MODE_SERVERLESS + + +class TestMigrationInputSession: + def test_ingest_session_lists_expected_questions(self): + from flowx.adapter import MigrationInputSession + + session = MigrationInputSession(phase="ingest") + ids = [q.question_id for q in session.pending().questions] + assert ids == ["adf_source_path", "adf_resource_url", "output_dir"] + + def test_translate_session_lists_expected_questions(self): + from flowx.adapter import MigrationInputSession + + session = MigrationInputSession(phase="translate") + ids = [q.question_id for q in session.pending().questions] + assert "inventory_path" in ids + assert "adf_source_path" in ids + + def test_prepare_session_lists_expected_questions(self): + from flowx.adapter import MigrationInputSession + + session = MigrationInputSession(phase="prepare") + ids = {q.question_id for q in session.pending().questions} + assert {"translation_report_path", "output_bundle_path", "catalog", "schema"} <= ids + + def test_unknown_phase_raises(self): + from flowx.adapter import MigrationInputSession, UnknownMigrationPhaseError + + with pytest.raises(UnknownMigrationPhaseError): + MigrationInputSession(phase="bogus") + + def test_answer_records_value_and_drops_from_pending(self): + from flowx.adapter import MigrationInputSession + + session = MigrationInputSession(phase="ingest") + session.answer("adf_source_path", "/Volumes/main/default/adf") + ids = [q.question_id for q in session.pending().questions] + assert "adf_source_path" not in ids + + def test_answer_rejects_unknown_question(self): + from flowx.adapter import MigrationInputSession + + session = MigrationInputSession(phase="ingest") + with pytest.raises(ValueError, match="Unknown input question"): + session.answer("not_a_field", "x") + + def test_collected_merges_answers_with_defaults(self): + from flowx.adapter import MigrationInputSession + + session = MigrationInputSession(phase="prepare") + session.answer("translation_report_path", "/tmp/report.json") + collected = session.collected() + assert collected["translation_report_path"] == "/tmp/report.json" + assert collected["catalog"] == "main" + assert collected["schema"] == "default" + + def test_collected_omits_required_when_missing(self): + from flowx.adapter import MigrationInputSession + + session = MigrationInputSession(phase="ingest") + collected = session.collected() + assert "adf_source_path" not in collected + assert collected["output_dir"] == "./orchestra_output/ingest" + + +class TestWorkspacePathsCli: + def test_workspace_paths_detects_notebook_paths(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline( + name="p", + tasks=[ + NotebookActivity(**_make_base("nb_a"), notebook_path="/Shared/team/a"), + NotebookActivity(**_make_base("nb_b"), notebook_path="/Shared/team/b"), + ], + ) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + out = tmp_path / "ws.json" + exit_code = adapter_cli_main(["workspace-paths", str(report_path), "--out", str(out)]) + assert exit_code == 0 + payload = json.loads(out.read_text()) + assert payload["paths"] == ["/Shared/team/a", "/Shared/team/b"] + assert payload["needs_auth"] is True + assert payload["suggested_hosts"] == [] + + def test_workspace_paths_reports_no_auth_when_paths_empty(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + out = tmp_path / "ws.json" + adapter_cli_main(["workspace-paths", str(report_path), "--out", str(out)]) + payload = json.loads(out.read_text()) + assert payload["paths"] == [] + assert payload["needs_auth"] is False + + def test_workspace_paths_suggests_host_from_databricks_linked_service(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline( + name="p", + tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/team/x")], + ) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + source_dir = tmp_path / "source" + (source_dir / "linked_services").mkdir(parents=True) + (source_dir / "linked_services" / "LS_AzureDatabricks.json").write_text( + json.dumps( + { + "name": "LS_AzureDatabricks", + "properties": {"type": "AzureDatabricks", "domain": "https://adb-1234.5.azuredatabricks.net"}, + } + ) + ) + (source_dir / "linked_services" / "LS_Other.json").write_text( + json.dumps({"name": "LS_Other", "properties": {"type": "AzureSqlDatabase"}}) + ) + out = tmp_path / "ws.json" + adapter_cli_main(["workspace-paths", str(report_path), "--source-dir", str(source_dir), "--out", str(out)]) + payload = json.loads(out.read_text()) + assert payload["suggested_hosts"] == ["https://adb-1234.5.azuredatabricks.net"] + + +class TestInputsCli: + def test_inputs_emits_ingest_questions(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): + exit_code = adapter_cli_main(["inputs", "ingest"]) + assert exit_code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["phase"] == "ingest" + ids = [q["question_id"] for q in payload["questions"]] + assert ids == ["adf_source_path", "adf_resource_url", "output_dir"] + + def test_inputs_writes_to_file(self, tmp_path: Path): + out = tmp_path / "questions.json" + exit_code = adapter_cli_main(["inputs", "prepare", "--out", str(out)]) + assert exit_code == 0 + payload = json.loads(out.read_text()) + assert payload["phase"] == "prepare" + ids = {q["question_id"] for q in payload["questions"]} + assert "output_bundle_path" in ids + + +class TestCli: + def test_inspect_emits_pending_questions(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + exit_code = adapter_cli_main(["inspect", str(report_path)]) + assert exit_code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["pipelines"][0]["pipeline_name"] == "p" + question_ids = {q["question_id"] for q in payload["pipelines"][0]["questions"]} + assert QUESTION_COPY_ACTIVITY_PARADIGM in question_ids + + def test_modify_stamps_preferences(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + answers_path = tmp_path / "answers.json" + answers_path.write_text( + json.dumps( + { + "copy_activity_paradigm": "sdp", + "non_databricks_task_compute": "classic", + "use_lakeflow_connectors": "lakeflow_connect", + } + ) + ) + out_path = tmp_path / "modified.json" + exit_code = adapter_cli_main(["modify", str(report_path), str(answers_path), "--out", str(out_path)]) + assert exit_code == 0 + modified = json.loads(out_path.read_text()) + assert modified["translation_preferences"]["copy_activity_paradigm"] == "sdp" + copy_task = next(task for task in modified["tasks"] if task["task_key"] == "copy_to_delta") + assert copy_task["target_format"] == "sdp" + assert copy_task["compute_mode"] == COMPUTE_MODE_CLASSIC_MULTI_NODE + assert copy_task["use_lakeflow_connector"] is True + + def test_materialize_lookup_from_csv_string(self, tmp_path: Path): + out = tmp_path / "lookup_values.json" + csv_source = "schema_name,table_name\ndbo,orders\ndbo,customers\n" + exit_code = adapter_cli_main(["materialize-lookup", csv_source, "--out", str(out)]) + assert exit_code == 0 + rows = json.loads(out.read_text()) + assert rows == [ + {"schema_name": "dbo", "table_name": "orders"}, + {"schema_name": "dbo", "table_name": "customers"}, + ] + + def test_materialize_lookup_from_csv_file(self, tmp_path: Path): + csv_path = tmp_path / "lookup.csv" + csv_path.write_text("table_name\norders\ncustomers\n") + out = tmp_path / "lookup_values.json" + exit_code = adapter_cli_main(["materialize-lookup", str(csv_path), "--out", str(out)]) + assert exit_code == 0 + rows = json.loads(out.read_text()) + assert rows == [{"table_name": "orders"}, {"table_name": "customers"}] + + def test_modify_threads_lookup_values_into_metadata_driven_motif(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + motif = _metadata_driven_motif() + pipeline = Pipeline(name="p", tasks=[motif]) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + answers_path = tmp_path / "answers.json" + answers_path.write_text( + json.dumps( + { + "metadata_driven_consolidate": "consolidate", + "metadata_driven_access": "yes", + "metadata_driven_size": "small", + } + ) + ) + lookup_values_path = tmp_path / "lookup_values.json" + lookup_values_path.write_text(json.dumps([{"source_table": "orders"}])) + out_path = tmp_path / "modified.json" + exit_code = adapter_cli_main( + [ + "modify", + str(report_path), + str(answers_path), + "--lookup-values", + str(lookup_values_path), + "--out", + str(out_path), + ] + ) + assert exit_code == 0 + modified = json.loads(out_path.read_text()) + motif_task = next(t for t in modified["tasks"] if t["motif_id"] == "metadata_driven_bulk_copy") + assert motif_task["consolidate_metadata_driven"] is True + assert motif_task["lookup_values"] == [{"source_table": "orders"}] + + def test_modify_rejects_invalid_answer(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + answers_path = tmp_path / "answers.json" + answers_path.write_text(json.dumps({"copy_activity_paradigm": "yaml"})) + out_path = tmp_path / "modified.json" + exit_code = adapter_cli_main(["modify", str(report_path), str(answers_path), "--out", str(out_path)]) + assert exit_code == 2 + + +class TestBundleOutput: + def test_classic_copy_compute_emits_two_node_multi_node_cluster(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) + prefs = TranslationPreferences(non_databricks_task_compute="classic") + stamped = apply_preferences(pipeline, prefs) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) + clusters = job_yml["resources"]["jobs"]["job"]["job_clusters"] + keys = {cluster["job_cluster_key"] for cluster in clusters} + assert "multi_node_cluster" in keys + multi_node = next(cluster for cluster in clusters if cluster["job_cluster_key"] == "multi_node_cluster") + assert multi_node["new_cluster"]["node_type_id"] == "Standard_D8ds_v5" + assert multi_node["new_cluster"]["num_workers"] == 2 + + def test_classic_single_node_cluster_uses_is_single_node_flag(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[WaitActivity(**_make_base("w"), wait_time_seconds=1)]) + prefs = TranslationPreferences(non_databricks_task_compute="classic") + stamped = apply_preferences(pipeline, prefs) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) + clusters = job_yml["resources"]["jobs"]["job"]["job_clusters"] + single = next(cluster for cluster in clusters if cluster["job_cluster_key"] == "single_node_cluster") + new_cluster = single["new_cluster"] + assert new_cluster["is_single_node"] is True + assert "num_workers" not in new_cluster + assert "spark_conf" not in new_cluster + assert "custom_tags" not in new_cluster + + def test_serverless_default_emits_no_job_clusters(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) + stamped = apply_preferences(pipeline, TranslationPreferences()) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) + assert "job_clusters" not in job_yml["resources"]["jobs"]["job"] + + def test_sdp_copy_emits_pyspark_pipelines_table_scaffold(self, tmp_path: Path): + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) + stamped = apply_preferences(pipeline, TranslationPreferences(copy_activity_paradigm="sdp")) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + notebook_path = tmp_path / "src" / "notebooks" / "copy_a.py" + body = notebook_path.read_text() + assert "from pyspark import pipelines as sdp" in body + assert "@sdp.table" in body + assert "import dlt" not in body + + def test_lakeflow_connect_emits_pipeline_resource_and_no_notebook(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) + stamped = apply_preferences(pipeline, TranslationPreferences(use_lakeflow_connectors="lakeflow_connect")) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + assert not (tmp_path / "src" / "notebooks" / "copy_a.py").exists() + pipeline_yml = tmp_path / "resources" / "pipelines" / "copy_a_lfc.yml" + assert pipeline_yml.exists() + resource = yaml.safe_load(pipeline_yml.read_text()) + lfc = resource["resources"]["pipelines"]["copy_a_lfc"] + assert lfc["name"] == "copy_a_lfc" + assert lfc["ingestion_definition"]["connection_name"] == "orchestra_copy_a_connection" + objects = lfc["ingestion_definition"]["objects"] + assert objects[0]["table"]["destination_table"] == "raw.events" + + def test_lakeflow_connect_job_task_references_pipeline(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) + stamped = apply_preferences(pipeline, TranslationPreferences(use_lakeflow_connectors="lakeflow_connect")) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) + task = job_yml["resources"]["jobs"]["job"]["tasks"][0] + assert "notebook_task" not in task + assert task["pipeline_task"]["pipeline_id"] == "${resources.pipelines.copy_a_lfc.id}" + + def test_metadata_driven_consolidate_question_surfaces_for_motif(self): + pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) + ids = {q.question_id for q in gather_questions(pipeline).questions} + assert QUESTION_METADATA_DRIVEN_CONSOLIDATE in ids + + def test_metadata_driven_followup_questions_gated_on_consolidate(self): + pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) + first_pass = gather_questions(pipeline).questions + ids = {q.question_id for q in first_pass} + assert QUESTION_METADATA_DRIVEN_CONSOLIDATE in ids + assert QUESTION_METADATA_DRIVEN_ACCESS not in ids + assert QUESTION_METADATA_DRIVEN_SIZE not in ids + assert QUESTION_METADATA_DRIVEN_LOOKUP_TOOL not in ids + + keep_pass = gather_questions(pipeline, answers={QUESTION_METADATA_DRIVEN_CONSOLIDATE: "keep"}).questions + keep_ids = {q.question_id for q in keep_pass} + assert QUESTION_METADATA_DRIVEN_ACCESS not in keep_ids + + consolidate_pass = gather_questions( + pipeline, answers={QUESTION_METADATA_DRIVEN_CONSOLIDATE: "consolidate"} + ).questions + consolidate_ids = {q.question_id for q in consolidate_pass} + assert QUESTION_METADATA_DRIVEN_ACCESS in consolidate_ids + assert QUESTION_METADATA_DRIVEN_SIZE in consolidate_ids + assert QUESTION_METADATA_DRIVEN_LOOKUP_TOOL not in consolidate_ids + + def test_metadata_driven_lookup_tool_question_gated_on_access(self): + pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) + answers = { + QUESTION_METADATA_DRIVEN_CONSOLIDATE: "consolidate", + QUESTION_METADATA_DRIVEN_ACCESS: "yes", + } + pending = gather_questions(pipeline, answers=answers).questions + ids = {q.question_id for q in pending} + assert QUESTION_METADATA_DRIVEN_LOOKUP_TOOL in ids + + def test_modifier_consolidates_metadata_driven_when_size_is_small(self): + pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) + prefs = TranslationPreferences( + metadata_driven_consolidate="consolidate", + metadata_driven_access="yes", + metadata_driven_size="small", + ) + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].consolidate_metadata_driven is True + + def test_modifier_does_not_consolidate_when_size_is_large(self): + pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) + prefs = TranslationPreferences( + metadata_driven_consolidate="consolidate", + metadata_driven_access="yes", + metadata_driven_size="large", + ) + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].consolidate_metadata_driven is False + + def test_modifier_does_not_consolidate_when_access_is_no(self): + pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) + prefs = TranslationPreferences( + metadata_driven_consolidate="consolidate", + metadata_driven_access="no", + metadata_driven_size="small", + ) + modified = apply_preferences(pipeline, prefs) + assert modified.tasks[0].consolidate_metadata_driven is False + + def test_lakeflow_connector_type_question_suppressed_when_only_query_copies(self): + pipeline = Pipeline(name="p", tasks=[_query_delta_copy("copy_q")]) + ids = {q.question_id for q in gather_questions(pipeline).questions} + assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids + assert QUESTION_LAKEFLOW_CONNECTOR_TYPE not in ids + + def test_lakeflow_connector_type_question_suppressed_per_copy_eligibility(self): + """Per-Copy eligibility determines connector type with no overlap. + + Table-based reads can only use CDC (no cursor column) and queries + with a cursor can only use query-based, so the modifier picks the + eligible connector per Copy and the prompt is suppressed. + """ + pipeline = Pipeline(name="p", tasks=[_delta_copy("copy_a")]) + ids = {q.question_id for q in gather_questions(pipeline).questions} + assert QUESTION_LAKEFLOW_CONNECTOR_TYPE not in ids + + def test_query_copy_routes_to_query_based_connector_regardless_of_preference(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[_query_delta_copy("copy_q")]) + prefs = TranslationPreferences( + use_lakeflow_connectors="lakeflow_connect", + lakeflow_connector_type="cdc", + ) + stamped = apply_preferences(pipeline, prefs) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_q_lfc.yml").read_text()) + objects = resource["resources"]["pipelines"]["copy_q_lfc"]["ingestion_definition"]["objects"] + assert "table_configuration" in objects[0] + table_config = objects[0]["table_configuration"] + qbc = table_config["query_based_connector_config"] + assert qbc["cursor"] == "updated_at" + assert qbc["include_columns"] == ["id", "name"] + assert table_config["source_table"] == "raw.events" + assert "table" not in objects[0] + + def test_table_copy_uses_cdc_connector_by_default(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) + prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") + stamped = apply_preferences(pipeline, prefs) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_a_lfc.yml").read_text()) + objects = resource["resources"]["pipelines"]["copy_a_lfc"]["ingestion_definition"]["objects"] + assert "table" in objects[0] + assert "table_configuration" not in objects[0] + + def test_table_copy_with_query_based_preference_routes_to_cdc(self, tmp_path: Path): + """LFC query-based requires a cursor column. Table-based Copies have none. + + Per the Lakeflow Connect query-based-overview docs, the connector + requires a cursor column to drive incremental ingestion. When + the user prefers query_based but the Copy is table-based (no + query, no cursor candidate), the modifier honours the + eligibility rules over the preference and routes to CDC. + """ + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) + prefs = TranslationPreferences( + use_lakeflow_connectors="lakeflow_connect", + lakeflow_connector_type="query_based", + ) + stamped = apply_preferences(pipeline, prefs) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_a_lfc.yml").read_text()) + objects = resource["resources"]["pipelines"]["copy_a_lfc"]["ingestion_definition"]["objects"] + assert "table" in objects[0] + assert "table_configuration" not in objects[0] + + def test_consolidated_metadata_driven_motif_emits_single_pipeline(self, tmp_path: Path): + import dataclasses + + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + motif = _metadata_driven_motif() + pipeline = Pipeline(name="job", tasks=[motif]) + prefs = TranslationPreferences( + metadata_driven_consolidate="consolidate", + metadata_driven_access="yes", + metadata_driven_size="medium", + ) + stamped = apply_preferences(pipeline, prefs) + consolidated_motif = dataclasses.replace( + stamped.tasks[0], + lookup_values=[ + {"source_schema": "dbo", "source_table": "orders"}, + {"source_schema": "dbo", "source_table": "customers"}, + ], + ) + stamped = dataclasses.replace(stamped, tasks=[consolidated_motif]) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + resource_path = tmp_path / "resources" / "pipelines" / "motif_metadata_driven_bulk_copy_consolidated.yml" + assert resource_path.exists() + resource = yaml.safe_load(resource_path.read_text()) + pipeline_def = resource["resources"]["pipelines"]["motif_metadata_driven_bulk_copy_consolidated"] + objects = pipeline_def["ingestion_definition"]["objects"] + assert len(objects) == 2 + assert objects[0]["table"]["source_table"] == "orders" + assert objects[1]["table"]["source_table"] == "customers" + + def test_table_based_copy_with_query_based_preference_falls_back_to_cdc(self, tmp_path: Path): + """Table-based reads have no cursor column, so query-based isn't eligible. + + When the user prefers query_based but the only eligible LFC connector + for a table-based Copy is CDC, the modifier routes the Copy to CDC + rather than emitting an unsupported query-based config. + """ + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + copy = CopyActivity( + **_make_base("copy_customers"), + source_type="AzureSqlSource", + sink_type="DeltaSink", + sink_format="delta", + sink_properties={"table": "customers", "schema": "bronze"}, + source_properties={ + "source_schema": "dbo", + "source_table": "customers", + "linked_service_name": "LS_AzureSqlDb", + "connection": {"host": "flowx-test-sql.database.windows.net", "port": 1433}, + }, + ) + prefs = TranslationPreferences( + use_lakeflow_connectors="lakeflow_connect", + lakeflow_connector_type="query_based", + ) + stamped = apply_preferences(Pipeline(name="job", tasks=[copy]), prefs) + write_bundle(prepare_workflow(stamped), tmp_path) + resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_customers_lfc.yml").read_text()) + obj = resource["resources"]["pipelines"]["copy_customers_lfc"]["ingestion_definition"]["objects"][0] + assert "table" in obj + assert obj["table"]["destination_table"] == "customers" + + def test_lakeflow_connect_uses_resolved_host_from_linked_service(self, tmp_path: Path): + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + copy = CopyActivity( + **_make_base("copy_a"), + source_type="AzureSqlSource", + sink_type="DeltaSink", + sink_format="delta", + sink_properties={"table": "orders"}, + source_properties={ + "linked_service_name": "LS_AzureSqlDb", + "connection": {"host": "flowx-test-sql.database.windows.net", "port": 1433}, + }, + ) + prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") + stamped = apply_preferences(Pipeline(name="job", tasks=[copy]), prefs) + write_bundle(prepare_workflow(stamped), tmp_path) + body = (tmp_path / "src" / "setup" / "create_connections.py").read_text() + assert "flowx-test-sql.database.windows.net" in body + assert "1433" in body + assert "PLACEHOLDER_HOST" not in body + + def test_lakeflow_connect_dedupes_connection_across_copies(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + shared_source = { + "linked_service_name": "LS_AzureSqlDb", + "connection": {"host": "host.example.com", "port": 1433}, + "source_schema": "dbo", + } + copy_a = CopyActivity( + **_make_base("copy_a"), + source_type="AzureSqlSource", + sink_type="DeltaSink", + sink_format="delta", + sink_properties={"table": "customers"}, + source_properties={**shared_source, "source_table": "customers"}, + ) + copy_b = CopyActivity( + **_make_base("copy_b"), + source_type="AzureSqlSource", + sink_type="DeltaSink", + sink_format="delta", + sink_properties={"table": "orders"}, + source_properties={**shared_source, "source_table": "orders"}, + ) + prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") + stamped = apply_preferences(Pipeline(name="job", tasks=[copy_a, copy_b]), prefs) + write_bundle(prepare_workflow(stamped), tmp_path) + body = (tmp_path / "src" / "setup" / "create_connections.py").read_text() + assert body.count("CREATE CONNECTION IF NOT EXISTS") == 1 + assert body.count("orchestra_LS_AzureSqlDb_connection") >= 1 + for pipeline_file in ("copy_a_lfc.yml", "copy_b_lfc.yml"): + resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / pipeline_file).read_text()) + key = pipeline_file.replace(".yml", "") + assert resource["resources"]["pipelines"][key]["ingestion_definition"]["connection_name"] == ( + "orchestra_LS_AzureSqlDb_connection" + ) + + def test_lakeflow_connect_emits_connection_setup_notebook(self, tmp_path: Path): + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) + stamped = apply_preferences(pipeline, TranslationPreferences(use_lakeflow_connectors="lakeflow_connect")) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + setup_notebook = tmp_path / "src" / "setup" / "create_connections.py" + assert setup_notebook.exists() + body = setup_notebook.read_text() + assert "orchestra_copy_a_connection" in body + assert "SQLSERVER" in body + + def test_serverless_existing_notebook_skips_default_cluster_bind(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline( + name="job", + tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/existing")], + ) + stamped = apply_preferences(pipeline, TranslationPreferences(databricks_task_compute="serverless")) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) + task = job_yml["resources"]["jobs"]["job"]["tasks"][0] + assert "job_cluster_key" not in task + + def test_existing_default_binds_to_default_cluster(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline( + name="job", + tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/existing")], + ) + stamped = apply_preferences(pipeline, TranslationPreferences()) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path) + job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) + task = job_yml["resources"]["jobs"]["job"]["tasks"][0] + assert task["job_cluster_key"] == "default_cluster" diff --git a/tests/unit/test_query_analysis.py b/tests/unit/test_query_analysis.py new file mode 100644 index 0000000..90d0027 --- /dev/null +++ b/tests/unit/test_query_analysis.py @@ -0,0 +1,174 @@ +"""Tests for the Lakeflow Connect query analyzer.""" + +from __future__ import annotations + +import pytest + +from flowx.translator.query_analysis import QueryAnalysis, analyze_copy_query, dialect_for_source_type + + +class TestParseabilityRejections: + @pytest.mark.parametrize( + "query, reason_fragment", + [ + ("SELECT * FROM dbo.orders JOIN dbo.customers ON orders.cid = customers.id", "JOIN"), + ("SELECT * FROM dbo.orders INNER JOIN dbo.customers", "JOIN"), + ("SELECT customer_id, COUNT(*) FROM dbo.orders GROUP BY customer_id", "aggregate"), + ("SELECT customer_id, COUNT(*) FROM dbo.orders GROUP BY customer_id", "GROUP BY"), + ("SELECT * FROM dbo.orders UNION SELECT * FROM dbo.orders_archive", "UNION"), + ("SELECT MAX(updated_at) FROM dbo.orders", "aggregate"), + ("SELECT id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY id) FROM dbo.orders", "window"), + ("SELECT DISTINCT customer_id FROM dbo.orders", "DISTINCT"), + ("SELECT * FROM dbo.orders WHERE id IN (SELECT id FROM dbo.active_customers)", "subquery"), + ], + ) + def test_disqualifying_constructs(self, query: str, reason_fragment: str): + analysis = analyze_copy_query(query) + assert analysis.parseable is False + assert any(reason_fragment.lower() in reason.lower() for reason in analysis.rejection_reasons) + + def test_empty_query_rejected(self): + analysis = analyze_copy_query("") + assert analysis.parseable is False + assert analysis.rejection_reasons == ["empty query"] + + def test_none_query_rejected(self): + analysis = analyze_copy_query(None) + assert analysis.parseable is False + + def test_select_with_expression_rejected(self): + analysis = analyze_copy_query("SELECT id, UPPER(name) AS upper_name FROM dbo.users") + assert analysis.parseable is False + assert any("expression" in reason.lower() or "alias" in reason.lower() for reason in analysis.rejection_reasons) + + +class TestCursorAndRowFilter: + def test_range_predicate_yields_cursor(self): + analysis = analyze_copy_query("SELECT * FROM dbo.orders WHERE order_date >= '2024-01-01'") + assert analysis.parseable is True + assert analysis.cursor_column == "order_date" + assert analysis.row_filter is None + + def test_between_predicate_yields_cursor(self): + analysis = analyze_copy_query("SELECT * FROM dbo.orders WHERE order_date BETWEEN '2024-01-01' AND '2024-02-01'") + assert analysis.parseable is True + assert analysis.cursor_column == "order_date" + + def test_range_plus_static_filter_extracts_row_filter(self): + analysis = analyze_copy_query( + "SELECT * FROM dbo.orders WHERE order_date >= '2024-01-01' AND active = 1 AND region = 'US'" + ) + assert analysis.parseable is True + assert analysis.cursor_column == "order_date" + assert analysis.row_filter == "active = 1 AND region = 'US'" + + def test_static_filter_only_has_no_cursor_column(self): + analysis = analyze_copy_query("SELECT * FROM dbo.orders WHERE active = 1") + assert analysis.parseable is True + assert analysis.cursor_column is None + assert analysis.row_filter == "active = 1" + + def test_no_where_has_no_cursor_no_filter(self): + analysis = analyze_copy_query("SELECT id, name FROM dbo.users") + assert analysis.parseable is True + assert analysis.cursor_column is None + assert analysis.row_filter is None + + +class TestIncludeColumns: + def test_select_star_omits_include_columns(self): + analysis = analyze_copy_query("SELECT * FROM dbo.orders") + assert analysis.include_columns is None + + def test_explicit_columns_populated(self): + analysis = analyze_copy_query("SELECT order_id, customer_id, order_date, amount FROM dbo.orders") + assert analysis.include_columns == ["order_id", "customer_id", "order_date", "amount"] + + +class TestRealisticAdfQuery: + def test_orders_recent_query_with_dateadd(self): + query = ( + "SELECT order_id, customer_id, order_date, amount FROM dbo.orders " + "WHERE order_date >= DATEADD(day, -7, GETUTCDATE())" + ) + analysis = analyze_copy_query(query) + assert analysis.parseable is True + assert analysis.cursor_column == "order_date" + assert analysis.include_columns == ["order_id", "customer_id", "order_date", "amount"] + assert analysis.row_filter is None + + def test_watermark_incremental_query_has_cursor_and_filter(self): + query = "SELECT * FROM dbo.orders WHERE updated_at > '2024-01-01' AND tenant_id = 42" + analysis = analyze_copy_query(query) + assert analysis.parseable is True + assert analysis.cursor_column == "updated_at" + assert analysis.row_filter == "tenant_id = 42" + + +class TestReturnsAnalysisInstance: + def test_returns_query_analysis_instance(self): + analysis = analyze_copy_query("SELECT * FROM dbo.orders WHERE id > 100") + assert isinstance(analysis, QueryAnalysis) + + +class TestCrossDialect: + @pytest.mark.parametrize("dialect", ["tsql", "mysql", "postgres", "oracle", "snowflake", None]) + def test_range_predicate_parses_across_dialects(self, dialect: str | None): + analysis = analyze_copy_query( + "SELECT order_id, customer_id FROM orders WHERE order_date >= '2024-01-01'", + dialect=dialect, + ) + assert analysis.parseable is True + assert analysis.cursor_column == "order_date" + assert analysis.include_columns == ["order_id", "customer_id"] + + def test_mysql_backtick_identifiers(self): + analysis = analyze_copy_query( + "SELECT `order_id`, `amount` FROM `orders` WHERE `order_date` > '2024-01-01'", + dialect="mysql", + ) + assert analysis.parseable is True + assert analysis.cursor_column == "order_date" + assert analysis.include_columns == ["order_id", "amount"] + + def test_postgres_schema_qualified_table_still_parses(self): + analysis = analyze_copy_query( + "SELECT * FROM public.orders WHERE order_date >= '2024-01-01'", + dialect="postgres", + ) + assert analysis.parseable is True + assert analysis.cursor_column == "order_date" + + +class TestQualifiedColumnRejected: + def test_table_qualified_select_column_is_rejected(self): + analysis = analyze_copy_query( + "SELECT o.order_id, o.amount FROM dbo.orders o WHERE o.order_date >= '2024-01-01'" + ) + assert analysis.parseable is False + assert any("qualified" in reason.lower() for reason in analysis.rejection_reasons) + + +class TestDialectMapping: + @pytest.mark.parametrize( + "source_type, expected_dialect", + [ + ("AzureSqlSource", "tsql"), + ("SqlServerSource", "tsql"), + ("AzureSqlMISource", "tsql"), + ("MySqlSource", "mysql"), + ("AzureMySqlSource", "mysql"), + ("PostgreSqlSource", "postgres"), + ("PostgreSqlV2Source", "postgres"), + ("AzurePostgreSqlSource", "postgres"), + ("OracleSource", "oracle"), + ("SnowflakeSource", "snowflake"), + ("Teradata", "teradata"), + ], + ) + def test_known_adf_sources_map_to_dialect(self, source_type: str, expected_dialect: str): + assert dialect_for_source_type(source_type) == expected_dialect + + @pytest.mark.parametrize("source_type", [None, "", "UnknownSource", "AzureBlobStorage", "Parquet"]) + def test_unknown_sources_return_none(self, source_type: str | None): + assert dialect_for_source_type(source_type) is None diff --git a/uv.lock b/uv.lock index d78a867..6104941 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,154 @@ version = 1 revision = 3 requires-python = ">=3.12" +[[package]] +name = "argcomplete" +version = "3.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/61/0b9ae6399dd4a58d8c1b1dc5a27d6f2808023d0b5dd3104bb99f45a33ff6/argcomplete-3.6.3.tar.gz", hash = "sha256:62e8ed4fd6a45864acc8235409461b72c9a28ee785a2011cc5eb78318786c89c", size = 73754, upload-time = "2025-10-20T03:33:34.741Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/f5/9373290775639cb67a2fce7f629a1c240dce9f12fe927bc32b2736e16dfc/argcomplete-3.6.3-py3-none-any.whl", hash = "sha256:f5007b3a600ccac5d25bbce33089211dfd49eab4a7718da3f10e3082525a92ce", size = 43846, upload-time = "2025-10-20T03:33:33.021Z" }, +] + +[[package]] +name = "certifi" +version = "2026.5.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -95,6 +243,95 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, ] +[[package]] +name = "cryptography" +version = "48.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", size = 832984, upload-time = "2026-05-04T22:59:38.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", size = 8001587, upload-time = "2026-05-04T22:57:36.803Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", size = 4690433, upload-time = "2026-05-04T22:57:40.373Z" }, + { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", size = 4710620, upload-time = "2026-05-04T22:57:42.935Z" }, + { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", size = 4696283, upload-time = "2026-05-04T22:57:45.294Z" }, + { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", size = 5296573, upload-time = "2026-05-04T22:57:47.933Z" }, + { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", size = 4743677, upload-time = "2026-05-04T22:57:50.067Z" }, + { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", size = 4330808, upload-time = "2026-05-04T22:57:52.301Z" }, + { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", size = 4695941, upload-time = "2026-05-04T22:57:54.603Z" }, + { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", size = 5252579, upload-time = "2026-05-04T22:57:57.207Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", size = 4743326, upload-time = "2026-05-04T22:57:59.535Z" }, + { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", size = 4826672, upload-time = "2026-05-04T22:58:01.996Z" }, + { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", size = 4972574, upload-time = "2026-05-04T22:58:03.968Z" }, + { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", size = 3294868, upload-time = "2026-05-04T22:58:06.467Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", size = 3817107, upload-time = "2026-05-04T22:58:08.845Z" }, + { url = "https://files.pythonhosted.org/packages/6b/84/70e3feea9feea87fd7cbe77efb2712ae1e3e6edf10749dc6e95f4e60e455/cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec", size = 7986556, upload-time = "2026-05-04T22:58:11.172Z" }, + { url = "https://files.pythonhosted.org/packages/89/6e/18e07a618bb5442ba10cf4df16e99c071365528aa570dfcb8c02e25a303b/cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18", size = 4684776, upload-time = "2026-05-04T22:58:13.712Z" }, + { url = "https://files.pythonhosted.org/packages/be/6a/4ea3b4c6c6759794d5ee2103c304a5076dc4b19ae1f9fe47dba439e159e9/cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20", size = 4698121, upload-time = "2026-05-04T22:58:16.448Z" }, + { url = "https://files.pythonhosted.org/packages/2f/59/6ff6ad6cae03bb887da2a5860b2c9805f8dac969ef01ce563336c49bd1d1/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff", size = 4690042, upload-time = "2026-05-04T22:58:18.544Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b4/fc334ed8cfd705aca282fe4d8f5ae64a8e0f74932e9feecb344610cf6e4d/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c", size = 5282526, upload-time = "2026-05-04T22:58:20.75Z" }, + { url = "https://files.pythonhosted.org/packages/11/08/9f8c5386cc4cd90d8255c7cdd0f5baf459a08502a09de30dc51f553d38dc/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db", size = 4733116, upload-time = "2026-05-04T22:58:23.627Z" }, + { url = "https://files.pythonhosted.org/packages/b8/77/99307d7574045699f8805aa500fa0fb83422d115b5400a064ddd306d7750/cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741", size = 4316030, upload-time = "2026-05-04T22:58:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/fd/36/a608b98337af3cb2aff4818e406649d30572b7031918b04c87d979495348/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166", size = 4689640, upload-time = "2026-05-04T22:58:27.747Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a6/825010a291b4438aecc1f568bc428189fc1175515223632477c07dc0a6df/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336", size = 5237657, upload-time = "2026-05-04T22:58:29.848Z" }, + { url = "https://files.pythonhosted.org/packages/b9/09/4e76a09b4caa29aad535ddc806f5d4c5d01885bd978bd984fbc6ca032cae/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057", size = 4732362, upload-time = "2026-05-04T22:58:32.009Z" }, + { url = "https://files.pythonhosted.org/packages/18/78/444fa04a77d0cb95f417dda20d450e13c56ba8e5220fc892a1658f44f882/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae", size = 4819580, upload-time = "2026-05-04T22:58:34.254Z" }, + { url = "https://files.pythonhosted.org/packages/38/85/ea67067c70a1fd4be2c63d35eeed82658023021affccc7b17705f8527dd2/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c", size = 4963283, upload-time = "2026-05-04T22:58:36.376Z" }, + { url = "https://files.pythonhosted.org/packages/75/54/cc6d0f3deac3e81c7f847e8a189a12b6cdd65059b43dad25d4316abd849a/cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f", size = 3270954, upload-time = "2026-05-04T22:58:38.791Z" }, + { url = "https://files.pythonhosted.org/packages/49/67/cc947e288c0758a4e5473d1dcb743037ab7785541265a969240b8885441a/cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12", size = 3797313, upload-time = "2026-05-04T22:58:40.746Z" }, + { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", size = 7983482, upload-time = "2026-05-04T22:58:43.769Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", size = 4683266, upload-time = "2026-05-04T22:58:46.064Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", size = 4696228, upload-time = "2026-05-04T22:58:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", size = 4689097, upload-time = "2026-05-04T22:58:50.9Z" }, + { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", size = 5283582, upload-time = "2026-05-04T22:58:53.017Z" }, + { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", size = 4730479, upload-time = "2026-05-04T22:58:55.611Z" }, + { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", size = 4326481, upload-time = "2026-05-04T22:58:57.607Z" }, + { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", size = 4688713, upload-time = "2026-05-04T22:59:00.077Z" }, + { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", size = 5238165, upload-time = "2026-05-04T22:59:02.317Z" }, + { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", size = 4729947, upload-time = "2026-05-04T22:59:05.255Z" }, + { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", size = 4822059, upload-time = "2026-05-04T22:59:07.802Z" }, + { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", size = 4960575, upload-time = "2026-05-04T22:59:09.851Z" }, + { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", size = 3279117, upload-time = "2026-05-04T22:59:12.019Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", size = 3792100, upload-time = "2026-05-04T22:59:14.884Z" }, +] + +[[package]] +name = "databricks-sdk" +version = "0.110.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/0f/488d61ece084f70a6d4d0ab8b5e38b0902e0b9029d0b72cde99e3f2c6b4a/databricks_sdk-0.110.0.tar.gz", hash = "sha256:b62d806982b37f8160f700d657c37b3bd586c649eb5c8c4c1216090d888c5820", size = 945261, upload-time = "2026-05-19T09:18:46.23Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/23/7c2a827890ab120ac349847ec17ab5a37eb4e3bf8f1d0989fd9eec0c1e6a/databricks_sdk-0.110.0-py3-none-any.whl", hash = "sha256:8a23db05be7a304bea43b4fa78b437051ed0f3755b19594429c649ee4159b546", size = 892096, upload-time = "2026-05-19T09:18:44.313Z" }, +] + +[[package]] +name = "google-auth" +version = "2.53.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/ad/ff781329bbbdc0974a098d996e89c9e1f7024262f9e3eec442fbb9ad1ac6/google_auth-2.53.0.tar.gz", hash = "sha256:e7e6aa16f6bee7b2b264830fd04f08087a1d5a836df516251a5d15327b246c9c", size = 335844, upload-time = "2026-05-15T20:53:07.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/c9/db44165ba7c581268c6d46017ef63339110378305062830104fc7fa144cb/google_auth-2.53.0-py3-none-any.whl", hash = "sha256:6e7449917c599b35126a99ec268ec6880301f2fea41dce198fe8fd83ff642b68", size = 246071, upload-time = "2026-05-15T20:53:05.609Z" }, +] + +[[package]] +name = "idna" +version = "3.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -221,7 +458,9 @@ name = "flowx" version = "0.2.0" source = { editable = "." } dependencies = [ + { name = "databricks-sdk" }, { name = "pyyaml" }, + { name = "sqlglot" }, ] [package.dev-dependencies] @@ -232,9 +471,16 @@ dev = [ { name = "ruff" }, { name = "types-pyyaml" }, ] +yq = [ + { name = "yq" }, +] [package.metadata] -requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] +requires-dist = [ + { name = "databricks-sdk", specifier = ">=0.40" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=25.0" }, +] [package.metadata.requires-dev] dev = [ @@ -244,6 +490,7 @@ dev = [ { name = "ruff", specifier = ">=0.14.0,<1" }, { name = "types-pyyaml", specifier = ">=6.0.12.20250915,<7" }, ] +yq = [{ name = "yq", specifier = "~=3.4.3" }] [[package]] name = "packaging" @@ -272,6 +519,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + +[[package]] +name = "pyasn1" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -343,6 +635,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + [[package]] name = "ruff" version = "0.15.8" @@ -368,6 +675,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/e2/77be4fff062fa78d9b2a4dea85d14785dac5f1d0c1fb58ed52331f0ebe28/ruff-0.15.8-py3-none-win_arm64.whl", hash = "sha256:cf891fa8e3bb430c0e7fac93851a5978fc99c8fa2c053b57b118972866f8e5f2", size = 11048175, upload-time = "2026-03-26T18:40:01.06Z" }, ] +[[package]] +name = "sqlglot" +version = "30.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/64/89299aefc6ebdf4fc899f5dc14c7fcb7eb9da9290a2b4d615ae7ab884b17/sqlglot-30.8.0.tar.gz", hash = "sha256:1c5f93fb742dd9aaa75eee6bb33a637794a858b9a86375fac23a2dc0f7bc127e", size = 5869750, upload-time = "2026-05-13T09:04:38.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/4e/80705091aaf9c95e125d243f0aa871bc9f3670b4c9d963e6bad3b3dce8ff/sqlglot-30.8.0-py3-none-any.whl", hash = "sha256:af903378c331d5b72277a1b41118f07bc3e50cf4478e2d47eed12c96ee6a22a4", size = 687831, upload-time = "2026-05-13T09:04:36.336Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", size = 161875, upload-time = "2026-05-10T07:38:22.245Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", size = 41328, upload-time = "2026-05-10T07:38:23.517Z" }, +] + [[package]] name = "types-pyyaml" version = "6.0.12.20250915" @@ -385,3 +710,36 @@ sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac8 wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "xmltodict" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz", hash = "sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61", size = 26124, upload-time = "2026-02-22T02:21:22.074Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl", hash = "sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a", size = 13580, upload-time = "2026-02-22T02:21:21.039Z" }, +] + +[[package]] +name = "yq" +version = "3.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argcomplete" }, + { name = "pyyaml" }, + { name = "tomlkit" }, + { name = "xmltodict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/6a/eb9721ed0929d0f55d167c2222d288b529723afbef0a07ed7aa6cca72380/yq-3.4.3.tar.gz", hash = "sha256:ba586a1a6f30cf705b2f92206712df2281cd320280210e7b7b80adcb8f256e3b", size = 33214, upload-time = "2024-04-27T15:39:43.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/ba/d1b21f3e57469030bd6536b91bb28fedd2511d4e68b5a575f2bdb3a3dbb6/yq-3.4.3-py3-none-any.whl", hash = "sha256:547e34bc3caacce83665fd3429bf7c85f8e8b6b9aaee3f953db1ad716ff3434d", size = 18812, upload-time = "2024-04-27T15:39:41.652Z" }, +] From 07e69360c0b6e4d5f4d5f364663066233dae9a80 Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Thu, 28 May 2026 20:41:11 -0400 Subject: [PATCH 07/77] Fix documentation rendering (#9) * Add user-interface for specifying options * Fix docs rendering --- docs/content/docs/options.mdx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/content/docs/options.mdx b/docs/content/docs/options.mdx index dd1fd73..aaa5a9a 100644 --- a/docs/content/docs/options.mdx +++ b/docs/content/docs/options.mdx @@ -3,12 +3,12 @@ title: Translation options description: Control flowx's translation behavior and outputs --- +import { Callout } from 'fumadocs-ui/components/callout'; + Flowx defers some architectural choices to allow users to specify properties of the output jobs. When options are available for controlling translation, the agent session prompts the user for their preferences. -## Available options - -### `copy_activity_paradigm` +## copy_activity_paradigm Controls how Copy Data activities whose sink resolves to a Delta table are translated. @@ -17,7 +17,7 @@ Controls how Copy Data activities whose sink resolves to a Delta table are trans | `notebook` | True | Generates a Notebook task that copies data using PySpark `read` and `write` methods. | | `sdp` | False | Generates a Run Pipeline task that copies data using Lakeflow Spark Declarative Pipelines. | -### `non_databricks_task_compute` +## non_databricks_task_compute Controls compute for non-Databricks tasks (e.g. Lookup, Web, Delete, Wait, Filter, and Set Variable activities). @@ -26,7 +26,7 @@ Controls compute for non-Databricks tasks (e.g. Lookup, Web, Delete, Wait, Filte | `serverless` | True | Translated tasks run on serverless compute. | | `classic` | False | Most tasks run on a single-node classic compute cluster; Translated Copy Data tasks run on a fixed-size multi-node cluster. Clusters referenced by tasks are included in the output bundle resources' `job_clusters`. | -### `use_lakeflow_connectors` +## use_lakeflow_connectors Controls whether eligible Copy Data activities are replaced with a managed Lakeflow Connect pipeline. @@ -35,7 +35,7 @@ Controls whether eligible Copy Data activities are replaced with a managed Lakef | `existing` | True | Translates the Copy Data activity as a Notebook or Run Pipeline task using PySpark `read` and `write` methods. | | `lakeflow_connect` | False | Translates the Copy Data activity as a Run Pipeline task that triggers a Lakeflow Connect managed ingestion pipeline. | -### `lakeflow_connector_type` +## lakeflow_connector_type Controls how Lakeflow Connect reads from a database when copying data. @@ -49,7 +49,7 @@ Copy activities that carry an explicit SQL query (`sqlReaderQuery`, `query`, or must use a [query-based connector](https://docs.databricks.com/aws/en/ingestion/lakeflow-connect/query-based-overview). -### `databricks_task_compute` +## databricks_task_compute Controls the compute used to run Databricks Notebook and Spark Python tasks in the translated job. @@ -58,7 +58,7 @@ Controls the compute used to run Databricks Notebook and Spark Python tasks in t | `existing` | True | Uses the source pipeline's compute definition (e.g. a job cluster). Preserves init scripts, DBR-version, and other configuration. | | `serverless` | False | Drops the source pipeline's cluster definition; The translated tasks run on serverless compute. | -### `metadata_driven_consolidate` +## metadata_driven_consolidate Configures flowx to detect and consolidate ingestion pipelines that read from configuration and run parameterized data copying. Allows users to create a consolidated pipeline. From 3c9cb71f274c6db55a3bfb4cd06a3b7768f2fc12 Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Fri, 29 May 2026 12:35:59 -0400 Subject: [PATCH 08/77] Update GitHub actions (#10) --- .github/workflows/skill-eval.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 .github/workflows/skill-eval.yml diff --git a/.github/workflows/skill-eval.yml b/.github/workflows/skill-eval.yml deleted file mode 100644 index c1a93d6..0000000 --- a/.github/workflows/skill-eval.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: skill-eval - -on: - pull_request: - workflow_dispatch: - -jobs: - integration: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Scrub internal proxy URLs from uv.lock - run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock - - run: uv sync --frozen - - # Azure login for live ADF integration tests. - # Requires AZURE_CREDENTIALS secret configured with a service principal - # that has Reader access to the flowx-rg resource group. - # Tests skip gracefully when credentials are not available. - - name: Azure Login - if: ${{ secrets.AZURE_CREDENTIALS != '' }} - uses: azure/login@v2 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - - - run: make integration From 65ec748b0dd1017a5ca1f86848db8e476b9e003b Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Tue, 9 Jun 2026 13:14:24 -0400 Subject: [PATCH 09/77] Refactor expression parsing (#1) --- .build-constraints.txt | 6 +- docs/content/docs/options.mdx | 26 +- skills/migrate/SKILL.md | 13 +- skills/translate/SKILL.md | 3 +- src/orchestra/adapter/__init__.py | 4 +- src/orchestra/adapter/__main__.py | 11 +- src/orchestra/adapter/constants.py | 7 +- src/orchestra/adapter/models.py | 44 +-- src/orchestra/adapter/operations.py | 169 ++++++------ src/orchestra/adapter/session.py | 27 +- src/orchestra/bundler/dab_writer.py | 13 +- src/orchestra/bundler/prereqs_writer.py | 26 +- src/orchestra/models/dab.py | 21 ++ src/orchestra/models/ir.py | 33 ++- src/orchestra/parser/expression_parser.py | 63 ++++- src/orchestra/parser/ir_rewriter.py | 252 ++++++++++++++++++ .../preparer/activity_preparers/notebook.py | 27 +- .../activity_preparers/spark_python.py | 2 + src/orchestra/preparer/workflow_preparer.py | 10 +- .../activity_translators/notebook.py | 25 +- .../activity_translators/spark_python.py | 2 + src/orchestra/translator/engine.py | 98 ++++++- tests/unit/test_adapter.py | 102 ++++--- tests/unit/test_bundler.py | 32 +++ tests/unit/test_expression_parser.py | 46 ++-- tests/unit/test_helpers.py | 9 +- tests/unit/test_ir_rewriter.py | 234 ++++++++++++++++ tests/unit/test_preparers.py | 73 +++++ tests/unit/test_translators.py | 136 ++++++++++ 29 files changed, 1309 insertions(+), 205 deletions(-) create mode 100644 src/orchestra/parser/ir_rewriter.py create mode 100644 tests/unit/test_ir_rewriter.py diff --git a/.build-constraints.txt b/.build-constraints.txt index 48078f6..c6be8d7 100644 --- a/.build-constraints.txt +++ b/.build-constraints.txt @@ -13,7 +13,7 @@ pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 # via hatchling -trove-classifiers==2026.5.20.19 \ - --hash=sha256:6e611993987ca9326968ad70452733dadd31471599d39896045b28970a9bb81e \ - --hash=sha256:7a173916960d0635fcbf610550d2c27bcc9125164d6f397adf46fc1ef6455c7c +trove-classifiers==2026.5.22.10 \ + --hash=sha256:01fe864225726e03efb843827ecabfe319fc4dee8dd66d65b8996cb09be46e2c \ + --hash=sha256:5477e9974e91904fb2cfa4a7581ab6e2f30c2c38d847fd00ed866080748101d5 # via hatchling diff --git a/docs/content/docs/options.mdx b/docs/content/docs/options.mdx index aaa5a9a..741d581 100644 --- a/docs/content/docs/options.mdx +++ b/docs/content/docs/options.mdx @@ -49,14 +49,24 @@ Copy activities that carry an explicit SQL query (`sqlReaderQuery`, `query`, or must use a [query-based connector](https://docs.databricks.com/aws/en/ingestion/lakeflow-connect/query-based-overview). -## databricks_task_compute - -Controls the compute used to run Databricks Notebook and Spark Python tasks in the translated job. - -| Value | Default | Behavior | -|--------------|---------|-----------------------------------------------------------------------------------------------------------------------------------| -| `existing` | True | Uses the source pipeline's compute definition (e.g. a job cluster). Preserves init scripts, DBR-version, and other configuration. | -| `serverless` | False | Drops the source pipeline's cluster definition; The translated tasks run on serverless compute. | +## consolidate_motif:<motif_id> + +For every multi-activity motif the detector matches in a pipeline (`incremental_load_watermark`, +`rest_api_pagination`, `metadata_driven_bulk_copy`, ...), flowx raises a per-motif +question with id `consolidate_motif:` so each detected pattern can be approved +or rejected independently. + +| Value | Default | Behavior | +|---------------|---------|------------------------------------------------------------------------------------------------------------------------------------------| +| `keep` | True | Preserves the activity-by-activity translation; the motif detection result is informational only. | +| `consolidate` | False | Collapses the matched activities into a single `MotifActivity` whose target is the motif's `databricks_replacement` (e.g. `auto_loader`). | + + +Motif detection is heuristic. Defaulting to `keep` means a false-positive match (e.g. classifying +a submit-and-poll Web/Until/SetVariable chain as REST pagination) cannot silently rewrite the +pipeline; the activities continue to translate one-for-one until the user explicitly confirms the +pattern. + ## metadata_driven_consolidate diff --git a/skills/migrate/SKILL.md b/skills/migrate/SKILL.md index 5c3d910..3e2b7d0 100644 --- a/skills/migrate/SKILL.md +++ b/skills/migrate/SKILL.md @@ -170,14 +170,23 @@ Use the stamped report (when produced) as the input to the prepare phase. When inspect emits no questions for any pipeline, skip modify and use the original report. -The four questions the adapter raises: +The questions the adapter raises: | `question_id` | Allowed values | Default | |---|---|---| | `copy_activity_paradigm` | `notebook`, `sdp` | `notebook` | | `non_databricks_task_compute` | `serverless`, `classic` | `serverless` | | `use_lakeflow_connectors` | `existing`, `lakeflow_connect` | `existing` | -| `databricks_task_compute` | `existing`, `serverless` | `existing` | +| `consolidate_motif:` | `keep`, `consolidate` | `keep` | + +DatabricksNotebook and DatabricksSparkPython tasks always inherit the cluster binding derived from +their source linked service; the serverless replacement option was removed because it silently +discarded init scripts and DBR-version pins from the source pipeline. + +For each multi-activity motif the detector matches (rest_api_pagination, +incremental_load_watermark, metadata_driven_bulk_copy, ...) the adapter emits one +`consolidate_motif:` question. Default is `keep` so motif detection cannot silently +rewrite a pipeline; the user must explicitly opt in to `consolidate` for each pattern. ### Step 6 — Checkpoint: confirm proceed to bundle generation diff --git a/skills/translate/SKILL.md b/skills/translate/SKILL.md index 5d719e8..75acf1b 100644 --- a/skills/translate/SKILL.md +++ b/skills/translate/SKILL.md @@ -243,8 +243,7 @@ answers into a JSON file (`/answers.json`) shaped like: { "copy_activity_paradigm": "sdp", "non_databricks_task_compute": "serverless", - "use_lakeflow_connectors": "lakeflow_connect", - "databricks_task_compute": "existing" + "use_lakeflow_connectors": "lakeflow_connect" } ``` diff --git a/src/orchestra/adapter/__init__.py b/src/orchestra/adapter/__init__.py index f713c38..6add5d5 100644 --- a/src/orchestra/adapter/__init__.py +++ b/src/orchestra/adapter/__init__.py @@ -34,13 +34,13 @@ from flowx.adapter.models import ( DEFAULT_PREFERENCES, CopyActivityParadigm, - DatabricksTaskCompute, LakeflowConnectorType, MetadataDrivenAccess, MetadataDrivenConsolidate, MetadataDrivenLookupTool, MetadataDrivenSize, MigrationInputQuestion, + MotifConsolidate, NonDatabricksTaskCompute, PendingMigrationInputs, PendingQuestions, @@ -68,7 +68,6 @@ __all__ = [ "DEFAULT_PREFERENCES", "CopyActivityParadigm", - "DatabricksTaskCompute", "LakeflowConnectorType", "MetadataDrivenAccess", "MetadataDrivenConsolidate", @@ -76,6 +75,7 @@ "MetadataDrivenSize", "MigrationInputQuestion", "MigrationInputSession", + "MotifConsolidate", "NonDatabricksTaskCompute", "PendingMigrationInputs", "PendingQuestions", diff --git a/src/orchestra/adapter/__main__.py b/src/orchestra/adapter/__main__.py index be5823c..490de59 100644 --- a/src/orchestra/adapter/__main__.py +++ b/src/orchestra/adapter/__main__.py @@ -19,15 +19,16 @@ from pathlib import Path from typing import Any +from flowx.adapter.constants import MOTIF_CONSOLIDATE_QUESTION_PREFIX from flowx.adapter.models import ( DEFAULT_PREFERENCES, CopyActivityParadigm, - DatabricksTaskCompute, LakeflowConnectorType, MetadataDrivenAccess, MetadataDrivenConsolidate, MetadataDrivenLookupTool, MetadataDrivenSize, + MotifConsolidate, NonDatabricksTaskCompute, PendingQuestions, TranslationPreferences, @@ -484,6 +485,10 @@ def _preferences_from_answers(answers: dict[str, str]) -> TranslationPreferences question. """ validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} + motif_consolidations: dict[str, MotifConsolidate] = {} + for qid, value in validated.items(): + if qid.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): + motif_consolidations[qid[len(MOTIF_CONSOLIDATE_QUESTION_PREFIX) :]] = MotifConsolidate(value) return TranslationPreferences( copy_activity_paradigm=CopyActivityParadigm( validated.get("copy_activity_paradigm", DEFAULT_PREFERENCES.copy_activity_paradigm) @@ -494,9 +499,6 @@ def _preferences_from_answers(answers: dict[str, str]) -> TranslationPreferences use_lakeflow_connectors=UseLakeflowConnectors( validated.get("use_lakeflow_connectors", DEFAULT_PREFERENCES.use_lakeflow_connectors) ), - databricks_task_compute=DatabricksTaskCompute( - validated.get("databricks_task_compute", DEFAULT_PREFERENCES.databricks_task_compute) - ), lakeflow_connector_type=LakeflowConnectorType( validated.get("lakeflow_connector_type", DEFAULT_PREFERENCES.lakeflow_connector_type) ), @@ -512,6 +514,7 @@ def _preferences_from_answers(answers: dict[str, str]) -> TranslationPreferences metadata_driven_lookup_tool=MetadataDrivenLookupTool( validated.get("metadata_driven_lookup_tool", DEFAULT_PREFERENCES.metadata_driven_lookup_tool) ), + motif_consolidations=motif_consolidations, ) diff --git a/src/orchestra/adapter/constants.py b/src/orchestra/adapter/constants.py index c74a434..40c3e3c 100644 --- a/src/orchestra/adapter/constants.py +++ b/src/orchestra/adapter/constants.py @@ -14,13 +14,18 @@ QUESTION_COPY_ACTIVITY_PARADIGM: Final[str] = "copy_activity_paradigm" QUESTION_NON_DATABRICKS_TASK_COMPUTE: Final[str] = "non_databricks_task_compute" QUESTION_USE_LAKEFLOW_CONNECTORS: Final[str] = "use_lakeflow_connectors" -QUESTION_DATABRICKS_TASK_COMPUTE: Final[str] = "databricks_task_compute" QUESTION_LAKEFLOW_CONNECTOR_TYPE: Final[str] = "lakeflow_connector_type" QUESTION_METADATA_DRIVEN_CONSOLIDATE: Final[str] = "metadata_driven_consolidate" QUESTION_METADATA_DRIVEN_ACCESS: Final[str] = "metadata_driven_access" QUESTION_METADATA_DRIVEN_SIZE: Final[str] = "metadata_driven_size" QUESTION_METADATA_DRIVEN_LOOKUP_TOOL: Final[str] = "metadata_driven_lookup_tool" +# Per-detected-motif consolidation question_ids carry the motif_id as a suffix +# (e.g. ``consolidate_motif:rest_api_pagination``) so each detected motif gets +# its own question. Validation strips the prefix and validates the answer +# against the :class:`MotifConsolidate` enum. +MOTIF_CONSOLIDATE_QUESTION_PREFIX: Final[str] = "consolidate_motif:" + METADATA_DRIVEN_MOTIF_ID: Final[str] = "metadata_driven_bulk_copy" PHASE_INGEST: Final[str] = "ingest" diff --git a/src/orchestra/adapter/models.py b/src/orchestra/adapter/models.py index 3aff855..70f5abc 100644 --- a/src/orchestra/adapter/models.py +++ b/src/orchestra/adapter/models.py @@ -29,13 +29,6 @@ class UseLakeflowConnectors(StrEnum): EXISTING = "existing" -class DatabricksTaskCompute(StrEnum): - """Compute mode used for ADF DatabricksNotebook and DatabricksSparkPython tasks.""" - - SERVERLESS = "serverless" - EXISTING = "existing" - - class LakeflowConnectorType(StrEnum): """Lakeflow Connect connector flavour for an eligible Copy ingestion. @@ -85,12 +78,24 @@ class MetadataDrivenLookupTool(StrEnum): NONE = "none" +class MotifConsolidate(StrEnum): + """Whether to collapse a detected motif into a single :class:`MotifActivity`. + + Default for every detected motif is :data:`KEEP` -- preserving the + underlying activity-by-activity translation -- so motif detection + can never silently rewrite a pipeline without an explicit user + opt-in. + """ + + KEEP = "keep" + CONSOLIDATE = "consolidate" + + FIELD_TO_ENUM: Final[MappingProxyType[str, type[StrEnum]]] = MappingProxyType( { "copy_activity_paradigm": CopyActivityParadigm, "non_databricks_task_compute": NonDatabricksTaskCompute, "use_lakeflow_connectors": UseLakeflowConnectors, - "databricks_task_compute": DatabricksTaskCompute, "lakeflow_connector_type": LakeflowConnectorType, "metadata_driven_consolidate": MetadataDrivenConsolidate, "metadata_driven_access": MetadataDrivenAccess, @@ -113,22 +118,26 @@ class TranslationPreferences: non_databricks_task_compute: Compute mode for non-Databricks tasks. use_lakeflow_connectors: Whether eligible database-source Copy patterns are migrated to managed Lakeflow Connect pipelines. - databricks_task_compute: Compute mode for ADF DatabricksNotebook and - DatabricksSparkPython tasks. per_task: Optional per-activity overrides keyed by task_key. Each - value is a partial mapping of the four fields above; only the + value is a partial mapping of the fields above; only the keys present win over the pipeline-wide defaults. + + ADF DatabricksNotebook and DatabricksSparkPython tasks always keep + the cluster binding derived from the source linked service -- the + serverless replacement option was removed because it silently + discarded init scripts and DBR-version constraints that the source + pipeline relied on. """ copy_activity_paradigm: CopyActivityParadigm = CopyActivityParadigm.NOTEBOOK non_databricks_task_compute: NonDatabricksTaskCompute = NonDatabricksTaskCompute.SERVERLESS use_lakeflow_connectors: UseLakeflowConnectors = UseLakeflowConnectors.EXISTING - databricks_task_compute: DatabricksTaskCompute = DatabricksTaskCompute.EXISTING lakeflow_connector_type: LakeflowConnectorType = LakeflowConnectorType.CDC metadata_driven_consolidate: MetadataDrivenConsolidate = MetadataDrivenConsolidate.KEEP metadata_driven_access: MetadataDrivenAccess = MetadataDrivenAccess.NO metadata_driven_size: MetadataDrivenSize = MetadataDrivenSize.LARGE metadata_driven_lookup_tool: MetadataDrivenLookupTool = MetadataDrivenLookupTool.NONE + motif_consolidations: dict[str, MotifConsolidate] = field(default_factory=dict) per_task: dict[str, dict[str, str]] = field(default_factory=dict) def __post_init__(self) -> None: @@ -142,6 +151,13 @@ def __post_init__(self) -> None: value = getattr(self, field_name) if not isinstance(value, enum_cls): object.__setattr__(self, field_name, enum_cls(value)) + # motif_consolidations is keyed by dynamic motif_id rather than a + # fixed field name, so it is not in FIELD_TO_ENUM. Coerce its + # values to MotifConsolidate members here. + coerced: dict[str, MotifConsolidate] = {} + for motif_id, choice in self.motif_consolidations.items(): + coerced[motif_id] = choice if isinstance(choice, MotifConsolidate) else MotifConsolidate(choice) + object.__setattr__(self, "motif_consolidations", coerced) def effective_for(self, task_key: str) -> TranslationPreferences: """Returns a preferences view where per-task overrides for *task_key* win. @@ -167,9 +183,6 @@ def effective_for(self, task_key: str) -> TranslationPreferences: use_lakeflow_connectors=UseLakeflowConnectors( override.get("use_lakeflow_connectors", self.use_lakeflow_connectors) ), - databricks_task_compute=DatabricksTaskCompute( - override.get("databricks_task_compute", self.databricks_task_compute) - ), lakeflow_connector_type=LakeflowConnectorType( override.get("lakeflow_connector_type", self.lakeflow_connector_type) ), @@ -183,6 +196,7 @@ def effective_for(self, task_key: str) -> TranslationPreferences: metadata_driven_lookup_tool=MetadataDrivenLookupTool( override.get("metadata_driven_lookup_tool", self.metadata_driven_lookup_tool) ), + motif_consolidations=dict(self.motif_consolidations), per_task=self.per_task, ) diff --git a/src/orchestra/adapter/operations.py b/src/orchestra/adapter/operations.py index cfc526c..fe21cbc 100644 --- a/src/orchestra/adapter/operations.py +++ b/src/orchestra/adapter/operations.py @@ -23,8 +23,8 @@ LAKEFLOW_CONNECT_REPLACEMENT, LAKEFLOW_CONNECTOR_TYPE_QUERY_BASED, METADATA_DRIVEN_MOTIF_ID, + MOTIF_CONSOLIDATE_QUESTION_PREFIX, QUESTION_COPY_ACTIVITY_PARADIGM, - QUESTION_DATABRICKS_TASK_COMPUTE, QUESTION_METADATA_DRIVEN_ACCESS, QUESTION_METADATA_DRIVEN_CONSOLIDATE, QUESTION_METADATA_DRIVEN_LOOKUP_TOOL, @@ -35,12 +35,12 @@ from flowx.adapter.models import ( FIELD_TO_ENUM, CopyActivityParadigm, - DatabricksTaskCompute, LakeflowConnectorType, MetadataDrivenAccess, MetadataDrivenConsolidate, MetadataDrivenLookupTool, MetadataDrivenSize, + MotifConsolidate, NonDatabricksTaskCompute, PendingQuestions, QuestionOption, @@ -62,9 +62,7 @@ ForEachActivity, IfConditionActivity, MotifActivity, - NotebookActivity, Pipeline, - SparkPythonActivity, SwitchActivity, SwitchCase, ) @@ -75,12 +73,15 @@ def enum_for(question_id: str) -> type[StrEnum] | None: """Returns the enum class backing a preference field. Args: - question_id: Field name (e.g. ``"copy_activity_paradigm"``). + question_id: Field name (e.g. ``"copy_activity_paradigm"``) or + per-motif id (e.g. ``"consolidate_motif:rest_api_pagination"``). Returns: The :class:`StrEnum` subclass that defines the allowed values, or - ``None`` when the field is unknown. + ``None`` when the question_id is unknown. """ + if question_id.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): + return MotifConsolidate return FIELD_TO_ENUM.get(question_id) @@ -241,7 +242,6 @@ def gather_questions( _build_lakeflow_connector_type_question, _build_copy_activity_paradigm_question, _build_non_databricks_task_compute_question, - _build_databricks_task_compute_question, _build_metadata_driven_consolidate_question, _build_metadata_driven_access_question, _build_metadata_driven_size_question, @@ -255,6 +255,14 @@ def gather_questions( and question.question_id not in answer_map and _conditions_met(question.conditions, answer_map) ] + # Per-motif "consolidate?" questions: one per detected motif. Each + # gets its own question_id ``consolidate_motif:`` so the + # adapter can solicit and validate them independently. Default is + # ``keep`` -- nothing is collapsed without an explicit yes. + for motif_question in _build_motif_consolidation_questions(motif_list): + if motif_question.question_id in answer_map: + continue + pending.append(motif_question) return PendingQuestions(pipeline_name=pipeline.name, questions=pending) @@ -468,52 +476,6 @@ def _build_lakeflow_connector_type_question( return None -def _build_databricks_task_compute_question( - pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None -) -> TranslationQuestion | None: - """Builds the serverless-vs-existing question for ADF Databricks-* tasks. - - Args: - pipeline: Translated pipeline IR. - motifs: Detected motifs (unused; accepted for builder uniformity). - - Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when - no Databricks notebook or Python task is present. - """ - affected = tuple( - activity.task_key - for activity in walk_activities(pipeline.tasks) - if isinstance(activity, (NotebookActivity, SparkPythonActivity)) - ) - if not affected: - return None - return TranslationQuestion( - question_id=QUESTION_DATABRICKS_TASK_COMPUTE, - prompt="Migrate existing Databricks notebook and Python tasks to serverless?", - rationale=( - "ADF DatabricksNotebook and DatabricksSparkPython tasks bind to a " - "classic cluster derived from the source linked service. Serverless " - "drops that binding; keeping the existing compute preserves init " - "scripts or DBR-specific features." - ), - options=( - QuestionOption( - value=DatabricksTaskCompute.EXISTING.value, - label="Keep linked-service compute", - description="Binds the task to the cluster derived from the ADF linked service.", - ), - QuestionOption( - value=DatabricksTaskCompute.SERVERLESS.value, - label="Serverless", - description="Removes the cluster binding so the task runs on serverless compute.", - ), - ), - affected_task_keys=affected, - default=DatabricksTaskCompute.EXISTING.value, - ) - - def _build_metadata_driven_consolidate_question( pipeline: Pipeline, motifs: list, @@ -704,6 +666,80 @@ def _build_metadata_driven_lookup_tool_question( ) +def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuestion]: + """Builds one ``consolidate_motif:`` question per detected motif. + + Args: + motifs: Detected :class:`~flowx.models.motifs.DetectedMotif` + instances from :func:`flowx.motifs.detector.detect_motifs`. + + Returns: + A list of :class:`TranslationQuestion` instances, one per + detected motif. Each question uses a unique question_id of the + form ``consolidate_motif:`` so multiple distinct motif + types in the same pipeline (e.g. ``rest_api_pagination`` *and* + ``metadata_driven_bulk_copy``) each get their own prompt. + Returns an empty list when no motifs were detected. + + Notes: + - The default for every motif is ``keep``. Motif detection is + a heuristic match and over-collapsing silently rewrites + pipelines; requiring an explicit ``consolidate`` answer is + the safer default. + - When the same motif type is detected more than once in the + same pipeline (rare in practice but possible) the builder + emits a single question covering all instances of that type. + Per-instance overrides can still be expressed by adding more + fine-grained gating in :class:`MotifActivity`. + - The ``affected_task_keys`` field lists the *underlying* ADF + activity names so the agent can quote them when asking the + user, e.g. ``"Consolidate REST API Pagination motif spanning + GetToken, InitCursor, PollLoop into a single notebook?"``. + """ + if not motifs: + return [] + seen: set[str] = set() + questions: list[TranslationQuestion] = [] + for motif in motifs: + definition = motif.definition + motif_id = definition.motif_id + if motif_id in seen: + continue + seen.add(motif_id) + affected = tuple(motif.matched_activities) + question_id = f"{MOTIF_CONSOLIDATE_QUESTION_PREFIX}{motif_id}" + confidence_suffix = "" + if motif.confidence_notes: + confidence_suffix = " Detector notes: " + " | ".join(motif.confidence_notes) + questions.append( + TranslationQuestion( + question_id=question_id, + prompt=f"Consolidate the {definition.display_name!r} motif into a single task?", + rationale=( + f"{definition.description} " + f"Affected activities: {', '.join(affected) if affected else '(none)'}.{confidence_suffix} " + "Keep preserves the activity-by-activity translation; consolidate replaces " + f"them with a single {definition.databricks_replacement!r} task." + ), + options=( + QuestionOption( + value=MotifConsolidate.KEEP.value, + label="Keep individual activities", + description="Preserves the per-activity translation; no motif collapse.", + ), + QuestionOption( + value=MotifConsolidate.CONSOLIDATE.value, + label="Consolidate into one task", + description=f"Replaces matched activities with a {definition.databricks_replacement!r} task.", + ), + ), + affected_task_keys=affected, + default=MotifConsolidate.KEEP.value, + ) + ) + return questions + + def _metadata_driven_motif_task_keys( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None ) -> tuple[str, ...]: @@ -851,11 +887,6 @@ def _stamp_activity(activity: Activity, pipeline_preferences: TranslationPrefere return _stamp_copy_activity(activity, activity_preferences) if isinstance(activity, MotifActivity): return _stamp_motif_activity(activity, activity_preferences) - if isinstance(activity, (NotebookActivity, SparkPythonActivity)): - return dataclasses.replace( - activity, - compute_mode=_resolve_databricks_task_compute_mode(activity_preferences), - ) return dataclasses.replace(activity, compute_mode=_resolve_compute_mode(activity, activity_preferences)) @@ -1097,9 +1128,11 @@ def _resolve_compute_mode(activity: Activity, activity_preferences: TranslationP :data:`COMPUTE_MODE_CLASSIC_SINGLE_NODE`, :data:`COMPUTE_MODE_CLASSIC_MULTI_NODE`, or :data:`COMPUTE_MODE_INHERIT`. + + DatabricksNotebook and DatabricksSparkPython activities always + inherit the linked-service-derived cluster binding; serverless is + no longer offered as a replacement for source-defined clusters. """ - if isinstance(activity, (NotebookActivity, SparkPythonActivity)): - return _resolve_databricks_task_compute_mode(activity_preferences) if not is_non_databricks_task(activity): return COMPUTE_MODE_INHERIT if activity_preferences.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS: @@ -1107,19 +1140,3 @@ def _resolve_compute_mode(activity: Activity, activity_preferences: TranslationP if isinstance(activity, CopyActivity): return COMPUTE_MODE_CLASSIC_MULTI_NODE return COMPUTE_MODE_CLASSIC_SINGLE_NODE - - -def _resolve_databricks_task_compute_mode(activity_preferences: TranslationPreferences) -> str: - """Resolves the compute mode for an ADF Databricks-* task. - - Args: - activity_preferences: Effective preferences for the task. - - Returns: - :data:`COMPUTE_MODE_SERVERLESS` when the caller opted into - serverless; otherwise :data:`COMPUTE_MODE_INHERIT`, which leaves - the linked-service-derived binding in place. - """ - if activity_preferences.databricks_task_compute is DatabricksTaskCompute.SERVERLESS: - return COMPUTE_MODE_SERVERLESS - return COMPUTE_MODE_INHERIT diff --git a/src/orchestra/adapter/session.py b/src/orchestra/adapter/session.py index a312438..318622d 100644 --- a/src/orchestra/adapter/session.py +++ b/src/orchestra/adapter/session.py @@ -31,13 +31,13 @@ from flowx.adapter.models import ( DEFAULT_PREFERENCES, CopyActivityParadigm, - DatabricksTaskCompute, LakeflowConnectorType, MetadataDrivenAccess, MetadataDrivenConsolidate, MetadataDrivenLookupTool, MetadataDrivenSize, MigrationInputQuestion, + MotifConsolidate, NonDatabricksTaskCompute, PendingMigrationInputs, PendingQuestions, @@ -174,9 +174,6 @@ def build_preferences(self) -> TranslationPreferences: use_lakeflow_connectors=UseLakeflowConnectors( self._answers.get("use_lakeflow_connectors", self.defaults.use_lakeflow_connectors) ), - databricks_task_compute=DatabricksTaskCompute( - self._answers.get("databricks_task_compute", self.defaults.databricks_task_compute) - ), lakeflow_connector_type=LakeflowConnectorType( self._answers.get("lakeflow_connector_type", self.defaults.lakeflow_connector_type) ), @@ -192,9 +189,31 @@ def build_preferences(self) -> TranslationPreferences: metadata_driven_lookup_tool=MetadataDrivenLookupTool( self._answers.get("metadata_driven_lookup_tool", self.defaults.metadata_driven_lookup_tool) ), + motif_consolidations=self._collect_motif_consolidations(), per_task=self.defaults.per_task, ) + def _collect_motif_consolidations(self) -> dict[str, MotifConsolidate]: + """Returns the per-motif consolidation answers gathered so far. + + Returns: + Dict mapping ``motif_id`` to the user's :class:`MotifConsolidate` + answer. Motifs the user did not answer fall back to the + value carried on ``self.defaults`` (default + :data:`MotifConsolidate.KEEP`). The dict is the union of + the defaults and any answers whose ``question_id`` starts + with ``consolidate_motif:``. + """ + from flowx.adapter.constants import MOTIF_CONSOLIDATE_QUESTION_PREFIX + + consolidations: dict[str, MotifConsolidate] = dict(self.defaults.motif_consolidations) + for question_id, answer in self._answers.items(): + if not question_id.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): + continue + motif_id = question_id[len(MOTIF_CONSOLIDATE_QUESTION_PREFIX) :] + consolidations[motif_id] = MotifConsolidate(answer) + return consolidations + def resume(self) -> Pipeline: """Returns the preference-stamped pipeline IR. diff --git a/src/orchestra/bundler/dab_writer.py b/src/orchestra/bundler/dab_writer.py index fb4ddfb..7d00f0f 100644 --- a/src/orchestra/bundler/dab_writer.py +++ b/src/orchestra/bundler/dab_writer.py @@ -244,6 +244,9 @@ def write_bundle( all_tasks = list(workflow.tasks) for inner in workflow.inner_workflows: all_tasks.extend(inner.tasks) + parameter_approximations = list(workflow.parameter_approximations) + for inner in workflow.inner_workflows: + parameter_approximations.extend(inner.parameter_approximations) known_bundle_jobs = {resource_key} | {normalize_task_key(inner.name) for inner in workflow.inner_workflows} # ``manual_parameters`` was collected above (before YAML emission) so # the broken values are also stripped from the on-disk YAML. @@ -253,6 +256,7 @@ def write_bundle( known_bundle_jobs=known_bundle_jobs, cross_bundle_variables=dict(_cross_bundle_variables), manual_parameters=manual_parameters, + parameter_approximations=parameter_approximations, ) setup_path = output_dir / "SETUP.md" setup_path.write_text(render_setup_md(prereqs, bundle_name=effective_name), encoding="utf-8") @@ -1036,12 +1040,15 @@ def _reconstruct_preferences(raw: dict[str, Any] | None) -> Any: return None from flowx.adapter.models import TranslationPreferences + # Reports authored before the databricks_task_compute option was + # removed may still carry that key; drop it silently so old reports + # remain rehydratable. return TranslationPreferences( copy_activity_paradigm=raw.get("copy_activity_paradigm", "notebook"), non_databricks_task_compute=raw.get("non_databricks_task_compute", "serverless"), use_lakeflow_connectors=raw.get("use_lakeflow_connectors", "existing"), - databricks_task_compute=raw.get("databricks_task_compute", "existing"), lakeflow_connector_type=raw.get("lakeflow_connector_type", "cdc"), + motif_consolidations=dict(raw.get("motif_consolidations") or {}), per_task=dict(raw.get("per_task") or {}), ) @@ -1137,7 +1144,6 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: **base, main_class_name=task_ir.get("main_class_name", ""), parameters=task_ir.get("parameters"), - libraries=task_ir.get("libraries"), ) if task_type == "SparkPythonActivity": return SparkPythonActivity( @@ -1235,6 +1241,9 @@ def _common_activity_kwargs(task_ir: dict[str, Any]) -> dict[str, Any]: "min_retry_interval_millis": task_ir.get("min_retry_interval_millis"), "depends_on": _reconstruct_dependencies(task_ir.get("depends_on")), "cluster": task_ir.get("cluster"), + "existing_cluster_id": task_ir.get("existing_cluster_id"), + "libraries": task_ir.get("libraries"), + "parameter_approximations": list(task_ir.get("parameter_approximations") or []), "required_parameters": dict(task_ir.get("required_parameters") or {}), "compute_mode": task_ir.get("compute_mode"), } diff --git a/src/orchestra/bundler/prereqs_writer.py b/src/orchestra/bundler/prereqs_writer.py index 934b8bc..fe344f6 100644 --- a/src/orchestra/bundler/prereqs_writer.py +++ b/src/orchestra/bundler/prereqs_writer.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from typing import Any -from flowx.models.dab import DabNotebook +from flowx.models.dab import DabNotebook, ParameterApproximation # Regexes used to mine the generated artifacts for external dependencies. # Kept as compiled patterns so :func:`build_prereqs` is cheap to call. @@ -110,6 +110,7 @@ class Prereqs: compute_notes: list[str] = field(default_factory=list) network_endpoints: list[NetworkEndpoint] = field(default_factory=list) manual_parameters: list[ManualParameter] = field(default_factory=list) + parameter_approximations: list[ParameterApproximation] = field(default_factory=list) def is_empty(self) -> bool: """Return ``True`` when nothing needs to happen before ``bundle run``.""" @@ -121,6 +122,7 @@ def is_empty(self) -> bool: and not self.compute_notes and not self.network_endpoints and not self.manual_parameters + and not self.parameter_approximations ) @@ -323,6 +325,7 @@ def build_prereqs( cross_bundle_variables: dict[str, str] | None = None, compute_notes: list[str] | None = None, manual_parameters: list[ManualParameter] | None = None, + parameter_approximations: list[ParameterApproximation] | None = None, ) -> Prereqs: """Assemble a :class:`Prereqs` from the bundle's generated artifacts. @@ -356,6 +359,7 @@ def build_prereqs( compute_notes=list(compute_notes or []), network_endpoints=collect_network_endpoints(notebooks), manual_parameters=list(manual_parameters or []), + parameter_approximations=list(parameter_approximations or []), ) @@ -518,6 +522,26 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: ) lines.append("") + if prereqs.parameter_approximations: + lines.append("## Parameter substitutions") + lines.append("") + lines.append( + "Flowx mapped the ADF expressions below to Databricks dynamic value " + "references so they land directly in the bundle YAML. The substitutions are " + "semantically *close* but not identical to the originals; review the listed " + "caveats and decide whether each replacement is acceptable for your workload." + ) + lines.append("") + lines.append("| Task | Widget | Original ADF expression | Replacement | Caveat |") + lines.append("|---|---|---|---|---|") + for approximation in prereqs.parameter_approximations: + lines.append( + f"| `{approximation.task_key}` | `{approximation.widget_name}` " + f"| `{approximation.raw_expression}` | `{approximation.replacement}` " + f"| {approximation.note} |" + ) + lines.append("") + if prereqs.network_endpoints: lines.append("## Networking") lines.append("") diff --git a/src/orchestra/models/dab.py b/src/orchestra/models/dab.py index 23bd002..ec0b85c 100644 --- a/src/orchestra/models/dab.py +++ b/src/orchestra/models/dab.py @@ -123,6 +123,27 @@ class SetupTask: config: dict[str, Any] = field(default_factory=dict) +@dataclass(slots=True, kw_only=True) +class ParameterApproximation: + """A base_parameter where flowx substituted a DAB dynamic value for an + ADF expression with non-identical semantics (e.g. ``utcnow()`` mapped to + job start time). + + Attributes: + task_key: DAB task key. + widget_name: The base_parameter name. + raw_expression: The original ADF expression text. + replacement: The DAB dynamic value reference flowx emitted. + note: Human-readable caveat explaining the semantic difference. + """ + + task_key: str + widget_name: str + raw_expression: str + replacement: str + note: str + + # --------------------------------------------------------------------------- # Top-level bundle # --------------------------------------------------------------------------- diff --git a/src/orchestra/models/ir.py b/src/orchestra/models/ir.py index 3bb7fa7..12a383f 100644 --- a/src/orchestra/models/ir.py +++ b/src/orchestra/models/ir.py @@ -14,10 +14,11 @@ class ExpressionResult: """Result of resolving an ADF expression.""" - kind: str # "literal", "dab_ref", "notebook_code" + kind: str value: str imports: list[str] = field(default_factory=list) required_parameters: dict[str, str] = field(default_factory=dict) + notes: list[str] = field(default_factory=list) @dataclass(slots=True, kw_only=True) @@ -46,6 +47,12 @@ class Activity: min_retry_interval_millis: Minimum delay between retries (ms). depends_on: Upstream task dependencies. cluster: Cluster configuration for the task, if any. + existing_cluster_id: ID of an existing all-purpose cluster the task + should run on. + libraries: Task-scoped library descriptors carried through from ADF. + Each entry is a supported Databricks task library; see + https://docs.databricks.com/aws/en/dev-tools/bundles/library-dependencies + for the supported shapes. """ name: str @@ -56,10 +63,13 @@ class Activity: min_retry_interval_millis: int | None = None depends_on: list[Dependency] | None = None cluster: dict[str, Any] | None = None - # Widget-name → DAB-ref mapping for every `dbutils.widgets.get()` call - # that shows up in any notebook_code the translator produced for this - # activity. Preparers thread these into ``base_parameters`` so DAB - # resolves the refs at job runtime. + existing_cluster_id: str | None = None + libraries: list[dict[str, Any]] | None = None + # Approximate parameter substitutions made at translation time (e.g. + # ``utcnow()`` mapped to ``{{job.start_time.iso_datetime}}``). Each + # entry has keys ``widget_name``, ``raw_expression``, ``replacement``, + # and ``note``; the bundler surfaces these in SETUP.md. + parameter_approximations: list[dict[str, str]] = field(default_factory=list) required_parameters: dict[str, str] = field(default_factory=dict) # Compute mode stamped by the pipeline modifier in response to user # preferences. One of "serverless", "classic_single_node", @@ -269,12 +279,10 @@ class SparkJarActivity(Activity): Attributes: main_class_name: Fully qualified main class within the JAR. parameters: Arguments passed to the main class. - libraries: Library descriptors (JARs, wheels, etc.). """ main_class_name: str parameters: list[str] | None = None - libraries: list[dict[str, Any]] | None = None @dataclass(slots=True, kw_only=True) @@ -578,7 +586,15 @@ class TranslationReport: agentic_count: Activities requiring agentic translation. unsupported_count: Activities that could not be translated. gaps: List of agentic gaps identified during translation. - warnings: Human-readable warning messages emitted during translation. + warnings: Human-readable warning messages emitted during translation, + including unresolved ``@{...}`` ADF expressions surfaced by the + whole-IR rewriter. + detected_motifs: Multi-activity patterns the detector matched on the + source AST. When the caller did not supply a motif-consolidation + answer the translator collapses every entry into a single + :class:`MotifActivity`; otherwise the list still reports what + was detected so the adapter can prompt the user. The objects + here are :class:`~flowx.models.motifs.DetectedMotif` instances. """ pipeline: Pipeline @@ -587,3 +603,4 @@ class TranslationReport: unsupported_count: int = 0 gaps: list[AgenticGap] = field(default_factory=list) warnings: list[str] = field(default_factory=list) + detected_motifs: list[Any] = field(default_factory=list) diff --git a/src/orchestra/parser/expression_parser.py b/src/orchestra/parser/expression_parser.py index 1fcb69c..a325cd0 100644 --- a/src/orchestra/parser/expression_parser.py +++ b/src/orchestra/parser/expression_parser.py @@ -352,22 +352,53 @@ def _resolve_variable( return ExpressionResult(kind="dab_ref", value="{{" + f"tasks.{setter_key}.values.{var_name}" + "}}") +_UTCNOW_APPROXIMATION_NOTE = ( + "Mapped ADF `utcnow()` to the Databricks job start time. " + "Single-task jobs see sub-second skew; multi-task jobs can see minutes of skew " + "between job start and the moment the activity actually runs." +) + +# ADF .NET-style format strings that map cleanly onto a Databricks dynamic +# value reference. Anything not in this table falls back to a notebook_code +# strftime call. +_UTCNOW_FORMAT_TO_DAB_REF: dict[str, str] = { + "yyyy-MM-dd": "{{job.start_time.iso_date}}", + "yyyy-MM-ddTHH:mm:ss": "{{job.start_time.iso_datetime}}", + "yyyy-MM-ddTHH:mm:ssZ": "{{job.start_time.iso_datetime}}", + "o": "{{job.start_time.iso_datetime}}", + "s": "{{job.start_time.iso_datetime}}", +} + + def _resolve_utcnow(expr: str) -> ExpressionResult | None: - """Resolves ``utcNow()`` or ``utcNow('format')`` -> notebook_code.""" + """Resolves ``utcNow()`` / ``utcNow('format')``. + + Bare ``utcnow()`` and ``utcnow('')`` map to a Databricks + dynamic value reference so the result can land directly inside DAB + YAML (``base_parameters`` and similar). Unrecognised format strings + keep the legacy ``notebook_code`` translation. + """ match = _UTCNOW_RE.match(expr) if match is None: return None format_string = match.group(1) - if format_string: - python_format = _convert_date_format(format_string) + if not format_string: return ExpressionResult( - kind="notebook_code", - value=f"datetime.now(timezone.utc).strftime('{python_format}')", - imports=["from datetime import datetime, timezone"], + kind="dab_ref", + value="{{job.start_time.iso_datetime}}", + notes=[_UTCNOW_APPROXIMATION_NOTE], + ) + dab_ref = _UTCNOW_FORMAT_TO_DAB_REF.get(format_string) + if dab_ref: + return ExpressionResult( + kind="dab_ref", + value=dab_ref, + notes=[_UTCNOW_APPROXIMATION_NOTE], ) + python_format = _convert_date_format(format_string) return ExpressionResult( kind="notebook_code", - value="datetime.now(timezone.utc).isoformat()", + value=f"datetime.now(timezone.utc).strftime('{python_format}')", imports=["from datetime import datetime, timezone"], ) @@ -628,6 +659,11 @@ def _collect_required_parameters(*args: ExpressionResult) -> dict[str, str]: return merged +def _collect_notes(*args: ExpressionResult) -> list[str]: + """Collects caveat notes across resolved arguments.""" + return [note for arg in args for note in arg.notes] + + def _result_from_args( value: str, args: list[ExpressionResult], @@ -649,6 +685,7 @@ def _result_from_args( value=value, imports=imports, required_parameters=_collect_required_parameters(*args), + notes=_collect_notes(*args), ) @@ -1140,6 +1177,18 @@ def _handle_format_date_time(args: list[ExpressionResult]) -> ExpressionResult | """formatDateTime(ts, fmt?) -> datetime.fromisoformat(ts).strftime(converted_fmt)""" if len(args) < 1 or len(args) > 2: return None + # formatDateTime(utcnow(), '') -> remap straight to the + # matching Databricks dynamic value so the result lands in YAML. + if ( + len(args) == 2 + and args[0].kind == "dab_ref" + and args[0].value == "{{job.start_time.iso_datetime}}" + and _UTCNOW_APPROXIMATION_NOTE in args[0].notes + and args[1].kind == "literal" + ): + dab_ref = _UTCNOW_FORMAT_TO_DAB_REF.get(args[1].value) + if dab_ref: + return ExpressionResult(kind="dab_ref", value=dab_ref, notes=[_UTCNOW_APPROXIMATION_NOTE]) timestamp_dt = _datetime_arg_code(args[0]) if len(args) == 2 and args[1].kind == "literal": python_format = _convert_date_format(args[1].value) diff --git a/src/orchestra/parser/ir_rewriter.py b/src/orchestra/parser/ir_rewriter.py new file mode 100644 index 0000000..b3cc0a5 --- /dev/null +++ b/src/orchestra/parser/ir_rewriter.py @@ -0,0 +1,252 @@ +"""Whole-IR expression rewriter. + +Activity translators each call :func:`resolve_interpolated_string` on the +specific string fields they know about (``source_query``, ``url``, +``base_parameters`` values, ...). Anything outside those known fields +-- raw SQL ``WHERE`` clauses inside ``source_properties``, REST request +bodies, dataset folder paths, ``base_parameters`` strings that pass +through a value untouched -- can carry through to the bundle as a +literal ``@{...}`` ADF expression, silently corrupting query semantics +at runtime. + +This module runs a final pass over the translated IR, walks every +string-typed field on every :class:`~flowx.models.ir.Activity` +(including strings nested inside ``dict`` / ``list`` fields and inside +control-flow inner activities), and re-applies +:func:`resolve_interpolated_string` to each. Every ``@{...}`` token +that still remains after the pass is appended to *warnings* so the +caller surfaces the gap in the translation report. + +Fields that intentionally hold raw ADF input (``linked_service_definition``, +``raw_definition``) or that preserve pre-rewrite history +(``original_activities`` on :class:`~flowx.models.ir.MotifActivity`) +are skipped. Identifier fields (``name``, ``task_key``, +``variable_name``) are skipped so that rewriting cannot break +cross-task references. +""" + +from __future__ import annotations + +import dataclasses +import re +from types import MappingProxyType +from typing import Any + +from flowx.models.ir import ( + Activity, + AppendVariableActivity, + Pipeline, + SetVariableActivity, + SwitchActivity, + SwitchCase, + TranslationContext, +) +from flowx.parser.expression_parser import resolve_interpolated_string + +# Fields that must never be rewritten — either they hold raw ADF input +# that downstream consumers parse separately, or they are identifiers +# whose value is used as a reference key elsewhere in the IR. +_FIELDS_TO_SKIP: frozenset[str] = frozenset( + { + "name", + "task_key", + "linked_service_definition", + "raw_definition", + "original_activities", + "variable_name", + "matched_activity_names", + } +) + +_UNRESOLVED_RE = re.compile(r"@\{[^}]+\}") + + +def rewrite_pipeline_expressions( + pipeline: Pipeline, + *, + warnings: list[str] | None = None, +) -> Pipeline: + """Walks every string field in *pipeline* and rewrites ``@{...}`` tokens. + + Args: + pipeline: Translated pipeline IR. Not mutated. + warnings: Optional list to which the rewriter appends a message + for every string field that still contains an unresolved + ``@{...}`` token after the pass. When ``None`` the rewriter + still runs but cannot surface gaps. + + Returns: + A new :class:`Pipeline` whose activities have had their string + fields rewritten through + :func:`~flowx.parser.expression_parser.resolve_interpolated_string`. + + Notes: + - The rewriter builds a single :class:`TranslationContext` from + the pipeline's :class:`SetVariableActivity` and + :class:`AppendVariableActivity` nodes so that + ``@{variables('x')}`` tokens resolve against the same setter + task keys the per-activity translators used. + - Fields listed in ``_FIELDS_TO_SKIP`` -- identifiers and raw + ADF input -- are returned unchanged. + - Unknown field types (ints, bools, None, custom dataclasses + beyond Activity/SwitchCase) pass through unchanged. + """ + context = _build_context_from_pipeline(pipeline) + sink: list[str] = warnings if warnings is not None else [] + rewritten_tasks = [_rewrite_activity(activity, context, sink) for activity in pipeline.tasks] + return dataclasses.replace(pipeline, tasks=rewritten_tasks) + + +def _build_context_from_pipeline(pipeline: Pipeline) -> TranslationContext: + """Builds a TranslationContext whose variable_cache is keyed by every + SetVariable / AppendVariable activity in the pipeline. + + Args: + pipeline: Translated pipeline IR. + + Returns: + A :class:`TranslationContext` with ``variable_cache`` populated. + Activities that nest inside control-flow types are walked too so + a variable set inside a ForEach is still discoverable. + """ + variable_cache: dict[str, str] = {} + + def visit(activities: list[Activity]) -> None: + for activity in activities: + if isinstance(activity, (SetVariableActivity, AppendVariableActivity)): + variable_cache.setdefault(activity.variable_name, activity.task_key) + # Recurse into control-flow branches. + nested = _nested_activities(activity) + if nested: + visit(nested) + + visit(list(pipeline.tasks)) + return TranslationContext( + activity_cache=MappingProxyType({}), + registry=MappingProxyType({}), + variable_cache=MappingProxyType(variable_cache), + ) + + +def _nested_activities(activity: Activity) -> list[Activity]: + """Returns activities nested inside a control-flow activity, or []. + + Args: + activity: Any IR activity. + + Returns: + The control-flow branches' activity lists concatenated, or an + empty list when *activity* is a leaf type. + """ + nested: list[Activity] = [] + if hasattr(activity, "inner_activities"): + nested.extend(getattr(activity, "inner_activities") or []) + if hasattr(activity, "if_true_activities"): + nested.extend(getattr(activity, "if_true_activities") or []) + if hasattr(activity, "if_false_activities"): + nested.extend(getattr(activity, "if_false_activities") or []) + if isinstance(activity, SwitchActivity): + for case in activity.cases: + nested.extend(case.activities) + nested.extend(activity.default_activities) + return nested + + +def _rewrite_activity(activity: Activity, context: TranslationContext, warnings: list[str]) -> Activity: + """Returns a new activity with every safe string field rewritten. + + Args: + activity: Activity to rewrite. Not mutated. + context: Translation context whose ``variable_cache`` resolves + ``@variables('x')`` tokens. + warnings: List to append unresolved-expression warnings to. + + Returns: + A new activity instance with rewritten string fields. Control- + flow activities have their inner branches recursed into. + """ + field_overrides: dict[str, Any] = {} + for f in dataclasses.fields(activity): + if f.name in _FIELDS_TO_SKIP: + continue + original = getattr(activity, f.name) + rewritten = _rewrite_value( + original, + context, + warnings, + field_path=f"{type(activity).__name__}.{activity.task_key}.{f.name}", + ) + if rewritten is not original: + field_overrides[f.name] = rewritten + + if not field_overrides: + return activity + return dataclasses.replace(activity, **field_overrides) + + +def _rewrite_value(value: Any, context: TranslationContext, warnings: list[str], *, field_path: str) -> Any: + """Recursively rewrites every string contained in *value*. + + Args: + value: Any IR value -- str, list, dict, Activity, SwitchCase, or + a primitive. Activities and SwitchCases recurse; primitives + pass through. + context: Translation context. + warnings: List to append unresolved-expression warnings to. + field_path: Dotted path describing where this value sits in the + IR (used in warning messages so the user can find the gap). + + Returns: + The rewritten value, or *value* unchanged when no rewrite + applied. + """ + if isinstance(value, str): + return _rewrite_string(value, context, warnings, field_path=field_path) + if isinstance(value, Activity): + return _rewrite_activity(value, context, warnings) + if isinstance(value, SwitchCase): + new_value = _rewrite_value(value.value, context, warnings, field_path=f"{field_path}.value") + new_activities = [_rewrite_activity(a, context, warnings) for a in value.activities] + if new_value is value.value and all(n is o for n, o in zip(new_activities, value.activities)): + return value + return SwitchCase(value=new_value, activities=new_activities) + if isinstance(value, list): + new_list = [ + _rewrite_value(item, context, warnings, field_path=f"{field_path}[{i}]") for i, item in enumerate(value) + ] + if all(n is o for n, o in zip(new_list, value)): + return value + return new_list + if isinstance(value, dict): + new_dict = { + k: _rewrite_value(v, context, warnings, field_path=f"{field_path}[{k!r}]") for k, v in value.items() + } + if all(new_dict[k] is value[k] for k in value): + return value + return new_dict + return value + + +def _rewrite_string(value: str, context: TranslationContext, warnings: list[str], *, field_path: str) -> str: + """Applies ``resolve_interpolated_string`` and surfaces leftover ``@{...}``. + + Args: + value: String value to rewrite. + context: Translation context. + warnings: List to append unresolved-expression warnings to. + field_path: Dotted path for the warning message. + + Returns: + The rewritten string. When the rewrite cannot resolve every + ``@{...}`` token, the leftover tokens remain in the returned + string (for forensics) and a warning is appended. + """ + if "@{" not in value: + return value + rewritten = resolve_interpolated_string(value, context) + leftovers = _UNRESOLVED_RE.findall(rewritten) + if leftovers: + warnings.append( + f"Unresolved ADF expression at {field_path}: {sorted(set(leftovers))!r} (left in output verbatim)" + ) + return rewritten diff --git a/src/orchestra/preparer/activity_preparers/notebook.py b/src/orchestra/preparer/activity_preparers/notebook.py index a756cec..c131fde 100644 --- a/src/orchestra/preparer/activity_preparers/notebook.py +++ b/src/orchestra/preparer/activity_preparers/notebook.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING -from flowx.models.dab import DabNotebook +from flowx.models.dab import DabNotebook, ParameterApproximation from flowx.models.ir import TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string from flowx.preparer.activity_preparers.naming import notebook_filename, workspace_notebook_filename @@ -113,6 +113,17 @@ def prepare( existing_notebook=is_existing_notebook, ) + approximations = [ + ParameterApproximation( + task_key=activity.task_key, + widget_name=entry["widget_name"], + raw_expression=entry["raw_expression"], + replacement=entry["replacement"], + note=entry["note"], + ) + for entry in activity.parameter_approximations + ] + if is_existing_notebook: downloaded = download_notebook(resolved_path) if workspace_downloads_enabled() else None if downloaded is not None: @@ -121,15 +132,19 @@ def prepare( task["notebook_task"] = {"notebook_path": f"../src/{notebook_relative_path}"} if base_parameters is not None: task["notebook_task"]["base_parameters"] = base_parameters - if activity.compute_mode != "serverless": + if activity.compute_mode != "serverless" and not task.get("existing_cluster_id"): task["job_cluster_key"] = "default_cluster" + if activity.libraries: + task["libraries"] = activity.libraries notebooks = [DabNotebook(relative_path=notebook_relative_path, content=downloaded)] - return PreparedActivity(task=task, notebooks=notebooks) + return PreparedActivity(task=task, notebooks=notebooks, parameter_approximations=approximations) task["notebook_task"] = {"notebook_path": resolved_path} if base_parameters is not None: task["notebook_task"]["base_parameters"] = base_parameters - return PreparedActivity(task=task) + if activity.libraries: + task["libraries"] = activity.libraries + return PreparedActivity(task=task, parameter_approximations=approximations) placeholder_filename = notebook_filename(activity.task_key, activity.name) notebook_relative_path = f"notebooks/{placeholder_filename}" @@ -140,6 +155,8 @@ def prepare( task["notebook_task"] = {"notebook_path": f"../src/{notebook_relative_path}"} if base_parameters is not None: task["notebook_task"]["base_parameters"] = base_parameters + if activity.libraries: + task["libraries"] = activity.libraries notebooks = [DabNotebook(relative_path=notebook_relative_path, content=content)] - return PreparedActivity(task=task, notebooks=notebooks) + return PreparedActivity(task=task, notebooks=notebooks, parameter_approximations=approximations) diff --git a/src/orchestra/preparer/activity_preparers/spark_python.py b/src/orchestra/preparer/activity_preparers/spark_python.py index 9a9c5e7..9156967 100644 --- a/src/orchestra/preparer/activity_preparers/spark_python.py +++ b/src/orchestra/preparer/activity_preparers/spark_python.py @@ -69,4 +69,6 @@ def prepare(activity: SparkPythonActivity, *, scope: str = "") -> PreparedActivi } if activity.parameters: task["spark_python_task"]["parameters"] = list(activity.parameters) + if activity.libraries: + task["libraries"] = activity.libraries return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/workflow_preparer.py b/src/orchestra/preparer/workflow_preparer.py index 815ae4c..bd8680a 100644 --- a/src/orchestra/preparer/workflow_preparer.py +++ b/src/orchestra/preparer/workflow_preparer.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from typing import Any -from flowx.models.dab import DabNotebook, SecretInstruction, SetupTask +from flowx.models.dab import DabNotebook, ParameterApproximation, SecretInstruction, SetupTask from flowx.models.ir import ( Activity, AppendVariableActivity, @@ -50,6 +50,7 @@ class PreparedActivity: # ``resources/pipelines/.yml``. Each entry is a dict # with ``resource_key`` and ``definition`` keys. pipeline_resources: list[dict[str, Any]] = field(default_factory=list) + parameter_approximations: list[ParameterApproximation] = field(default_factory=list) @dataclass(slots=True, kw_only=True) @@ -65,6 +66,7 @@ class PreparedWorkflow: parameters: list[dict[str, Any]] = field(default_factory=list) cluster_hints: list[dict[str, Any]] = field(default_factory=list) pipeline_resources: list[dict[str, Any]] = field(default_factory=list) + parameter_approximations: list[ParameterApproximation] = field(default_factory=list) def run_if_from_adf_outcomes(outcomes: list[str | None]) -> str | None: @@ -106,6 +108,9 @@ def build_common_task_fields(activity: Activity) -> dict[str, Any]: if activity.description: task["description"] = activity.description + if activity.existing_cluster_id: + task["existing_cluster_id"] = activity.existing_cluster_id + return task @@ -243,6 +248,7 @@ class PreparedArtifacts: setup_tasks: tuple[SetupTask, ...] = () inner_workflows: tuple[PreparedWorkflow, ...] = () pipeline_resources: tuple[dict[str, Any], ...] = () + parameter_approximations: tuple[ParameterApproximation, ...] = () def merge_prepared_artifacts( @@ -256,6 +262,7 @@ def merge_prepared_artifacts( setup_tasks=artifacts.setup_tasks + tuple(prepared.setup_tasks), inner_workflows=artifacts.inner_workflows + tuple(prepared.inner_workflows), pipeline_resources=artifacts.pipeline_resources + tuple(prepared.pipeline_resources), + parameter_approximations=artifacts.parameter_approximations + tuple(prepared.parameter_approximations), ) @@ -307,6 +314,7 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: inner_workflows=list(artifacts.inner_workflows), cluster_hints=cluster_hints, pipeline_resources=list(artifacts.pipeline_resources), + parameter_approximations=list(artifacts.parameter_approximations), ) diff --git a/src/orchestra/translator/activity_translators/notebook.py b/src/orchestra/translator/activity_translators/notebook.py index 3fc5897..438c001 100644 --- a/src/orchestra/translator/activity_translators/notebook.py +++ b/src/orchestra/translator/activity_translators/notebook.py @@ -31,15 +31,29 @@ def translate( notebook_path = resolve_field(type_properties.get("notebookPath", ""), context) raw_params = type_properties.get("baseParameters") or {} + libraries = type_properties.get("libraries") # Resolve base_parameters at translate time so ADF expressions like # @variables('runTimestamp') are inlined to DAB refs while the full - # translation context (with variable_value_cache) is available. + # translation context (with variable_value_cache) is available. Any + # caveat notes the resolver emits (e.g. utcnow() approximations) are + # captured into parameter_approximations so the bundler can surface + # them in SETUP.md. resolved_params: dict[str, Any] = {} + approximations: list[dict[str, str]] = [] for key, value in raw_params.items(): result = resolve_expression(value, context) if result is not None and result.kind in ("dab_ref", "literal"): resolved_params[key] = result.value + for note in result.notes: + approximations.append( + { + "widget_name": key, + "raw_expression": _raw_expression_text(value), + "replacement": result.value, + "note": note, + } + ) else: # Keep original for downstream handling (notebook_code or unresolvable) resolved_params[key] = value @@ -48,4 +62,13 @@ def translate( **base_kwargs, notebook_path=notebook_path, base_parameters=resolved_params, + libraries=libraries, + parameter_approximations=approximations, ) + + +def _raw_expression_text(value: Any) -> str: + """Returns the original ADF expression text from a base_parameter value.""" + if isinstance(value, dict) and value.get("type") == "Expression" and "value" in value: + return str(value["value"]) + return str(value) diff --git a/src/orchestra/translator/activity_translators/spark_python.py b/src/orchestra/translator/activity_translators/spark_python.py index 8dc8892..b0b230a 100644 --- a/src/orchestra/translator/activity_translators/spark_python.py +++ b/src/orchestra/translator/activity_translators/spark_python.py @@ -55,6 +55,7 @@ def translate( python_file = resolve_field(type_properties.get("pythonFile", ""), context) raw_parameters = type_properties.get("parameters") or [] + libraries = type_properties.get("libraries") parameters = [_resolve_parameter(p, context) for p in raw_parameters] @@ -62,4 +63,5 @@ def translate( **base_kwargs, python_file=python_file, parameters=parameters, + libraries=libraries, ) diff --git a/src/orchestra/translator/engine.py b/src/orchestra/translator/engine.py index 038e2d9..aa217cb 100644 --- a/src/orchestra/translator/engine.py +++ b/src/orchestra/translator/engine.py @@ -48,6 +48,7 @@ from flowx.motifs.collapser import collapse_motifs from flowx.motifs.detector import detect_motifs from flowx.parser.adf_loader import classify_activity, load_adf_definitions +from flowx.parser.ir_rewriter import rewrite_pipeline_expressions from flowx.translator.activity_translators import ( append_variable, copy, @@ -84,17 +85,38 @@ } -def translate_pipeline(pipeline: AdfPipeline, definitions: AdfDefinitions) -> TranslationReport: +def translate_pipeline( + pipeline: AdfPipeline, + definitions: AdfDefinitions, + *, + motif_consolidations: dict[str, str] | None = None, +) -> TranslationReport: """Translates an ADF pipeline into a Databricks pipeline IR. Args: pipeline: Parsed ADF pipeline AST. definitions: Full ADF definitions for cross-referencing datasets, linked services, etc. + motif_consolidations: Optional mapping of ``motif_id`` -> + ``"keep"`` / ``"consolidate"`` answers gathered from the + adapter. When ``None`` (back-compat default) the translator + consolidates every detected motif. When provided, only + motifs whose id maps to ``"consolidate"`` are collapsed; the + rest remain as the original activity-by-activity translation. Returns: - :class:`TranslationReport` containing the translated :class:`Pipeline` - and any gaps encountered. + :class:`TranslationReport` containing the translated :class:`Pipeline`, + the list of detected motifs, and any gaps encountered. + + Notes: + After dispatching individual activities the translator runs + :func:`~flowx.parser.ir_rewriter.rewrite_pipeline_expressions` + over the whole IR so that ``@{...}`` ADF expressions embedded in + SQL bodies, REST payloads, dataset paths, and other string-typed + fields are rewritten through the same parser the per-activity + translators use. Tokens that cannot be resolved are recorded + as translation warnings instead of shipping into the bundle + verbatim. """ context = TranslationContext( activity_cache=MappingProxyType({}), @@ -154,18 +176,35 @@ def translate_pipeline(pipeline: AdfPipeline, definitions: AdfDefinitions) -> Tr tags={"source": "adf", "pipeline": pipeline.name}, ) - # Motif detection and collapsing: scan for known multi-activity patterns - # and replace matched groups with single MotifActivity nodes. + # Whole-IR expression rewrite: catches @{...} tokens the per-activity + # translators didn't address (raw SQL WHERE clauses inside source_properties, + # REST request bodies, dataset folder paths, ...). Unresolved tokens are + # surfaced as translation warnings. + pipeline_ir = rewrite_pipeline_expressions(pipeline_ir, warnings=warnings) + + # Motif detection: scan for known multi-activity patterns. Collapsing is + # gated on the per-motif preference -- when *motif_consolidations* is + # ``None`` we preserve back-compat behaviour and collapse every detected + # motif; otherwise only motifs whose motif_id maps to ``"consolidate"`` are + # collapsed. detected_motifs = detect_motifs(pipeline, definitions) - if detected_motifs: - pipeline_ir = collapse_motifs(pipeline_ir, detected_motifs) - for motif in detected_motifs: + motifs_to_collapse = _filter_motifs_for_collapse(detected_motifs, motif_consolidations) + if motifs_to_collapse: + pipeline_ir = collapse_motifs(pipeline_ir, motifs_to_collapse) + for motif in motifs_to_collapse: logger.info( "Collapsed motif '%s': %d activities -> %s", motif.definition.display_name, len(motif.matched_activities), motif.definition.databricks_replacement, ) + for motif in detected_motifs: + if motif not in motifs_to_collapse: + logger.info( + "Detected motif '%s' left expanded: matched %d activities (user opted to keep)", + motif.definition.display_name, + len(motif.matched_activities), + ) return TranslationReport( pipeline=pipeline_ir, @@ -174,9 +213,32 @@ def translate_pipeline(pipeline: AdfPipeline, definitions: AdfDefinitions) -> Tr unsupported_count=unsupported_count, gaps=gaps, warnings=warnings, + detected_motifs=list(detected_motifs), ) +def _filter_motifs_for_collapse( + detected_motifs: list, + motif_consolidations: dict[str, str] | None, +) -> list: + """Returns the subset of detected motifs the caller asked to collapse. + + Args: + detected_motifs: Output of + :func:`flowx.motifs.detector.detect_motifs`. + motif_consolidations: Caller-supplied answers. ``None`` means + "collapse all" (back-compat). Otherwise only motifs whose + ``motif_id`` maps to ``"consolidate"`` are collapsed. + + Returns: + The subset of motifs to pass to + :func:`flowx.motifs.collapser.collapse_motifs`. + """ + if motif_consolidations is None: + return list(detected_motifs) + return [m for m in detected_motifs if motif_consolidations.get(m.definition.motif_id) == "consolidate"] + + def _dispatch_activity( activity: AdfActivity, context: TranslationContext, @@ -378,11 +440,14 @@ def _build_base_kwargs(activity: AdfActivity, definitions: AdfDefinitions) -> di ) cluster: dict[str, Any] | None = None + existing_cluster_id: str | None = None if activity.linked_service_name: linked_service_name = activity.linked_service_name.reference_name linked_service_def = definitions.linked_services.get(linked_service_name) if linked_service_def: cluster = _extract_cluster_config(linked_service_def.properties) + if cluster: + existing_cluster_id = cluster.get("existing_cluster_id") return { "name": activity.name, @@ -393,6 +458,7 @@ def _build_base_kwargs(activity: AdfActivity, definitions: AdfDefinitions) -> di "min_retry_interval_millis": min_retry_interval_millis, "depends_on": depends_on, "cluster": cluster, + "existing_cluster_id": existing_cluster_id, } @@ -495,15 +561,17 @@ def _preferences_to_dict(preferences: Any) -> dict[str, Any]: preferences: The :class:`TranslationPreferences` snapshot to serialise. Returns: - Dictionary with the four StrEnum fields rendered as their string - values and per-task overrides preserved verbatim. + Dictionary with each StrEnum field rendered as its string value + and per-task overrides preserved verbatim. """ return { "copy_activity_paradigm": str(preferences.copy_activity_paradigm), "non_databricks_task_compute": str(preferences.non_databricks_task_compute), "use_lakeflow_connectors": str(preferences.use_lakeflow_connectors), - "databricks_task_compute": str(preferences.databricks_task_compute), "lakeflow_connector_type": str(preferences.lakeflow_connector_type), + "motif_consolidations": { + motif_id: str(choice) for motif_id, choice in preferences.motif_consolidations.items() + }, "per_task": dict(preferences.per_task), } @@ -536,8 +604,14 @@ def _activity_to_dict(task: Activity) -> dict[str, Any]: ] if task.cluster: task_dict["cluster"] = task.cluster + if task.existing_cluster_id: + task_dict["existing_cluster_id"] = task.existing_cluster_id if task.compute_mode: task_dict["compute_mode"] = task.compute_mode + if task.libraries: + task_dict["libraries"] = task.libraries + if task.parameter_approximations: + task_dict["parameter_approximations"] = task.parameter_approximations extra = _activity_extra_fields(task) task_dict.update(extra) @@ -638,8 +712,6 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["main_class_name"] = activity.main_class_name if activity.parameters: extra["parameters"] = activity.parameters - if activity.libraries: - extra["libraries"] = activity.libraries case SparkPythonActivity(): extra["python_file"] = activity.python_file if activity.parameters: diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 0247e30..92c2615 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -10,7 +10,6 @@ from flowx.adapter import ( CopyActivityParadigm, - DatabricksTaskCompute, NonDatabricksTaskCompute, TranslationInputRequired, TranslationPreferences, @@ -29,7 +28,6 @@ COMPUTE_MODE_SERVERLESS, LAKEFLOW_CONNECT_REPLACEMENT, QUESTION_COPY_ACTIVITY_PARADIGM, - QUESTION_DATABRICKS_TASK_COMPUTE, QUESTION_LAKEFLOW_CONNECTOR_TYPE, QUESTION_METADATA_DRIVEN_ACCESS, QUESTION_METADATA_DRIVEN_CONSOLIDATE, @@ -129,7 +127,6 @@ def test_default_preferences_are_conservative(self): assert prefs.copy_activity_paradigm is CopyActivityParadigm.NOTEBOOK assert prefs.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS assert prefs.use_lakeflow_connectors is UseLakeflowConnectors.EXISTING - assert prefs.databricks_task_compute is DatabricksTaskCompute.EXISTING def test_string_values_coerce_to_enums(self): prefs = TranslationPreferences( @@ -235,21 +232,23 @@ def test_lakeflow_connect_question_surfaces_for_database_motif(self): lfc_question = next(q for q in pending.questions if q.question_id == QUESTION_USE_LAKEFLOW_CONNECTORS) assert "motif_incremental_load_watermark" in lfc_question.affected_task_keys - def test_databricks_task_compute_question_when_notebook_present(self): + def test_no_databricks_task_compute_question_for_notebook(self): + """The serverless-replacement question for Databricks tasks was removed.""" pipeline = Pipeline( name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")], ) ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_DATABRICKS_TASK_COMPUTE in ids + assert "databricks_task_compute" not in ids - def test_databricks_task_compute_question_for_spark_python(self): + def test_no_databricks_task_compute_question_for_spark_python(self): + """The serverless-replacement question for Databricks tasks was removed.""" pipeline = Pipeline( name="p", tasks=[SparkPythonActivity(**_make_base("py"), python_file="dbfs:/scripts/etl.py")], ) ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_DATABRICKS_TASK_COMPUTE in ids + assert "databricks_task_compute" not in ids def test_already_answered_filters_pending(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) @@ -272,6 +271,56 @@ def test_walks_into_for_each_inner_activities(self): assert question is not None assert "inner_copy" in question.affected_task_keys + def test_motif_consolidation_question_emitted_per_detected_motif(self): + """Each detected motif produces a ``consolidate_motif:`` question.""" + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + motifs = [ + DetectedMotif( + definition=MOTIF_INCREMENTAL_LOAD_WATERMARK, + matched_activities=["WatermarkLookup", "DeltaCopy"], + source_type_hint="database", + confidence_notes=["Detector matched Lookup→Copy→SP chain"], + ) + ] + pending = gather_questions(pipeline, motifs) + ids = {q.question_id for q in pending.questions} + assert "consolidate_motif:incremental_load_watermark" in ids + motif_question = next( + q for q in pending.questions if q.question_id == "consolidate_motif:incremental_load_watermark" + ) + assert motif_question.default == "keep" + assert {opt.value for opt in motif_question.options} == {"keep", "consolidate"} + assert "WatermarkLookup" in motif_question.affected_task_keys + # Confidence note must surface in the rationale so the agent can quote it + assert "Detector matched Lookup→Copy→SP chain" in motif_question.rationale + + def test_motif_consolidation_question_filtered_by_answer(self): + """Once answered the per-motif question must drop out of pending.""" + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + motifs = [ + DetectedMotif( + definition=MOTIF_INCREMENTAL_LOAD_WATERMARK, + matched_activities=["WatermarkLookup"], + source_type_hint=None, + confidence_notes=[], + ) + ] + pending = gather_questions( + pipeline, + motifs, + answers={"consolidate_motif:incremental_load_watermark": "consolidate"}, + ) + ids = {q.question_id for q in pending.questions} + assert "consolidate_motif:incremental_load_watermark" not in ids + + def test_motif_consolidation_validate_answer_accepts_keep_or_consolidate(self): + assert validate_answer("consolidate_motif:rest_api_pagination", "keep") == "keep" + assert validate_answer("consolidate_motif:rest_api_pagination", "consolidate") == "consolidate" + + def test_motif_consolidation_validate_answer_rejects_unknown_value(self): + with pytest.raises(ValueError, match="Invalid answer"): + validate_answer("consolidate_motif:rest_api_pagination", "merge") + class TestValidateAnswer: def test_accepts_allowed_value(self): @@ -302,14 +351,19 @@ def test_classic_compute_routes_copy_to_multi_node_cluster(self): assert modified.tasks[0].compute_mode == COMPUTE_MODE_CLASSIC_MULTI_NODE assert modified.tasks[1].compute_mode == COMPUTE_MODE_CLASSIC_SINGLE_NODE - def test_databricks_task_serverless_stamps_serverless(self): + def test_databricks_task_always_inherits_linked_service_cluster(self): + """DatabricksNotebook activities always inherit the source linked-service cluster + binding; the serverless replacement option was removed.""" pipeline = Pipeline(name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")]) - prefs = TranslationPreferences(databricks_task_compute="serverless") - modified = apply_preferences(pipeline, prefs) - assert modified.tasks[0].compute_mode == COMPUTE_MODE_SERVERLESS + modified = apply_preferences(pipeline, TranslationPreferences()) + assert modified.tasks[0].compute_mode == COMPUTE_MODE_INHERIT - def test_databricks_task_existing_stamps_inherit(self): - pipeline = Pipeline(name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")]) + def test_spark_python_always_inherits_linked_service_cluster(self): + """DatabricksSparkPython activities always inherit the source linked-service cluster + binding; the serverless replacement option was removed.""" + pipeline = Pipeline( + name="p", tasks=[SparkPythonActivity(**_make_base("py"), python_file="dbfs:/scripts/etl.py")] + ) modified = apply_preferences(pipeline, TranslationPreferences()) assert modified.tasks[0].compute_mode == COMPUTE_MODE_INHERIT @@ -459,7 +513,6 @@ def test_preferences_survive_json_roundtrip(self): copy_activity_paradigm="sdp", non_databricks_task_compute="classic", use_lakeflow_connectors="lakeflow_connect", - databricks_task_compute="serverless", ) stamped = apply_preferences(pipeline, prefs) roundtripped, _ = pipeline_dict_to_ir(json.loads(json.dumps(_pipeline_to_dict(stamped), default=str))) @@ -467,7 +520,9 @@ def test_preferences_survive_json_roundtrip(self): assert roundtripped.tasks[0].target_format == "sdp" assert roundtripped.tasks[0].use_lakeflow_connector is True assert roundtripped.tasks[0].compute_mode == COMPUTE_MODE_CLASSIC_MULTI_NODE - assert roundtripped.tasks[1].compute_mode == COMPUTE_MODE_SERVERLESS + # NotebookActivity always inherits the linked-service cluster binding now + # that the serverless replacement option has been removed. + assert roundtripped.tasks[1].compute_mode == COMPUTE_MODE_INHERIT class TestMigrationInputSession: @@ -1123,23 +1178,6 @@ def test_lakeflow_connect_emits_connection_setup_notebook(self, tmp_path: Path): assert "orchestra_copy_a_connection" in body assert "SQLSERVER" in body - def test_serverless_existing_notebook_skips_default_cluster_bind(self, tmp_path: Path): - import yaml - - from flowx.bundler.dab_writer import write_bundle - from flowx.preparer.workflow_preparer import prepare_workflow - - pipeline = Pipeline( - name="job", - tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/existing")], - ) - stamped = apply_preferences(pipeline, TranslationPreferences(databricks_task_compute="serverless")) - workflow = prepare_workflow(stamped) - write_bundle(workflow, tmp_path) - job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) - task = job_yml["resources"]["jobs"]["job"]["tasks"][0] - assert "job_cluster_key" not in task - def test_existing_default_binds_to_default_cluster(self, tmp_path: Path): import yaml diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index 605dd89..bddcf21 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -261,6 +261,38 @@ def test_load_report_handles_aggregated_translations_format(self, tmp_path): assert task_keys == {"pause", "run_nb"} +class TestSetupMd: + def test_parameter_approximations_render_to_setup_md(self, tmp_path): + pipeline = Pipeline( + name="approx_pipeline", + tasks=[ + NotebookActivity( + name="Score", + task_key="score", + notebook_path="/Shared/score", + base_parameters={"scoring_date": "{{job.start_time.iso_date}}"}, + parameter_approximations=[ + { + "widget_name": "scoring_date", + "raw_expression": "@formatDateTime(utcnow(), 'yyyy-MM-dd')", + "replacement": "{{job.start_time.iso_date}}", + "note": "Mapped ADF `utcnow()` to the Databricks job start time.", + } + ], + ), + ], + ) + wf = prepare_workflow(pipeline) + write_bundle(wf, tmp_path) + setup_md = (tmp_path / "SETUP.md").read_text() + assert "## Parameter substitutions" in setup_md + assert "`score`" in setup_md + assert "`scoring_date`" in setup_md + assert "@formatDateTime(utcnow(), 'yyyy-MM-dd')" in setup_md + assert "{{job.start_time.iso_date}}" in setup_md + assert "Mapped ADF `utcnow()`" in setup_md + + class TestSetupGenerator: def test_secrets_setup_notebook_content(self): from flowx.bundler.setup_generator import generate_setup_tasks diff --git a/tests/unit/test_expression_parser.py b/tests/unit/test_expression_parser.py index b88bfdf..d672c62 100644 --- a/tests/unit/test_expression_parser.py +++ b/tests/unit/test_expression_parser.py @@ -153,22 +153,32 @@ def test_item(self): class TestUtcNow: def test_utcnow_no_format(self): - # ``@utcNow()`` resolves to Python ``datetime.now(...).isoformat()`` - # rather than the DAB ref ``{{job.start_time.iso_datetime}}`` so - # compositions like ``@formatDateTime(utcNow(), '...')`` chain - # correctly. DAB does not evaluate ADF expressions, so wrapping a - # DAB ref in another ADF function would emit broken YAML. + # ``@utcNow()`` maps to the Databricks job start time so the result + # lands directly in DAB YAML. The translator attaches a note so + # the bundler can surface the activity-vs-job-start skew caveat in + # SETUP.md. result = resolve_expression("@utcNow()", _context()) assert result is not None - assert result.kind == "notebook_code" - assert result.value == "datetime.now(timezone.utc).isoformat()" + assert result.kind == "dab_ref" + assert result.value == "{{job.start_time.iso_datetime}}" + assert any("utcnow" in note.lower() for note in result.notes) - def test_utcnow_with_format(self): + def test_utcnow_with_known_iso_format(self): result = resolve_expression("@utcNow('yyyy-MM-dd')", _context()) assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.start_time.iso_date}}" + assert any("utcnow" in note.lower() for note in result.notes) + + def test_utcnow_with_unknown_format_falls_back_to_notebook_code(self): + # ``yyyyMMdd`` (no separators) is not in the DAB dynamic-value + # vocabulary, so flowx keeps the legacy Python strftime path. + result = resolve_expression("@utcNow('yyyyMMdd')", _context()) + assert result is not None assert result.kind == "notebook_code" assert "strftime" in result.value - assert "%Y-%m-%d" in result.value + assert "%Y%m%d" in result.value + assert result.notes == [] def test_utcnow_expression_dict(self): result = resolve_expression( @@ -176,7 +186,8 @@ def test_utcnow_expression_dict(self): _context(), ) assert result is not None - assert result.kind == "notebook_code" + assert result.kind == "dab_ref" + assert result.value == "{{job.start_time.iso_date}}" class TestConcat: @@ -197,10 +208,13 @@ def test_concat_with_variable(self): assert "runDate" in result.value def test_concat_with_utcnow(self): + # ``utcNow('yyyy-MM-dd')`` is now a DAB ref, so concat wraps it as a + # widget read. The result is still notebook_code because concat + # composes Python strings. result = resolve_expression("@concat('date_', utcNow('yyyy-MM-dd'))", _context()) assert result is not None assert result.kind == "notebook_code" - assert "strftime" in result.value + assert "dbutils.widgets.get('iso_date')" in result.value def test_concat_with_pipeline_param(self): result = resolve_expression( @@ -743,13 +757,11 @@ def test_parse_expression_for_dab_returns_ref(self): result = parse_expression_for_dab("@pipeline().RunId") assert result == "{{job.run_id}}" - def test_parse_expression_for_dab_returns_none_for_utcnow(self): - # ``@utcNow()`` now resolves to notebook_code so it composes correctly - # with other ADF time functions. ``parse_expression_for_dab`` only - # returns dab_ref kinds, so utcNow now yields ``None`` (the caller - # routes through the notebook_code path instead). + def test_parse_expression_for_dab_returns_ref_for_utcnow(self): + # ``@utcNow()`` maps to the Databricks job start time dynamic value + # so it can land in DAB YAML directly. result = parse_expression_for_dab("@utcNow()") - assert result is None + assert result == "{{job.start_time.iso_datetime}}" def test_parse_expression_for_dab_returns_none_for_non_expression(self): result = parse_expression_for_dab("plain_string") diff --git a/tests/unit/test_helpers.py b/tests/unit/test_helpers.py index aea8086..269076e 100644 --- a/tests/unit/test_helpers.py +++ b/tests/unit/test_helpers.py @@ -57,10 +57,17 @@ def test_interpolated_resolves(self): def test_notebook_code_returns_raw_for_manual_handling(self): """Expressions that resolve to Python code return raw text (manual handling).""" - raw = "@formatDateTime(utcnow(), 'yyyy-MM-dd')" + # Pick a format that intentionally does NOT map to a DAB dynamic + # value so the resolver falls back to notebook_code. + raw = "@formatDateTime(utcnow(), 'dd MMM yyyy')" result = resolve_param_value(raw) assert result == raw + def test_formatdatetime_utcnow_known_format_resolves_to_dab_ref(self): + """``formatDateTime(utcnow(), '')`` short-circuits to a DAB ref.""" + result = resolve_param_value("@formatDateTime(utcnow(), 'yyyy-MM-dd')") + assert result == "{{job.start_time.iso_date}}" + class TestBuildNotebookTaskArtifacts: def test_returns_task_dict_and_one_notebook(self): diff --git a/tests/unit/test_ir_rewriter.py b/tests/unit/test_ir_rewriter.py new file mode 100644 index 0000000..bdadef6 --- /dev/null +++ b/tests/unit/test_ir_rewriter.py @@ -0,0 +1,234 @@ +"""Unit tests for the whole-IR expression rewriter.""" + +from __future__ import annotations + +from flowx.models.ir import ( + CopyActivity, + Dependency, + ForEachActivity, + IfConditionActivity, + LookupActivity, + NotebookActivity, + Pipeline, + SetVariableActivity, + SwitchActivity, + SwitchCase, + WebActivity, +) +from flowx.parser.ir_rewriter import rewrite_pipeline_expressions + + +def _base(task_key: str, name: str | None = None) -> dict[str, object]: + return {"name": name or task_key, "task_key": task_key} + + +class TestRewritePipelineExpressions: + def test_rewrites_sql_inside_source_properties_dict(self): + """Raw @{} tokens inside Copy's source_properties must be rewritten.""" + copy = CopyActivity( + **_base("copy_orders"), + source_type="AzureSqlSource", + source_properties={ + "sql": "SELECT * FROM dbo.orders WHERE modified_dt >= '@{pipeline().parameters.watermark}'" + }, + sink_type="ParquetSink", + ) + pipeline = Pipeline(name="p", parameters=[{"name": "watermark", "default": None}], tasks=[copy]) + rewritten = rewrite_pipeline_expressions(pipeline) + sql = rewritten.tasks[0].source_properties["sql"] + assert "@{" not in sql + assert "{{job.parameters.watermark}}" in sql + + def test_rewrites_strings_inside_lists(self): + """@{} tokens inside list-valued fields are rewritten too.""" + # CopyActivity has no list-of-strings field, so use a list inside a dict + copy = CopyActivity( + **_base("copy"), + source_type="AzureSqlSource", + source_properties={ + "partitions": ["p_@{pipeline().parameters.region}_1", "p_@{pipeline().parameters.region}_2"] + }, + ) + pipeline = Pipeline(name="p", tasks=[copy]) + rewritten = rewrite_pipeline_expressions(pipeline) + partitions = rewritten.tasks[0].source_properties["partitions"] + assert all("@{" not in p for p in partitions) + assert all("{{job.parameters.region}}" in p for p in partitions) + + def test_rewrites_web_activity_url_body_and_headers(self): + """Web activities embed @{} tokens in url/body/headers.""" + web = WebActivity( + **_base("call_api"), + url="https://api.example.com/jobs/@{pipeline().parameters.job_id}", + method="POST", + body='{"run":"@{pipeline().RunId}"}', + headers={"X-Tenant": "@{pipeline().parameters.tenant}"}, + ) + pipeline = Pipeline(name="p", tasks=[web]) + rewritten = rewrite_pipeline_expressions(pipeline) + task = rewritten.tasks[0] + assert "{{job.parameters.job_id}}" in task.url + assert "{{job.run_id}}" in task.body + assert "{{job.parameters.tenant}}" in task.headers["X-Tenant"] + + def test_recurses_into_foreach_inner_activities(self): + inner = CopyActivity( + **_base("inner_copy"), + source_type="AzureSqlSource", + source_properties={"sql": "SELECT * FROM @{item().table_name}"}, + ) + for_each = ForEachActivity( + **_base("loop"), + items_expression="@activity('lookup').output.value", + inner_activities=[inner], + ) + pipeline = Pipeline(name="p", tasks=[for_each]) + rewritten = rewrite_pipeline_expressions(pipeline) + inner_rewritten = rewritten.tasks[0].inner_activities[0] + assert "@{" not in inner_rewritten.source_properties["sql"] + assert "{{input.table_name}}" in inner_rewritten.source_properties["sql"] + + def test_recurses_into_if_condition_branches(self): + true_branch = NotebookActivity( + **_base("true_nb"), + notebook_path="/Shared/promote", + base_parameters={"score": "@{activity('lookup').output.firstRow.quality_score}"}, + ) + false_branch = NotebookActivity( + **_base("false_nb"), + notebook_path="/Shared/remediate", + base_parameters={"score": "@{activity('lookup').output.firstRow.quality_score}"}, + ) + ifc = IfConditionActivity( + **_base("gate"), + op="greaterOrEquals", + left="@activity('lookup').output.firstRow.quality_score", + right="0.95", + if_true_activities=[true_branch], + if_false_activities=[false_branch], + ) + pipeline = Pipeline(name="p", tasks=[ifc]) + rewritten = rewrite_pipeline_expressions(pipeline) + ifc_out = rewritten.tasks[0] + for nb in (ifc_out.if_true_activities[0], ifc_out.if_false_activities[0]): + assert "@{" not in nb.base_parameters["score"] + + def test_recurses_into_switch_cases(self): + case_nb = NotebookActivity( + **_base("case_nb"), + notebook_path="/Shared/h", + base_parameters={"d": "@{pipeline().parameters.dt}"}, + ) + default_nb = NotebookActivity( + **_base("default_nb"), + notebook_path="/Shared/d", + base_parameters={"d": "@{pipeline().parameters.dt}"}, + ) + switch = SwitchActivity( + **_base("sw"), + on_expression="@pipeline().parameters.mode", + cases=[SwitchCase(value="full", activities=[case_nb])], + default_activities=[default_nb], + ) + pipeline = Pipeline(name="p", tasks=[switch]) + rewritten = rewrite_pipeline_expressions(pipeline) + case_out = rewritten.tasks[0].cases[0].activities[0] + default_out = rewritten.tasks[0].default_activities[0] + assert "{{job.parameters.dt}}" in case_out.base_parameters["d"] + assert "{{job.parameters.dt}}" in default_out.base_parameters["d"] + + def test_skips_linked_service_definition_raw_field(self): + """linked_service_definition holds raw ADF input and must not be touched.""" + raw_ls = {"type": "AzureSqlDatabase", "connectionString": "@{pipeline().parameters.cs}"} + nb = NotebookActivity( + **_base("nb"), + notebook_path="/Shared/x", + linked_service_definition=raw_ls, + ) + pipeline = Pipeline(name="p", tasks=[nb]) + rewritten = rewrite_pipeline_expressions(pipeline) + # linked_service_definition stays verbatim — raw ADF passthrough + assert rewritten.tasks[0].linked_service_definition == raw_ls + + def test_unresolved_tokens_remain_and_surface_as_warning(self): + """An expression the parser can't resolve must (a) stay in the output and + (b) be recorded as a warning so the user sees the gap.""" + copy = CopyActivity( + **_base("copy"), + source_type="AzureSqlSource", + source_properties={"sql": "SELECT * FROM @{activity('NonexistentActivity').output.unknownField}"}, + ) + pipeline = Pipeline(name="p", tasks=[copy]) + warnings: list[str] = [] + rewritten = rewrite_pipeline_expressions(pipeline, warnings=warnings) + # Unresolved tokens get replaced by their best-effort dab_ref by the + # parser (activity-output → tasks.X.values...). When even the parser + # cannot produce *anything* it leaves the original @{} verbatim; in + # that case we must surface a warning. The activity-output regex + # *does* match unknownField, so this particular case resolves + # silently. Confirm the more pathological "completely unknown + # function" case raises a warning instead. + del rewritten + + weird_copy = CopyActivity( + **_base("weird"), + source_type="AzureSqlSource", + source_properties={"sql": "SELECT @{nonsense('foo')} FROM t"}, + ) + warnings = [] + rewritten = rewrite_pipeline_expressions(Pipeline(name="p", tasks=[weird_copy]), warnings=warnings) + assert "@{" in rewritten.tasks[0].source_properties["sql"] + assert any("Unresolved ADF expression" in w for w in warnings) + + def test_identifier_fields_are_left_alone(self): + """task_key and name must never be rewritten — they are reference identifiers.""" + nb = NotebookActivity( + name="@{pipeline().parameters.foo}", # intentionally bizarre + task_key="weird_name_with_@{x}_token", + notebook_path="/Shared/x", + ) + pipeline = Pipeline(name="p", tasks=[nb]) + rewritten = rewrite_pipeline_expressions(pipeline) + assert rewritten.tasks[0].name == "@{pipeline().parameters.foo}" + assert rewritten.tasks[0].task_key == "weird_name_with_@{x}_token" + + def test_pipeline_with_no_tokens_returns_equivalent_pipeline(self): + nb = NotebookActivity( + **_base("nb"), + notebook_path="/Shared/etl", + base_parameters={"date": "2026-05-30"}, + ) + pipeline = Pipeline(name="p", tasks=[nb]) + rewritten = rewrite_pipeline_expressions(pipeline) + # Same logical content + assert rewritten.tasks[0].base_parameters == {"date": "2026-05-30"} + + def test_variable_tokens_resolve_via_set_variable_setter(self): + """@{variables('x')} should pick up the SetVariableActivity's task_key.""" + setter = SetVariableActivity( + **_base("set_x"), + variable_name="x", + variable_value="42", + ) + consumer = NotebookActivity( + **_base("nb"), + notebook_path="/Shared/q", + base_parameters={"x_value": "@{variables('x')}"}, + depends_on=[Dependency(task_key="set_x")], + ) + pipeline = Pipeline(name="p", tasks=[setter, consumer]) + rewritten = rewrite_pipeline_expressions(pipeline) + consumer_out = rewritten.tasks[1] + assert "@{" not in consumer_out.base_parameters["x_value"] + # Resolves to task value reference for the setter + assert "tasks.set_x.values.x" in consumer_out.base_parameters["x_value"] + + def test_lookup_source_query_rewritten(self): + lookup = LookupActivity( + **_base("lookup_w"), + source_type="AzureSqlSource", + source_query="SELECT MAX(modified_dt) FROM dbo.@{pipeline().parameters.table_name}", + ) + pipeline = Pipeline(name="p", tasks=[lookup]) + rewritten = rewrite_pipeline_expressions(pipeline) + assert "{{job.parameters.table_name}}" in rewritten.tasks[0].source_query diff --git a/tests/unit/test_preparers.py b/tests/unit/test_preparers.py index 17b318a..302eadf 100644 --- a/tests/unit/test_preparers.py +++ b/tests/unit/test_preparers.py @@ -166,6 +166,66 @@ def test_prepare_notebook_resolves_expression_params(self): assert params["env"] == "dev" assert params["trigger_time"] == "{{job.start_time.iso_datetime}}" + def test_prepare_notebook_emits_libraries(self): + libraries = [ + {"whl": "dbfs:/libs/pkg.whl"}, + {"pypi": {"package": "requests"}}, + ] + activity = NotebookActivity( + **_make_base("NB", "nb"), + notebook_path="/Shared/nb", + libraries=libraries, + ) + prepared = prepare_activity(activity) + assert prepared.task["libraries"] == libraries + + def test_prepare_notebook_emits_existing_cluster_id(self): + activity = NotebookActivity( + **{**_make_base("NB", "nb"), "existing_cluster_id": "1234-567890-abcde123"}, + notebook_path="/Shared/nb", + ) + prepared = prepare_activity(activity) + assert prepared.task["existing_cluster_id"] == "1234-567890-abcde123" + assert "job_cluster_key" not in prepared.task + + def test_prepare_notebook_surfaces_parameter_approximations(self): + activity = NotebookActivity( + **_make_base("Score", "score"), + notebook_path="/Shared/score", + base_parameters={"scoring_date": "{{job.start_time.iso_date}}"}, + parameter_approximations=[ + { + "widget_name": "scoring_date", + "raw_expression": "@formatDateTime(utcnow(), 'yyyy-MM-dd')", + "replacement": "{{job.start_time.iso_date}}", + "note": "Mapped ADF `utcnow()` to the Databricks job start time.", + } + ], + ) + prepared = prepare_activity(activity) + assert len(prepared.parameter_approximations) == 1 + approximation = prepared.parameter_approximations[0] + assert approximation.task_key == "score" + assert approximation.widget_name == "scoring_date" + assert approximation.raw_expression == "@formatDateTime(utcnow(), 'yyyy-MM-dd')" + assert approximation.replacement == "{{job.start_time.iso_date}}" + + def test_prepare_notebook_existing_cluster_id_wins_over_default_cluster_bind(self, monkeypatch): + """When a downloaded workspace notebook would otherwise bind to default_cluster, + an explicit existing_cluster_id from the linked service takes precedence.""" + monkeypatch.setattr(workspace_downloader, "_downloads_enabled", True) + monkeypatch.setattr( + "flowx.preparer.activity_preparers.notebook.download_notebook", + lambda path: "# Databricks notebook source\nprint('hi')\n", + ) + activity = NotebookActivity( + **{**_make_base("NB", "nb"), "existing_cluster_id": "9876-543210-zyxwv987"}, + notebook_path="/Shared/ETL/transform", + ) + prepared = prepare_activity(activity) + assert prepared.task["existing_cluster_id"] == "9876-543210-zyxwv987" + assert "job_cluster_key" not in prepared.task + class TestCopyPreparer: def test_prepare_copy_generates_notebook(self): @@ -229,6 +289,19 @@ def test_prepare_spark_python_task(self): assert "scripts/etl.py" in prepared.notebooks[0].relative_path assert "dbfs:/scripts/etl.py" in prepared.notebooks[0].content + def test_prepare_spark_python_emits_libraries(self): + libraries = [ + {"pypi": {"package": "pandas"}}, + {"maven": {"coordinates": "org.example:lib:1.0"}}, + ] + activity = SparkPythonActivity( + **_make_base("Py Task", "py_task"), + python_file="dbfs:/scripts/etl.py", + libraries=libraries, + ) + prepared = prepare_activity(activity) + assert prepared.task["libraries"] == libraries + class TestLookupPreparer: def test_prepare_lookup_generates_notebook(self): diff --git a/tests/unit/test_translators.py b/tests/unit/test_translators.py index d213604..bab80ab 100644 --- a/tests/unit/test_translators.py +++ b/tests/unit/test_translators.py @@ -176,6 +176,110 @@ def test_translate_notebook_no_params(self): assert isinstance(result, NotebookActivity) assert result.base_parameters == {} + def test_translate_notebook_passes_libraries_through(self): + from flowx.translator.activity_translators.notebook import translate + + libraries = [ + {"jar": "dbfs:/libs/util.jar"}, + {"whl": "dbfs:/libs/pkg-1.0-py3-none-any.whl"}, + {"pypi": {"package": "requests==2.31.0"}}, + {"maven": {"coordinates": "org.jsoup:jsoup:1.7.2", "exclusions": ["slf4j:slf4j"]}}, + {"cran": {"package": "ada", "repo": "https://cran.us.r-project.org"}}, + ] + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb", "libraries": libraries}, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.libraries == libraries + + def test_translate_notebook_captures_utcnow_approximation(self): + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Score", + "DatabricksNotebook", + { + "notebookPath": "/Shared/score", + "baseParameters": { + "scoring_date": {"value": "@formatDateTime(utcnow(), 'yyyy-MM-dd')", "type": "Expression"}, + "env": "dev", + }, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.base_parameters["scoring_date"] == "{{job.start_time.iso_date}}" + assert result.base_parameters["env"] == "dev" + assert len(result.parameter_approximations) == 1 + approximation = result.parameter_approximations[0] + assert approximation["widget_name"] == "scoring_date" + assert approximation["raw_expression"] == "@formatDateTime(utcnow(), 'yyyy-MM-dd')" + assert approximation["replacement"] == "{{job.start_time.iso_date}}" + assert "utcnow" in approximation["note"].lower() + + +class TestCommonAttributes: + def test_existing_cluster_id_extracted_from_linked_service(self): + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="AzureDatabricks_LS", + type="AzureDatabricks", + properties={ + "typeProperties": { + "existingClusterId": "1234-567890-abcde123", + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"AzureDatabricks_LS": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference(reference_name="AzureDatabricks_LS"), + ) + kwargs = _build_base_kwargs(activity, definitions) + assert kwargs["existing_cluster_id"] == "1234-567890-abcde123" + assert kwargs["cluster"] == {"existing_cluster_id": "1234-567890-abcde123"} + + def test_existing_cluster_id_none_when_linked_service_uses_new_cluster(self): + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="AzureDatabricks_LS", + type="AzureDatabricks", + properties={ + "typeProperties": { + "newClusterSparkVersion": "15.4.x-scala2.12", + "newClusterNumOfWorker": 2, + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"AzureDatabricks_LS": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference(reference_name="AzureDatabricks_LS"), + ) + kwargs = _build_base_kwargs(activity, definitions) + assert kwargs["existing_cluster_id"] is None + class TestSparkJarTranslator: def test_translate_spark_jar(self): @@ -211,6 +315,22 @@ def test_translate_spark_python(self): assert result.python_file == "dbfs:/scripts/etl.py" assert result.parameters == ["--mode", "batch"] + def test_translate_spark_python_passes_libraries_through(self): + from flowx.translator.activity_translators.spark_python import translate + + libraries = [ + {"egg": "dbfs:/libs/util.egg"}, + {"pypi": {"package": "pandas", "repo": "https://pypi.example.com"}}, + ] + activity = _make_activity( + "Run Python", + "DatabricksSparkPython", + {"pythonFile": "dbfs:/scripts/etl.py", "libraries": libraries}, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, SparkPythonActivity) + assert result.libraries == libraries + class TestLookupTranslator: def test_translate_lookup_first_row(self): @@ -512,6 +632,8 @@ def test_translate_set_variable_literal(self): def test_translate_set_variable_utcnow(self): from flowx.translator.activity_translators.set_variable import translate + # ``utcNow('yyyy-MM-dd')`` now maps to a DAB dynamic value, so the + # SetVariable result is dab_ref rather than notebook_code. activity = _make_activity( "SetRunDate", "SetVariable", @@ -519,6 +641,20 @@ def test_translate_set_variable_utcnow(self): ) result, context = translate(activity, _base_kwargs("SetRunDate"), _context(), _EMPTY_DEFS) assert isinstance(result, SetVariableActivity) + assert result.value_kind == "dab_ref" + assert result.variable_value == "{{job.start_time.iso_date}}" + + def test_translate_set_variable_utcnow_unknown_format(self): + from flowx.translator.activity_translators.set_variable import translate + + # Unrecognised format falls back to the legacy notebook_code path. + activity = _make_activity( + "SetRunDate", + "SetVariable", + {"variableName": "runDate", "value": {"type": "Expression", "value": "@utcNow('yyyyMMdd')"}}, + ) + result, context = translate(activity, _base_kwargs("SetRunDate"), _context(), _EMPTY_DEFS) + assert isinstance(result, SetVariableActivity) assert result.value_kind == "notebook_code" assert result.notebook_code is not None assert "strftime" in result.notebook_code From 6b318320ee3772e2a3b29b6ccc5fc335985546de Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Tue, 9 Jun 2026 13:24:03 -0400 Subject: [PATCH 10/77] Revert "Update GitHub actions (#10)" This reverts commit 3c9cb71f274c6db55a3bfb4cd06a3b7768f2fc12. --- .github/workflows/skill-eval.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/skill-eval.yml diff --git a/.github/workflows/skill-eval.yml b/.github/workflows/skill-eval.yml new file mode 100644 index 0000000..c1a93d6 --- /dev/null +++ b/.github/workflows/skill-eval.yml @@ -0,0 +1,30 @@ +name: skill-eval + +on: + pull_request: + workflow_dispatch: + +jobs: + integration: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Scrub internal proxy URLs from uv.lock + run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock + - run: uv sync --frozen + + # Azure login for live ADF integration tests. + # Requires AZURE_CREDENTIALS secret configured with a service principal + # that has Reader access to the flowx-rg resource group. + # Tests skip gracefully when credentials are not available. + - name: Azure Login + if: ${{ secrets.AZURE_CREDENTIALS != '' }} + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - run: make integration From 8ce994414f9b9cb30c8c2784e245861d8157444a Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Tue, 9 Jun 2026 14:05:31 -0400 Subject: [PATCH 11/77] Improve control flow conversion (#2) Improves conversion of control flow, expression-based parameters, and schedule triggers. Co-authored-by: Isaac --- .claude-plugin/plugin.json | 1 + .github/workflows/push.yml | 5 + AGENTS.md | 11 +- FIX_0603_CHANGES.md | 672 ++++++++ Makefile | 10 +- docs/content/docs/installation.mdx | 50 +- requirements.txt | 32 + scripts/bootstrap.sh | 111 ++ skills/ingest/SKILL.md | 22 + skills/migrate/SKILL.md | 29 +- skills/prepare/SKILL.md | 22 + skills/setup/SKILL.md | 94 ++ skills/translate/SKILL.md | 54 +- src/orchestra/bundler/dab_writer.py | 452 +++++- src/orchestra/bundler/inner_job_params.py | 57 +- src/orchestra/bundler/prereqs_writer.py | 217 ++- src/orchestra/bundler/setup_generator.py | 19 +- src/orchestra/models/adf_ast.py | 38 + src/orchestra/models/ir.py | 181 ++- src/orchestra/parser/adf_loader.py | 46 + src/orchestra/parser/expression_parser.py | 383 ++++- .../preparer/activity_preparers/for_each.py | 225 ++- .../activity_preparers/if_condition.py | 109 +- .../preparer/activity_preparers/notebook.py | 164 +- .../activity_preparers/set_variable.py | 21 +- .../preparer/activity_preparers/switch.py | 127 +- .../activity_preparers/web_activity.py | 126 +- src/orchestra/preparer/code_generator.py | 213 ++- src/orchestra/preparer/workflow_preparer.py | 202 ++- .../activity_translators/execute_pipeline.py | 60 +- .../activity_translators/for_each.py | 24 + .../activity_translators/if_condition.py | 217 ++- .../translator/activity_translators/lookup.py | 167 ++ .../activity_translators/notebook.py | 151 +- .../activity_translators/resolve.py | 82 +- .../activity_translators/set_variable.py | 74 + .../translator/activity_translators/switch.py | 49 +- src/orchestra/translator/engine.py | 755 ++++++++- tests/unit/test_bundler.py | 680 ++++++++- tests/unit/test_code_generator.py | 81 + tests/unit/test_expression_parser.py | 294 +++- tests/unit/test_for_each_inner_job_params.py | 148 ++ tests/unit/test_preparers.py | 473 +++++- tests/unit/test_prereqs_writer.py | 79 + tests/unit/test_resolve_field.py | 5 +- tests/unit/test_translators.py | 1346 +++++++++++++++++ 46 files changed, 8123 insertions(+), 255 deletions(-) create mode 100644 FIX_0603_CHANGES.md create mode 100644 requirements.txt create mode 100755 scripts/bootstrap.sh create mode 100644 skills/setup/SKILL.md create mode 100644 tests/unit/test_for_each_inner_job_params.py create mode 100644 tests/unit/test_prereqs_writer.py diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 28e8695..cf9403a 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -9,6 +9,7 @@ "license": "MIT", "keywords": ["adf", "databricks", "migration", "dabs", "lakeflow", "orchestration", "azure-data-factory"], "skills": [ + "./skills/setup", "./skills/ingest", "./skills/translate", "./skills/prepare", diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 97d7338..71081c5 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -18,6 +18,11 @@ jobs: run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock - run: uv sync --frozen - run: make test + - name: Verify requirements.txt is in sync with the lockfile + run: | + make requirements + git diff --exit-code -- requirements.txt \ + || { echo "requirements.txt is stale. Run 'make requirements' (or 'make precommit') and commit it."; exit 1; } fmt: runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index dc79f38..6d5d79e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,13 +3,22 @@ ## Quick Command Reference ```bash -make dev # Install dependencies +make dev # Install dependencies (development; uses uv) make test # Unit tests make integration # Integration tests (requires ADF fixtures) make fmt # Format + lint (ruff + mypy) make clean # Remove build artifacts ``` +To run the **plugin skills** (ingest/translate/prepare/migrate) without a uv-based dev setup, +bootstrap a self-contained virtual environment with pip via the `setup` skill or directly: + +```bash +bash scripts/bootstrap.sh # creates .venv and pip-installs requirements.txt +# then run plugin code with src/ on PYTHONPATH: +PYTHONPATH=src .venv/bin/python -m flowx.adapter inputs ingest +``` + ## Project Overview Flowx is an agent plugin that translates Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs via Declarative Automation Bundles (DABs). diff --git a/FIX_0603_CHANGES.md b/FIX_0603_CHANGES.md new file mode 100644 index 0000000..5f1e5f0 --- /dev/null +++ b/FIX_0603_CHANGES.md @@ -0,0 +1,672 @@ +# FIX_0603 Changes + +Track of changes applied to branch `fix-0603` from the validation-implementation workflow. + +## Iteration 1 — 2026-06-03 + +Implemented 9 of 13 plan changes (4 deferred to a future iteration: see "Deferred" below). + +### Change: `expr-resolver-globalparams-and-wrappers` (P0) + +- **Title**: Teach expression resolver about factory globalParameters, @json/@string/@array no-op wrappers, trailing-whitespace expressions, item() safe-nav, and @linkedService().X refs. +- **Rationale**: Drives NB-1, NB-4 (partial), NB-5, CF-003, LSC-001, LSC-005, VAR-003, VAR-005 — five separate gaps trace to the same resolver weaknesses. Loading globalParameters once at parse time and stripping leading-`@` wrappers unblocks all of them without refactor. +- **Files changed**: + - `src/flowx/models/adf_ast.py` — added `AdfDefinitions.global_parameters`. + - `src/flowx/models/ir.py` — added `TranslationContext.global_parameters` and `linked_service_parameters`, plus `get_global_parameter` / `get_linked_service_parameter` / `with_linked_service_parameters` helpers. + - `src/flowx/parser/adf_loader.py` — added `factory_dir` loader and `_parse_factory_global_parameters` helper; same for the ARM-template branch. + - `src/flowx/parser/expression_parser.py` — added `_resolve_pipeline_global_param`, `_resolve_linked_service_param`, `_resolve_item_safe_nav`, `_unwrap_noop_call`; widened `_FUNCTION_CALL_RE` to accept trailing whitespace. + - `src/flowx/translator/engine.py` — thread `definitions.global_parameters` into the seeded `TranslationContext`. +- **Tests added**: 5 new test classes in `tests/unit/test_expression_parser.py` covering global parameters (literal substitution + dict-shape + missing fallback + concat reduction), no-op wrappers (json/string/array), trailing-whitespace function calls, item safe-nav, and linked-service parameter resolution. +- **Commit**: `2cd8929` + +### Change: `linked-service-parameter-resolution` (P0) + +- **Title**: Resolve `@linkedService().X` against activity-supplied parameters with LS defaultValue fallback. +- **Rationale**: NB-4 and LSC-001 — `AdfLinkedServiceReference` dropped the activity's `parameters` dict at parse time. 166/327 emitted databricks.yml files had `spark_version: '@linkedService().clusterVersion'` and failed to deploy. Also coerces `num_workers='1'` strings to int. +- **Files changed**: + - `src/flowx/models/adf_ast.py` — added `AdfLinkedServiceReference.parameters` field. + - `src/flowx/parser/adf_loader.py` — populate `parameters` from `linkedServiceName.parameters`. + - `src/flowx/translator/engine.py` — added `_resolve_ls_parameters`, `_substitute_ls_params`, `_coerce_int`; `_extract_cluster_config` now accepts `ls_param_overrides` and walks string values through `_substitute_ls_params` before reading fields. +- **Tests added**: `TestCommonAttributes.test_linked_service_parameter_overrides_cluster_version` in `tests/unit/test_translators.py`. +- **Commit**: `e33ca26` + +### Change: `linked-service-cluster-field-coverage` (P1) + +- **Title**: Extend `_extract_cluster_config` to lift `spark_env_vars`, `custom_tags`, `driver_node_type_id`, `init_scripts`, `data_security_mode`, `cluster_log_conf`; propagate to emitted clusters. +- **Rationale**: NB-3, LSC-003 — five LS keys lost; default-cluster builder never consulted per-task cluster. +- **Files changed**: + - `src/flowx/translator/engine.py` — `_extract_cluster_config` now lifts the extended fields. + - `src/flowx/bundler/dab_writer.py` — added `_infer_bundle_cluster_extras`; `_build_default_cluster` now accepts extras and merges them into `new_cluster`. +- **Tests added**: `TestCommonAttributes.test_extended_cluster_fields_propagated` in `tests/unit/test_translators.py`; `TestClusterExtrasPropagation.test_extras_merged_into_default_cluster` in `tests/unit/test_bundler.py`. +- **Commit**: `b033123` + +### Change: `library-resolution-and-stub-binding` (P0) + +- **Title**: Resolve library jar/whl/maven/pypi expressions; bind stub notebooks with jar libraries to a real cluster. +- **Rationale**: NB-1 + LSC-005 + NB-2 — library jar paths shipped as literal `@concat(...)`; stub tasks with libraries were skipped during cluster binding, so the Jobs API rejected them. +- **Files changed**: + - `src/flowx/translator/activity_translators/notebook.py` — added `_resolve_libraries` that walks each library entry through `resolve_expression`. + - `src/flowx/bundler/dab_writer.py` — `_bind_cluster_to_notebook_tasks` now binds the default cluster for any stub or serverless-mode task that ships jar/whl/maven/pypi libraries. +- **Tests added**: 2 tests in `tests/unit/test_translators.py` (library resolution + global parameter substitution); 3 tests in `tests/unit/test_bundler.py` under `TestStubLibraryBinding`. +- **Commit**: `9b1af19` + +### Change: `base-parameters-cleanup-and-stub-widgets` (P1) + +- **Title**: Strip unresolved ADF expressions from stub notebook base_parameters. +- **Rationale**: NB-5 — stub notebook tasks shipped raw `@string(coalesce(...))` strings as widget defaults; `dbutils.widgets.get` returns them verbatim, which fails at runtime. +- **Files changed**: + - `src/flowx/bundler/dab_writer.py` — dropped the `notebook_path.startswith('/')` gate on `_extract_manual_parameters_from_existing_notebook_tasks` so both absolute-path and bundle-relative stubs are walked. +- **Tests added**: `TestStubBaseParameterCleanup.test_stub_notebook_strips_unresolvable_adf_expression` in `tests/unit/test_bundler.py`. +- **Commit**: `e1ba336` + +### Change: `lookup-file-dataset-support` (P0) + +- **Title**: Resolve Lookup `typeProperties.dataset` and emit `spark.read` for file-source lookups instead of `spark.sql('')`. +- **Rationale**: Three gaps in the lookup dimension share root cause — translator never read `typeProperties.dataset`, code_generator emitted `spark.sql('')`, multiline JSON handling was missing. +- **Files changed**: + - `src/flowx/translator/activity_translators/lookup.py` — resolve `typeProperties.dataset` / `activity.inputs` to the bound `AdfDataset`; surface `dataset_type`, `container`, `folder_path`, `file_name`, `multiLineJson` onto `source_properties` for file-source dataset types. + - `src/flowx/preparer/code_generator.py` — added `_is_file_lookup`, `_file_lookup_body`; the existing JDBC / non-DB branches now fall through to the file-source branch when the dataset is file-shaped. +- **Tests added**: `TestLookupTranslator.test_translate_lookup_resolves_json_file_dataset` in `tests/unit/test_translators.py`; `TestGenerateLookupNotebook.test_file_source_lookup_emits_spark_read` in `tests/unit/test_code_generator.py`. +- **Commit**: `3b669f6` + +### Change: `foreach-inner-extra-tasks` (P0) + +- **Title**: Carry inner-activity `extra_tasks` through ForEach inner-job assembly so IfCondition/Switch branches survive. +- **Rationale**: CF-001 — for_each preparer's multi-child path appended only `child_prepared.task`, dropping `extra_tasks` where IfCondition/Switch branch bodies live. 12 pipelines lost branches. +- **Files changed**: + - `src/flowx/preparer/activity_preparers/for_each.py` — multi-child path now `.extend(child_prepared.extra_tasks)`; single-child path escalates to the sub-job pattern when the sole child contributes extra_tasks. +- **Tests added**: 2 tests in `tests/unit/test_preparers.py` under `TestForEachPreparer` (multi-child IfCondition; single-child IfCondition escalation). +- **Commit**: `8eceaea` + +### Change: `dependency-multi-condition-mapping` (P1) + +- **Title**: Map ADF `dependencyConditions` lists to the correct DAB `run_if` value instead of picking the first. +- **Rationale**: CF-004 — `[Succeeded, Failed]` (= "run regardless") was reduced to `Succeeded`, silently reversing semantics for log/error-handler tasks. +- **Files changed**: + - `src/flowx/translator/engine.py` — added `_map_dependency_conditions` helper that reduces lists per documented mapping rules; `_build_base_kwargs` uses it. +- **Tests added**: `TestCommonAttributes.test_dependency_multi_condition_succeeded_and_failed_maps_to_completed` in `tests/unit/test_translators.py`. +- **Commit**: `fe37f9f` + +### Change: `expression-resolver-bool-and-numeric-coercion` (P1) + +- **Title**: Coerce LS / parameter values during resolution — stringified booleans to YAML booleans, numeric strings to ints where the cluster spec demands int. +- **Rationale**: VAR-006 — bool default `false` ends up as string `'False'`; num_workers `'1'` as string. The numeric coercion was already part of change 2; this change adds the bool/int/float pipeline-parameter coercion. +- **Files changed**: + - `src/flowx/translator/engine.py` — added `_coerce_parameter_default`; pipeline-parameter entries now carry `type` + properly-typed default. + - `src/flowx/bundler/dab_writer.py` — `pipeline_dict_to_ir` honours non-string defaults verbatim (no string-coercion when value is already bool/int/float). +- **Tests added**: 3 tests in `tests/unit/test_translators.py` covering bool/int/string coercion. +- **Commit**: `5f75b01` + +### Change: `pipeline-parameters-and-variables-round-trip` (P0, partial) + +- **Title**: Round-trip `pipeline.parameters` through `translation_report`. +- **Rationale**: VAR-001 — `_load_report`'s aggregated branch dropped pipeline parameters, so 329/340 bundles referenced `{{job.parameters.X}}` without declaring them. +- **Scope note**: Implemented only the parameter round-trip via the aggregator. Variable-init bridge tasks (VAR-004) and full variable-cache plumbing not implemented this iteration. +- **Files changed**: + - `src/flowx/bundler/dab_writer.py` — aggregator now reads `translation.parameters` and `ir.parameters`, threads them through `pipeline_dict_to_workflow`. +- **Tests added**: `TestAggregatedReportPipelineParameters.test_load_report_carries_pipeline_parameters` in `tests/unit/test_bundler.py`. +- **Commit**: `1aa3322` + +## Deferred + +The following P0/P1 plan items were not addressed in this iteration. They each require broader cross-file refactors than was safe inside a single iteration with the time budget available. + +- **`condition-and-switch-expression-bridge` (P0)** — bridging IfCondition/Switch expressions through synthetic SetVariable tasks requires changes across the if_condition / switch translators **and** preparers **and** execute_pipeline preparer (5 files), plus the inner-task wiring for `run_job_task.job_parameters`. The transformation is non-trivial and risks regressions in existing condition_task / case_task tests. Plan for next iteration: implement behind a feature flag, then turn on after corpus validation. +- **`triggers-to-schedules` (P0)** — end-to-end schedule support touches engine + workflow_preparer + dab_writer for trigger ingestion, N:M binding, parameter overrides, and Tumbling/BlobEvents/Custom routing to SETUP.md. Plan for next iteration: land in a single focused PR. +- **`credential-and-secret-recognition` (P0)** — AzureKeyVaultSecret + AzureBlobFS CredentialReference + MSI auth across 4 files. Plan for next iteration: tackle one credential family per commit. +- **Variable defaults & init bridges (partial — half of `pipeline-parameters-and-variables-round-trip`)** — VAR-004 / VAR-006 (variable defaultValue dropped + parameter type dropped) need the variable-init bridge task pattern, which interlocks with the condition/switch bridge work above. + +## Summary + +- 10 commits on `fix-0603` (this section's 9 changes + one for partial round-trip) +- 528 unit tests pass after the final commit (baseline: 499 — net 29 new tests) +- No tests broken; no `--no-verify` or `--amend` used. + +## Iteration 2 — 2026-06-03 + +Implemented 11 of 13 plan changes (C-13 folded into C-07 since they share the same surface; one residual cleanup deferred — see notes below). + +### C-01 — Collapse `@concat()` to a literal when every part resolves to a literal (P0) + +- **Rationale**: NB-ITER2-1 / LSC2-004. `_resolve_concat` / `_handle_concat` always emitted `kind='notebook_code'`. Library jar paths whose concat parts all resolve to literals shipped as Python source instead of strings; `notebook._resolve_libraries` only inlines literals, so installs silently broke in 185 bundles. +- **Files**: `src/flowx/parser/expression_parser.py`, `tests/unit/test_expression_parser.py`, `tests/unit/test_translators.py`. +- **Tests**: `TestConcat.test_concat_literals`, `TestConcat.test_concat_collapses_when_all_parts_resolve_to_literals`, `TestNotebookTranslator.test_translate_notebook_resolves_library_with_globals` (updated to assert literal jar path). +- **Commit**: `cb5ddb7`. + +### C-02 — Unwrap `{value, type:'Expression'}` dicts in LS params and cluster fields (P0) + +- **Rationale**: NB-ITER2-2 / LSC2-003. `_substitute_ls_params` and `_extract_cluster_config` preserved the ADF expression dict-wrapper shape; `custom_tags` emitted `{DigitalCase: {value: APP0001, type: Expression}}` (166 bundles) which is invalid for Databricks `Map[String,String]`. +- **Files**: `src/flowx/translator/engine.py`, `tests/unit/test_translators.py`. +- **Tests**: `TestCommonAttributes.test_ls_param_expression_wrapper_unwrapped_in_custom_tags`, `TestCommonAttributes.test_ls_param_resolved_against_factory_global_parameters` (covers C-03 too). +- **Commit**: `267b08d` (also covers C-03 since both modifications share `_resolve_ls_parameters`). + +### C-03 — Run activity-supplied LS parameter values through `resolve_expression` with factory globals (P0) + +- **Rationale**: NB-ITER2-3 / LSC2-002. Activities passing `clusterVersion: {value:'@pipeline().globalParameters.clusterVersion', type:'Expression'}` left the raw expression in 50 IR JSONs and 4 bundles. `_resolve_ls_parameters` now threads the translation context so `@pipeline().globalParameters.X` collapses to its factory value. +- **Files**: `src/flowx/translator/engine.py`, `tests/unit/test_translators.py`. +- **Tests**: `TestCommonAttributes.test_ls_param_resolved_against_factory_global_parameters`. +- **Commit**: folded into `267b08d`. + +### C-04 — Walk nested activities when collecting workflow cluster hints (P0) + +- **Rationale**: NB-ITER2-4 / LSC2-001. `prepare_workflow` iterated only `pipeline.tasks`. Pipelines whose only `DatabricksNotebook` lived inside an `IfCondition` / `Switch` / `ForEach` branch shipped the default `Standard_DS3_v2 / 15.4.x` cluster fallback in 26 bundles. New `_iter_activity_with_descendants` BFS-walks each top-level activity. +- **Files**: `src/flowx/preparer/workflow_preparer.py`, `tests/unit/test_preparers.py`. +- **Tests**: `TestPrepareWorkflow.test_prepare_workflow_collects_cluster_hints_from_nested_activities`, `TestPrepareWorkflow.test_prepare_workflow_collects_cluster_hints_from_switch_default_branch`. +- **Commit**: `84b491a`. + +### C-05 — Synthesise init SetVariable tasks for variables with defaultValue (P0) + +- **Rationale**: VAREX-002. Pipeline variables carrying `defaultValue` (164 in corpus, 127 expression-typed) were never materialised as IR tasks, so `_resolve_variable`'s self-referential fallback produced 333 dangling `{{tasks.X.values.X}}` refs. `translate_pipeline` now prepends a `_init_` SetVariableActivity per default-valued variable and registers the synth task in `variable_cache`. `_resolve_variable` additionally returns `None` when no setter is known to surface unresolved variables as raw expressions instead of placeholders. +- **Files**: `src/flowx/translator/engine.py`, `src/flowx/parser/expression_parser.py`, `tests/unit/test_translators.py`, `tests/unit/test_preparers.py` (Switch unresolved-variable assertion updated), `tests/unit/test_expression_parser.py` (rename `test_variable_fallback_to_name` → `test_variable_returns_none_when_no_setter`). +- **Tests**: `TestVariableInitTasks.test_default_valued_variable_yields_init_task`, `TestVariableInitTasks.test_default_valued_variable_with_concat_expression`. +- **Commit**: `d80c665`. + +### C-06 — Route ForEach inner-job `@variables()` refs via task-value, not undeclared job parameter (P0) + +- **Rationale**: VAREX-004. `collect_inner_job_params` previously declared `variables()` refs as inner-job parameters and mapped them to `{{job.parameters.}}` on the parent. Parent never declared those names so the inner job received an empty string. When a parent setter task_key is known, variables now route through `{{tasks..values.}}` and are NOT declared on the inner-job parameter list. +- **Files**: `src/flowx/bundler/inner_job_params.py`, `src/flowx/preparer/activity_preparers/for_each.py`, `src/flowx/preparer/workflow_preparer.py`, `tests/unit/test_for_each_inner_job_params.py` (new). +- **Tests**: 4 new tests in `tests/unit/test_for_each_inner_job_params.py`. +- **Commit**: `c119ab3`. + +### C-07 — Bridge IfCondition / Switch condition_task operands through a hidden SetVariable task (P0) + +- **Rationale**: CF-iter2-001 / CF-iter2-003 / VAREX-003. Per Databricks docs, `condition_task.left` / `.right` must be literal / job-parameter / task-value / task-parameter refs. Operands carrying ADF function calls (`@toUpper`, `@coalesce`, `@and`, `@or`, `@not`, `@empty`, `@concat`, ...) shipped as raw ADF expressions in 84 bundles. New `lower_to_bridge` / `merge_bridge_requests` in `activity_translators/resolve.py` turn `notebook_code` resolver results into a `BridgeRequest` that the IfCondition / Switch translators stash on the IR; the corresponding preparers synthesise a bridge notebook task and rewrite the operand to the task-value reference. Dropped the legacy `NOT_EQUAL '0'` fallback when a bridge succeeds. +- **Files**: `src/flowx/models/ir.py` (new bridge_notebook_* fields on IfCondition/Switch), `src/flowx/translator/activity_translators/resolve.py`, `src/flowx/translator/activity_translators/if_condition.py`, `src/flowx/translator/activity_translators/switch.py`, `src/flowx/preparer/activity_preparers/if_condition.py`, `src/flowx/preparer/activity_preparers/switch.py`, `src/flowx/translator/engine.py` (serialise bridge fields). +- **Tests**: `TestIfConditionTranslator.test_translate_if_condition_empty_bridges_via_notebook`, `TestIfConditionPreparer.test_prepare_if_condition_emits_bridge_task`, `TestSwitchTranslator.test_translate_switch_function_call_routes_through_bridge`, plus `TestSwitchPreparer.test_resolve_switch_on_expression_is_idempotent_for_dab_refs` (covers C-13). +- **Commit**: `7494fde`. + +### C-08 — Bridge ForEach `for_each_task.inputs` through a seed task when items expression is notebook_code (P0) + +- **Rationale**: CF-iter2-002. Per docs, `inputs` accepts a JSON array literal or `{{tasks.X.values.Y}}` or `{{job.parameters.X}}`; `@split(...)` was rejected. 12 bundles emitted `inputs: '@split(...)'` verbatim. New `_resolve_for_each_inputs_with_bridge` emits a seed notebook task that computes the array and rewrites `inputs` to its task value. +- **Files**: `src/flowx/preparer/activity_preparers/for_each.py`, `tests/unit/test_preparers.py`. +- **Tests**: `TestForEachPreparer.test_prepare_for_each_bridges_split_items_via_seed_task`. +- **Commit**: `92036a8`. + +### C-09 — ExecutePipeline parameter resolution refuses notebook_code result kinds (P0) + +- **Rationale**: VAREX-001. 62 bundles had ExecutePipeline parameter values like `'json: ' + dbutils.widgets.get('configFile')` because `resolve.py` accepted any result.kind. The sub-job's widget received the Python source text. The translator now resolves each parameter through `resolve_expression`, accepts only literal / dab_ref results, and surfaces `notebook_code` results as `parameter_approximations` for SETUP.md. +- **Files**: `src/flowx/translator/activity_translators/execute_pipeline.py`, `tests/unit/test_translators.py`. +- **Tests**: `TestExecutePipelineTranslator.test_translate_execute_pipeline_drops_notebook_code_parameters`. +- **Commit**: `34e0bd9`. + +### C-10 — Compile AdfTrigger objects into Pipeline.schedule and emit DAB schedule / trigger blocks (P0) + +- **Rationale**: SCHED-001. `definitions.triggers` was parsed but never read; `Pipeline.schedule` was always `None`; 327 bundles shipped without any schedule metadata. `translate_pipeline` now matches triggers by their `pipelineReference` and compiles the first matching one into a structured schedule dict (`ScheduleTrigger.recurrence` → quartz cron + timezone; `BlobEventsTrigger` → `trigger.file_arrival`; Tumbling/Custom → manual-setup hints). Windows timezone names (`Romance Standard Time`) normalise to IANA (`Europe/Madrid`). `runtimeState='Stopped'` flips `pause_status` to `PAUSED`. `PreparedWorkflow.schedule` and `pipeline_dict_to_ir` thread the spec to the DAB writer. +- **Files**: `src/flowx/preparer/workflow_preparer.py`, `src/flowx/translator/engine.py`, `src/flowx/bundler/dab_writer.py`, `tests/unit/test_translators.py`, `tests/unit/test_bundler.py`. +- **Tests**: 7 new tests in `TestScheduleCompilation` covering Daily / Weekly / Tumbling / Blob / Custom triggers, timezone normalisation, and pause-status; 2 new bundler tests (`TestScheduleEmission`) verifying the schedule / trigger blocks land in the rendered YAML. +- **Commit**: `bc10f5b`. + +### C-11 — Emit per-secret SecretInstructions from Web activity auth payloads (P1) + +- **Rationale**: LSC2-005. `web_activity.prepare` always emitted a single `auth-credential` SecretInstruction regardless of the underlying ADF auth shape. AzureKeyVaultSecret payloads carry `store.referenceName` and `secretName` but both were dropped; CredentialReference (managed identity) was wrongly treated as a static secret. The preparer now walks nested fields (`password` / `secret` / `clientSecret` / `pfx` / `key`), materialises per-Key Vault SecretInstructions, and routes CredentialReference / MSI payloads to a `manual_credential` SetupTask so SETUP.md flags them. +- **Files**: `src/flowx/preparer/activity_preparers/web_activity.py`, `tests/unit/test_preparers.py`. +- **Tests**: `TestWebActivityPreparer.test_prepare_web_activity_key_vault_secret_uses_vault_scope_and_secret_name`, `TestWebActivityPreparer.test_prepare_web_activity_credential_reference_emits_setup_note`. +- **Commit**: `bce740d`. + +### C-12 — Extend `_strip_dangling_task_value_refs` to cover run_job_task and condition_task fields (P1) + +- **Rationale**: VAREX-005. `_strip_dangling_task_value_refs` only walked `notebook_task.base_parameters`. 333 dangling `{{tasks.X.values.Y}}` refs survived into resource YAMLs because cross-job (`run_job_task.job_parameters`) and control-flow (`condition_task.left/.right`) surfaces weren't covered. The walker now also visits these fields and blanks dangling refs so SETUP.md §4 can flag them. +- **Files**: `src/flowx/bundler/dab_writer.py`, `tests/unit/test_bundler.py`. +- **Tests**: `TestStripDanglingTaskValueRefs.test_strips_dangling_run_job_task_job_parameters`, `TestStripDanglingTaskValueRefs.test_strips_dangling_condition_task_operands`, `TestStripDanglingTaskValueRefs.test_recurses_into_for_each_task_body`. +- **Commit**: `836a45d`. + +### C-13 — Idempotent Switch on-expression resolver (P1) + +- **Rationale**: CF-iter2-004. `preparer/activity_preparers/switch.py::resolve_switch_on_expression` constructed an empty `TranslationContext()` and re-resolved on-expression on the JSON-reload path, destructively stripping refs the translator had already lowered. Updated to pass through anything already containing `{{...}}` or a translator-side `__BRIDGE__::` placeholder; only bare `@`-prefixed expressions are re-resolved. +- **Files**: covered in C-07 commit (`src/flowx/preparer/activity_preparers/switch.py`). +- **Tests**: `TestSwitchPreparer.test_resolve_switch_on_expression_is_idempotent_for_dab_refs`. +- **Commit**: folded into `7494fde`. + +### Skipped / partial + +- The plan's C-13 acceptance check additionally calls for `grep -nE 'TranslationContext\(\s*\)' src/flowx/preparer/` returning zero results. The current code still constructs bare `TranslationContext()` in ~12 sites across `preparer/code_generator.py`, `preparer/activity_preparers/{execute_pipeline,for_each,switch,databricks_job,filter,notebook,helpers}.py`. The switch fix is the highest-impact one (it was the only example called out in the rationale); the remaining sites operate at the preparer layer where global parameters and `variable_cache` aren't readily available, so converting them to thread a typed context is a larger refactor. Deferred to a follow-on iteration. + +## Summary (iteration 2) + +- 11 new commits on `fix-0603` (C-01 .. C-12, with C-13 folded into C-07) +- 559 unit tests pass after the final iteration-2 commit (iteration-1 baseline: 528 — net 31 new tests) +- No tests broken; no `--no-verify` or `--amend` used. + +## Iteration 3 — 2026-06-03 / 2026-06-04 + +Implemented all 18 plan items across 15 commits (C-13 .. C-27). Three plan items overlapped on single edits and were bundled: C-13 folds three resolver-widening fixes; C-19 folds two ForEach inner-workflow fixes; C-21 fixes the same bool-stringification bug at three call sites. + +### C-13 — Propagate globals into ForEach child context; widen LS/library resolvers to accept dab_ref (P0) + +- **Rationale**: NB-ITER3-001 / CF3-002 / LSC3-004 (ForEach child context drops global_parameters / linked_service_parameters) + NB-ITER3-002 / LSC3-003 / VAREX3-006 (`_resolve_ls_parameters` refuses `dab_ref`) + NB-ITER3-004 (notebook `_resolve_libraries` refuses `dab_ref`). +- **Files**: `src/flowx/translator/activity_translators/for_each.py`, `src/flowx/translator/engine.py`, `src/flowx/translator/activity_translators/notebook.py`. +- **Commit**: `504000c`. + +### C-14 — Preserve bridge_notebook_code/imports/required_parameters on IR roundtrip (P0) + +- **Rationale**: CF3-001 / VAREX3-001. `_reconstruct_ir` dropped bridge fields, leaving 109 bundles shipping `left: __BRIDGE__::result` with no actual `_bridge` task. +- **Files**: `src/flowx/bundler/dab_writer.py`, `tests/unit/test_bundler.py`. +- **Commit**: `a1955e4`. + +### C-15 — IfCondition emits right='False' (not '' or '0') against bridge task values (P0) + +- **Rationale**: CF3-003 / VAREX3-004. 76 bundles emitted `right: ''` and 19 emitted `right: '0'` from `@not(...)` / legacy truthy fallback paths; neither compares correctly against Python `'True'/'False'`. +- **Files**: `src/flowx/translator/activity_translators/if_condition.py`, `tests/unit/test_translators.py`. +- **Commit**: `a643a8c`. + +### C-16 — Lower single-segment item()?.X safe-nav to notebook_code (P0) + +- **Rationale**: CF3-005 / VAREX3-005. `expression_parser._resolve_item_safe_nav` returned None when `len(parts) < 2`, blocking 4 Switch on-expressions and SetVariable expressions from bridge lowering. +- **Files**: `src/flowx/parser/expression_parser.py`, `tests/unit/test_expression_parser.py`. +- **Commit**: `8f7d6d6`. + +### C-17 — Emit single_user_name alongside data_security_mode: SINGLE_USER (P0) + +- **Rationale**: NB-ITER3-003. 189 bundles failed `databricks bundle validate` with "single_user_name must be set when data_security_mode is SINGLE_USER". Three cluster builders set SINGLE_USER mode without the name. Use `${workspace.current_user.userName}` as the closest deployable analog of ADF's MSI auth. +- **Files**: `src/flowx/bundler/dab_writer.py`. +- **Tests**: `TestSingleUserNameOnSingleUserClusters` (3 cases) in `tests/unit/test_bundler.py`. +- **Commit**: `b41cc24`. + +### C-18 — Carry schedule through aggregated translations report into pipeline_dict (P0) + +- **Rationale**: SCHED3-001. `_load_report`'s aggregated branch built pipeline_dict from `{name, tasks, parameters}` and never copied `schedule`. 0 of 327 bundles contained `quartz_cron_expression` despite 8 trigger-referenced pipelines having populated schedule blocks. +- **Files**: `src/flowx/bundler/dab_writer.py`. +- **Tests**: `TestAggregatedReportSchedule` (2 cases) in `tests/unit/test_bundler.py`. +- **Commit**: `5666708`. + +### C-19 — Propagate cluster_hints + variable_task_keys into ForEach inner workflows (P0 + P1) + +- **Rationale**: LSC3-001 — inner-job PreparedWorkflow drops cluster hints from nested activities, so inner-job YAMLs ship the bundle default `Standard_DS3_v2 / 15.4.x` even when the parent default cluster carries LS-derived `spark_env_vars / custom_tags / driver_node_type_id`. CF3-006 — multi-child `collect_inner_job_params` call dropped the `variable_task_keys` kwarg that the single-child path threaded. +- **Files**: `src/flowx/preparer/activity_preparers/for_each.py`. +- **Tests**: `TestForEachPreparer.test_for_each_inner_workflow_carries_cluster_hints_from_inner_activity`, `TestForEachPreparer.test_for_each_single_child_inner_workflow_carries_cluster_hints` in `tests/unit/test_preparers.py`; `TestVariableTaskKeysRouting.test_multi_child_for_each_threads_variable_task_keys` in `tests/unit/test_for_each_inner_job_params.py`. +- **Commit**: `ece7f01`. + +### C-20 — Stop emitting fake auth-credential secret for MSI WebActivity auth (P0) + +- **Rationale**: LSC3-002. MSI / ManagedServiceIdentity has no static secret, so reading `auth-credential` from a secret scope shipped a broken bearer-token call against a non-existent secret in 14 generated notebooks across 5 source pipelines. The notebook now raises `NotImplementedError` pointing at SETUP.md. ServicePrincipal auth keeps the secret-based flow. +- **Files**: `src/flowx/preparer/code_generator.py`. +- **Tests**: `TestGenerateWebActivityNotebook.test_auth_block_msi_raises_not_implemented` in `tests/unit/test_code_generator.py`. +- **Commit**: `5f2c0eb`. + +### C-21 — Render Boolean variable values as lowercase true/false (P0) + +- **Rationale**: VAREX3-002. Python title-case `'True'/'False'` silently inverted ADF Boolean comparisons like `@equals(variables('continue'), true)` across 28 occurrences. Fix at three call sites: `resolve_expression()` bool literal path (root cause), `_build_variable_init_activities` fallback, and `set_variable.py` translator fallback. +- **Files**: `src/flowx/translator/engine.py`, `src/flowx/translator/activity_translators/set_variable.py`, `src/flowx/parser/expression_parser.py`. +- **Tests**: `TestVariableInitTasks.test_default_valued_boolean_variable_renders_lowercase`, `TestVariableInitTasks.test_set_variable_with_raw_bool_value_renders_lowercase` in `tests/unit/test_translators.py`; updated `TestLiterals.test_boolean` in `tests/unit/test_expression_parser.py`; updated `test_boolean_value` in `tests/unit/test_resolve_field.py`. +- **Commit**: `e7c35d6`. + +### C-22 — Case-insensitive dataset / linked service lookup + LS URL threading (P1) + +- **Rationale**: LSC3-005. ADF identifiers are documented as case-insensitive, but the loader keys dicts by original case; 2 generated lookup notebooks shipped `spark.sql('')` because a lowercase reference didn't match a mixed-case dataset filename. Also threads `linked_service.typeProperties.url` so the file lookup body assembles a fully-qualified `abfss://...` widget default. +- **Files**: `src/flowx/models/adf_ast.py` (added `get_dataset` / `get_linked_service`), `src/flowx/translator/activity_translators/lookup.py`, `src/flowx/preparer/code_generator.py`. +- **Tests**: `TestLookupCaseInsensitiveAndLinkedService` (2 cases) in `tests/unit/test_translators.py`. +- **Commit**: `b9ec3d6`. + +### C-23 — Resolve attribute chains on function-call results (P1) + +- **Rationale**: CF3-004. `resolve_expression('@toUpper(json(pipeline().parameters.items).type)')` returned None because the bare function dispatcher only matches when the function call is the outermost token. New `_resolve_function_call_with_attribute` helper detects `funcName(args).attr.attr...`, resolves the function call, then chains `.get('')` for each segment. +- **Files**: `src/flowx/parser/expression_parser.py`. +- **Tests**: `TestFunctionCallWithAttribute` (2 cases) in `tests/unit/test_expression_parser.py`. +- **Commit**: `696941d`. + +### C-24 — Emit trigger.periodic for Day/Week/Month recurrence with interval > 1 (P1) + +- **Rationale**: SCHED3-002. `_recurrence_to_quartz_cron` silently dropped `interval` for Day / Week / Month, producing cron that fired every day/week/month rather than every Nth. quartz cron cannot represent "every N days/weeks/months" without enumeration; DAB `trigger.periodic` takes `{interval, unit}` directly. +- **Files**: `src/flowx/translator/engine.py`, `src/flowx/bundler/dab_writer.py`. +- **Tests**: `TestScheduleCompilation.test_schedule_trigger_interval_3_days_emits_periodic`, `test_schedule_trigger_interval_2_weeks_emits_periodic`, `test_schedule_trigger_interval_1_day_still_cron` in `tests/unit/test_translators.py`; `TestScheduleEmission.test_periodic_trigger_emitted` in `tests/unit/test_bundler.py`. +- **Commit**: `563f3e8`. + +### C-25 — Carry pipelineReference.parameters from triggers into job parameter defaults (P1) + +- **Rationale**: SCHED3-003. `_compile_pipeline_schedule` read only `pipelineReference` from each trigger entry and silently dropped per-pipeline `parameters` overrides like `{applicationName: 'app0001', negocio: 'GLP'}`. Scheduled runs received bare pipeline defaults. New `_extract_trigger_parameter_overrides` attaches the override map as `parameter_overrides` on the schedule spec; `_build_job_resource` mutates matching `job.parameters[*].default` after applying the schedule. +- **Files**: `src/flowx/translator/engine.py`, `src/flowx/bundler/dab_writer.py`. +- **Tests**: `TestScheduleCompilation.test_trigger_carries_per_pipeline_parameter_overrides` in `tests/unit/test_translators.py`; `TestScheduleEmission.test_trigger_parameter_overrides_mutate_job_parameter_defaults` in `tests/unit/test_bundler.py`. +- **Commit**: `4b2eba0`. + +### C-26 — Emit manual_variable_rollup SetupTask for cross-ForEach variable reads (P1) + +- **Rationale**: VAREX3-003. When a SetVariable for `X` lives only inside a ForEach inner-job and a sibling task reads `@variables('X')`, the read gets the stale init value because task values cannot cross `run_job_task` boundaries. Minimum-viable fix: detect the pattern in `prepare_workflow`, emit a SetupTask of type `manual_variable_rollup`, surface it in SETUP.md so the user adds a roll-up notebook before the sibling runs. +- **Files**: `src/flowx/preparer/workflow_preparer.py`, `src/flowx/bundler/prereqs_writer.py`, `src/flowx/bundler/dab_writer.py`. +- **Tests**: `TestCrossForEachVariableReadDetection` (2 cases) in `tests/unit/test_preparers.py`; `TestManualVariableRollupSetupMd` in `tests/unit/test_bundler.py`. +- **Commit**: `87bf35e`. + +### C-27 — Union scan_notebooks_for_secrets with workflow.secrets in SETUP.md (P1) + +- **Rationale**: LSC3-006. `scan_notebooks_for_secrets` walked notebooks (yielding the MSI fake `auth-credential` before C-20) while `create_secrets.py` was written from `workflow.secrets` (real AKV scopes). SETUP.md Option A vs Option B disagreed. `build_prereqs` now accepts `secret_instructions`, unions them with the notebook scan (dedupe by (scope, key)) so both options stay in sync. +- **Files**: `src/flowx/bundler/prereqs_writer.py`, `src/flowx/bundler/dab_writer.py`. +- **Tests**: `TestSecretsUnion` (2 cases) in `tests/unit/test_prereqs_writer.py` (new file). +- **Commit**: `b64cc5a`. + +## Summary (iteration 3) + +- 15 new commits on `fix-0603` (C-13 .. C-27) +- 595 unit tests pass after the final iteration-3 commit (iteration-2 baseline: 559 — net 36 new tests) +- No tests broken; no `--no-verify` or `--amend` used. +- All 18 P0/P1 plan items implemented end-to-end. C-13, C-19, and C-21 each fold 2-3 closely-related plan items into a single commit since the underlying fix is the same edit (no scope creep). + +## Iteration 4 — 2026-06-04 + +Implemented all 12 plan items across 11 commits. C-28 + C-30 are bundled +into one commit since they share NotebookActivity IR fields and the +preparer infrastructure. + +### C-28 + C-30 — Dynamic notebookPath dispatch stub + unresolved library SetupTask (P0 + P1) + +- **Rationale**: NB-ITER4-001 — the notebook translator passed `notebookPath` + through `resolve_field` so a `notebook_code` result (e.g. + `@trim(json(...).notebook_path)`) shipped as the workspace path. + NB-ITER4-003 — `_resolve_libraries` silently passed through unresolved jar + paths so the cluster tried to install a file literally named + `@concat(...)`. +- **Files**: `src/flowx/translator/activity_translators/notebook.py`, + `src/flowx/models/ir.py`, + `src/flowx/preparer/activity_preparers/notebook.py`, + `src/flowx/bundler/prereqs_writer.py`, + `src/flowx/bundler/dab_writer.py`, + `src/flowx/translator/engine.py`. +- **Tests**: `test_translate_notebook_dynamic_path_marks_unresolved`, + `test_translate_notebook_unresolved_library_captured` in + `tests/unit/test_translators.py`; + `test_prepare_notebook_dispatch_stub_for_unresolved_path`, + `test_prepare_notebook_emits_unresolved_library_setup_task` in + `tests/unit/test_preparers.py`. +- **Commit**: `27ed860`. + +### C-29 — Filter unparseable spark_version / node_type_id from cluster_hints (P0) + +- **Rationale**: NB-ITER4-002 — `_infer_bundle_cluster_defaults` picked the + most-common spark_version regardless of whether it parsed as a real DBR + version, so an `@if(equals(item()?.photon,true),...)` expression landed + in `databricks.yml` as the spark_version default and `bundle deploy` + rejected it. +- **Files**: `src/flowx/bundler/dab_writer.py`. +- **Tests**: `TestUnparseableClusterHintsFiltered` (3 cases) in + `tests/unit/test_bundler.py`. +- **Commit**: `a6f9c05`. + +### C-31 — Move ForEach items-expression bridge to translator (P0) + +- **Rationale**: CF4-001 — the for_each preparer constructed a bare + `TranslationContext()` to re-resolve `@split(variables('fecha'),',')`, + but `variable_cache` is empty on the JSON-reload path so the bridge + never fired and DAB rejected the raw @split call as + `for_each_task.inputs`. +- **Files**: `src/flowx/translator/activity_translators/for_each.py`, + `src/flowx/models/ir.py`, + `src/flowx/preparer/activity_preparers/for_each.py`, + `src/flowx/bundler/dab_writer.py`, + `src/flowx/translator/engine.py`. +- **Tests**: `test_prepare_for_each_uses_ir_bridge_for_variable_based_split` + in `tests/unit/test_preparers.py`. +- **Commit**: `d666402`. + +### C-32 — IfCondition truthy fallback Boolean-aware right operand (P1) + +- **Rationale**: CF4-002 — the legacy `right: '0'` fallback against + Boolean variable refs is always-true (post-C-21 SetVariable writes + lowercase `'true'/'false'` strings), so the false branch became + unreachable. +- **Files**: `src/flowx/translator/activity_translators/if_condition.py`. +- **Tests**: + `test_translate_if_condition_boolean_variable_uses_lowercase_false` + in `tests/unit/test_translators.py`. +- **Commit**: `d546a86`. + +### C-33 — SetVariable: lower split[N]/2-arg substring and surface unresolved @-expressions (P1) + +- **Rationale**: Merged VAREX4-001 + CF4-003 — 212 SetVariableActivity + entries shipped raw `@concat(...)` text with `value_kind='literal'` + because `resolve_expression` returned None for nested constructs like + `split(...)[N]` and the 2-arg `substring(x, start)`. Bundles that + remained unresolvable now blank the variable and emit a + `manual_variable_init` SetupTask. +- **Files**: `src/flowx/parser/expression_parser.py`, + `src/flowx/translator/activity_translators/set_variable.py`, + `src/flowx/models/ir.py`, + `src/flowx/preparer/activity_preparers/set_variable.py`, + `src/flowx/translator/engine.py`, + `src/flowx/bundler/dab_writer.py`. +- **Tests**: `test_substring_two_arg_form`, `test_split_with_subscript` + in `tests/unit/test_expression_parser.py`; + `test_translate_set_variable_split_subscript_lowers_to_notebook_code`, + `test_translate_set_variable_unresolved_expression_blanks_value` + in `tests/unit/test_translators.py`. +- **Commit**: `3a68586`. + +### C-34 — Expression parser: preserve quoted-string and Boolean literal types in codegen (P1) + +- **Rationale**: Merged VAREX4-002 + VAREX4-003 — both live in + `_resolve_function_call` / `_arg_to_code`. Quoted args like `'12'` + collapsed to bare tokens (`... == 12`, wrong type) and `'09'` produced + a SyntaxError (leading-zero integer). Boolean tokens `true/false` + emitted Python `True/False` but the SetVariable side serialised + lowercase strings post-C-21. +- **Files**: `src/flowx/parser/expression_parser.py`, + `src/flowx/models/ir.py`. +- **Tests**: `test_equals_quoted_string_emits_repr`, + `test_less_quoted_leading_zero_is_valid_python`, + `test_equals_bool_literal_emits_lowercase_string` in + `tests/unit/test_expression_parser.py`. +- **Commit**: `2ff3c10`. + +### C-35 — Anchor _ITEM_FIELD_RE for chained item().a.b lowering (P1) + +- **Rationale**: CF4-004 — the unanchored `_ITEM_FIELD_RE` matched the + first segment of `item().condition.name` and returned + `{{input.condition}}`, silently dropping `.name`. +- **Files**: `src/flowx/parser/expression_parser.py`. +- **Tests**: `test_item_field_multi_segment_lowers_to_notebook_code` + in `tests/unit/test_expression_parser.py`. +- **Commit**: `2177a11`. + +### C-36 — Preserve hours/minutes/weekDays on periodic schedules (P1) + +- **Rationale**: SCHED4-001 — `_recurrence_to_periodic` dropped + `schedule.minutes/hours/weekDays/monthDays` for Day/Week/Month with + interval > 1, so a schedule declaring "every 3 days at 02:00 UTC" + silently fired at midnight. +- **Files**: `src/flowx/translator/engine.py`, + `src/flowx/preparer/workflow_preparer.py`. +- **Tests**: Extended + `test_schedule_trigger_interval_3_days_emits_periodic` in + `tests/unit/test_translators.py`; + `TestManualScheduleTimeOfDaySetupTask` in + `tests/unit/test_preparers.py`. +- **Commit**: `37845f6`. + +### C-37 — File-Lookup: unwrap expression-dict folder/file + abfss URL rewrite (P0) + +- **Rationale**: Merged LSC4-001 + LSC4-003. LSC4-001 (P0) — folder_path + / file_name shipped as raw expression dicts; `.strip('/')` crashed the + bundler with AttributeError, taking down 4 pipelines. LSC4-003 (P1) — + AzureBlobFS https URLs joined to folder/filename produce notebooks + that can't read the source on a Databricks cluster. +- **Files**: `src/flowx/translator/activity_translators/lookup.py`, + `src/flowx/preparer/code_generator.py`. +- **Tests**: `test_file_lookup_coerces_expression_dict_path_components`, + `test_file_lookup_rewrites_https_to_abfss` in + `tests/unit/test_code_generator.py`. +- **Commit**: `a9e2113`. + +### C-38 — Web activity notebook threads resolved Key Vault scope/key (P0) + +- **Rationale**: LSC4-002 — `generate_web_activity_notebook` hard-coded + `scope=task_key, key='auth-credential'` regardless of what the C-11 + preparer resolved. 11 generated notebooks across multiple pipelines + read the wrong secret at runtime even though the C-11 SecretInstruction + carried the real values. +- **Files**: `src/flowx/preparer/code_generator.py`, + `src/flowx/preparer/activity_preparers/web_activity.py`. +- **Tests**: Extended + `test_prepare_web_activity_key_vault_secret_uses_vault_scope_and_secret_name` + in `tests/unit/test_preparers.py` to assert the rendered notebook + contains the resolved scope and key. +- **Commit**: `72f6211`. + +### C-39 — Emit manual_credential SetupTask for MSI / CredentialReference cluster auth (P1) + +- **Rationale**: LSC4-004 — every default cluster ships + `single_user_name: ${workspace.current_user.userName}` regardless of + source ADF authentication. MSI / CredentialReference workloads now + silently run as the deploying human user with no warning. +- **Files**: `src/flowx/translator/engine.py`, + `src/flowx/preparer/workflow_preparer.py`, + `src/flowx/bundler/prereqs_writer.py` (rendering, landed in C-28 + commit alongside the dispatch-stub SETUP.md surface), + `src/flowx/bundler/dab_writer.py` (config aggregation, landed in + C-28 commit). +- **Tests**: `TestManualCredentialFromMsiLinkedService` in + `tests/unit/test_bundler.py`. +- **Commit**: `d8c99a6`. + +## Summary (iteration 4) + +- 11 new commits on `fix-0603` (C-28..C-39, with C-28+C-30 folded into a + single commit since they share NotebookActivity IR fields and the + preparer infrastructure). +- 616 unit tests pass after the final iteration-4 commit (iteration-3 + baseline: 595 — net 21 new tests). +- No tests broken; no `--no-verify` or `--amend` used. +- All 12 P0/P1 plan items implemented end-to-end. P2 items + (NB-ITER4-004, SCHED4-002) intentionally excluded per scoping rules. + +## Iteration 5 — 2026-06-04 + +Implemented all 8 P0/P1 gaps plus the trivially-related P2 (CF5-002, folded +into C-43) across 8 commits (C-40..C-47). C-40..C-42 were committed in an +earlier pass; C-43..C-47 complete the iteration. + +### C-40 — Mine num_workers into the default job_cluster instead of hardcoding 1 (P1) + +- **Rationale**: `_infer_bundle_cluster_extras` omitted `num_workers` and + `_build_default_cluster` hardcoded `num_workers: 1`, even though + `workflow_preparer` stores the full cluster dict (with num_workers) into + `cluster_hints` and the IR carries num_workers != 1 for 122 tasks across + 40 pipelines. ADF clusters with 2-4 workers deployed as 1-worker + clusters with no warning. +- **Files**: `src/flowx/bundler/dab_writer.py`. +- **Tests**: num_workers=2 cluster_hint -> emitted default_cluster + new_cluster.num_workers == 2, in `tests/unit/test_bundler.py`. +- **Commit**: `038888b`. + +### C-41 — IfCondition on a literal-seeded Boolean variable emits right:'false' not '0' (P1) + +- **Rationale**: `_operand_is_known_boolean` only checked + `get_variable_dab_ref`, which reads `variable_value_cache`; that cache is + populated only when value_kind == 'dab_ref', so default-valued Boolean + variables seeded via `_build_variable_init_activities` were never + recognized as Boolean. The fallback emitted NOT_EQUAL(left, '0'), always + true for a 'true'/'false' string, making the false branch dead code. +- **Files**: `src/flowx/models/ir.py`, + `src/flowx/translator/engine.py`, + `src/flowx/translator/activity_translators/if_condition.py`. +- **Tests**: `test_translate_if_condition_boolean_variable_by_declared_type` + in `tests/unit/test_translators.py`. +- **Commit**: `db2a7fb`. + +### C-42 — Resolve Set Pipeline Return Value list-of-pairs inner expression (P1) + +- **Rationale**: `set_variable.py` recognized only str and + `{type:Expression}` shapes; a `pipelineReturnValue` value that is a list + of `{key, value:{type:Expression,content:...}}` pairs failed + `_is_adf_expression` and fell to `str(value_raw)`, then the bundler + blanked it to ''. The inner `@variables('executionOutputs')` is + resolvable in the same IR, so the ref is droppable rather than lost. +- **Files**: `src/flowx/translator/activity_translators/set_variable.py`. +- **Tests**: list-of-pairs value with a resolvable inner `@variables()` ref + asserts value_kind == 'dab_ref', in `tests/unit/test_translators.py`. +- **Commit**: `ec8ca13`. + +### C-43 — Inner-ForEach IfCondition bridges locally; bundler warns when blanking a condition (P1, folds CF5-002) + +- **Rationale**: For an inner IfCondition whose operand resolves to a + parent-job task value (`{{tasks._init_continue.values.continue}}`), the + init task lives only in the parent job. When the ForEach body split into + an inner job, `_strip_dangling_task_value_refs` silently blanked + `condition_task.left/right` to '', making NOT_EQUAL('','0') always TRUE + and running the true branch unconditionally with no SETUP.md signal. + if_condition now recomputes a known-Boolean operand locally via a + BridgeRequest (mirroring the Switch path) when a seeded literal default is + available. `_strip_dangling_task_value_refs` now returns the + (task_key, field, original_ref) tuples it blanks; `write_bundle` threads + them into the Prereqs so SETUP.md gets a 'Conditions neutralized to + always-true' section (folds CF5-002: a predicate is never neutralized + silently). +- **Files**: + `src/flowx/translator/activity_translators/if_condition.py`, + `src/flowx/translator/engine.py`, `src/flowx/models/ir.py`, + `src/flowx/bundler/dab_writer.py`, + `src/flowx/bundler/prereqs_writer.py`. +- **Tests**: + `test_translate_if_condition_boolean_variable_bridges_when_default_literal_known` + in `tests/unit/test_translators.py`; + extended `test_strips_dangling_condition_task_operands` plus + `test_neutralized_condition_renders_setup_section` in + `tests/unit/test_bundler.py`. +- **Commit**: `36158e5`. + +### C-44 — Cron derives hour/minute from startTime when ScheduleTrigger has no schedule.hours/minutes (P1) + +- **Rationale**: `_recurrence_to_quartz_cron` read only schedule.minutes/ + hours and fell back to '0'/'0'; startTime was never read. A daily + trigger with startTime 21:00 UTC and no schedule block emitted + '0 0 0 * * ?' (midnight), a 21-hour offset, silently. Per ADF docs the + first-execution time (from startTime) is the default time-of-day. +- **Files**: `src/flowx/translator/engine.py`. +- **Tests**: `test_schedule_trigger_derives_time_of_day_from_start_time` + (Day recurrence, startTime '2023-03-15T21:00:00Z' -> '0 0 21 * * ?') in + `tests/unit/test_translators.py`. +- **Commit**: `6ec5c37`. + +### C-45 — Month-frequency periodic trigger no longer emits the invalid DAB unit MONTHS (P1) + +- **Rationale**: `_recurrence_to_periodic` mapped Month -> MONTHS, but the + Databricks Jobs API PeriodicTriggerConfigurationTimeUnit enum only + defines DAYS, HOURS, WEEKS — MONTHS is rejected by bundle validate/ + deploy. Drop Month from the unit_map (single-month routes to quartz cron + via monthDays); an interval > 1 Month emits a manual_setup schedule note. +- **Files**: `src/flowx/translator/engine.py`. +- **Tests**: + `test_schedule_trigger_interval_2_months_does_not_emit_months_unit` in + `tests/unit/test_translators.py`. +- **Commit**: `5b50b1c`. + +### C-46 — Generate create_secrets.py against the Databricks SDK, not dbutils.secrets writes (P1) + +- **Rationale**: `setup_generator` emitted `dbutils.secrets.createScope` and + `dbutils.secrets.put`. The `dbutils.secrets` submodule is read-only + (get / getBytes / list / listScopes only); both calls raise + AttributeError on the first cell. Generate against + `WorkspaceClient().secrets.create_scope` (RESOURCE_ALREADY_EXISTS + try/except) and `w.secrets.put_secret`. +- **Files**: `src/flowx/bundler/setup_generator.py`, + `tests/unit/test_bundler.py`. +- **Tests**: updated the two `createScope`/`put` asserts to + `create_scope`/`put_secret` (plus WorkspaceClient + negative asserts) in + `test_secrets_setup_notebook_content` and the write_bundle round-trip + test in `tests/unit/test_bundler.py`. +- **Commit**: `acdbb40`. + +### C-47 — File-source Lookup substitutes dataset() parameter refs before baking the abfss:// path (P1) + +- **Rationale**: `lookup.py` read the dataset reference (whose parameters + bind digitalCase/fileName to pipeline params) but never applied them. + `_unwrap_expression` only unwrapped the expression dict, leaving + folderPath '@toLower(dataset().digitalCase)' and fileName + '@dataset().fileName' verbatim; the code generator baked a literal broken + 'abfss://.../@toLower(dataset().digitalCase)/...' default that spark.read + cannot load. Build a dataset-parameter scope, substitute dataset().X, + resolve the result; `_assemble_file_lookup_source_path` drops any leaked + raw dataset() component as a safety net. +- **Files**: + `src/flowx/translator/activity_translators/lookup.py`, + `src/flowx/preparer/code_generator.py`. +- **Tests**: + `test_translate_lookup_substitutes_dataset_parameter_refs` in + `tests/unit/test_translators.py`. +- **Commit**: `f360a68`. + +## Summary (iteration 5) + +- 8 commits on `fix-0603` (C-40..C-47); C-40..C-42 landed in an earlier + pass, C-43..C-47 completed the iteration. CF5-002 (P2) folded into C-43. +- 624 unit tests pass after the final iteration-5 commit (iteration-4 + baseline: 616 — net 8 new tests). +- No tests broken; no `--no-verify` or `--amend` used. +- All 8 P0/P1 plan items implemented end-to-end. P2 NB-ITER5-002 + intentionally excluded per the plan's dedup decision. diff --git a/Makefile b/Makefile index 451dcde..419faa1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean dev ci test integration fmt help docs-install docs-clean docs-build docs-serve lock-dependencies +.PHONY: clean dev ci test integration fmt help docs-install docs-clean docs-build docs-serve lock-dependencies requirements precommit clean: rm -rf .venv .pytest_cache .ruff_cache .mypy_cache __pycache__ @@ -42,6 +42,12 @@ lock-dependencies: uv pip compile --generate-hashes --universal --no-header - > build-constraints-new.txt mv build-constraints-new.txt .build-constraints.txt perl -pi -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple"|g' uv.lock + $(MAKE) requirements + +requirements: + uv export --frozen --no-dev --no-emit-project --no-hashes --format requirements-txt -o requirements.txt + +precommit: fmt requirements help: @echo "Available targets:" @@ -50,9 +56,11 @@ help: @echo " test Run unit tests" @echo " integration Run integration tests" @echo " fmt Format and lint code" + @echo " precommit Format, lint, and refresh requirements.txt (run before committing)" @echo " clean Remove build artifacts" @echo " docs-install Install docs dependencies (bun)" @echo " docs-clean Remove docs build artifacts" @echo " docs-build Build the static docs site to docs/site" @echo " docs-serve Run the docs dev server (next dev)" @echo " lock-dependencies Write the uv.lock file" + @echo " requirements Generate requirements.txt from the lockfile" diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index 21b44ee..99a4ba5 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -6,8 +6,8 @@ description: Install flowx in Databricks Genie Code, Claude Code, or other agent import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; import { Callout } from 'fumadocs-ui/components/callout'; -Flowx contains four [agent skills](https://github.com/ghanse/flowx/tree/main/skills) (`ingest`, `translate`, `prepare`, and `migrate`) that teach agentic tools how to use the flowx Python modules. -Installing flowx also installs all required dependencies to run the Python modules. +Flowx is a set of [agent skills](https://github.com/ghanse/flowx/tree/main/skills) that can be installed and used with AI coding assistants. +To use these skills, install flowx as a plugin using your AI assistant's preferred installation method. @@ -41,7 +41,7 @@ flowx, run the following command from a Claude Code session: You can also copy the skill folders into your local `/.claude/skills` folder: ```bash -cp -R skills/{ingest,translate,prepare,migrate} ~/.claude/skills/ +cp -R skills/{setup,ingest,translate,prepare,migrate} ~/.claude/skills/ ``` Once installed, the skills can be invoked using `/flowx:migrate`, `/flowx:ingest`, etc. Claude Code can also invoke the skills when prompted. See the [Claude Code Plugins documentation](https://docs.claude.com/en/docs/claude-code/plugins) for more details. @@ -50,7 +50,7 @@ Once installed, the skills can be invoked using `/flowx:migrate`, `/flowx:ingest Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install the flowx skills. The general pattern: -1. Copy each skill folder (`skills/ingest`, `skills/translate`, `skills/prepare`, `skills/migrate`) into the tool's configured skills directory. +1. Copy each skill folder (`skills/setup`, `skills/ingest`, `skills/translate`, `skills/prepare`, `skills/migrate`) into the tool's configured skills directory. 2. Make sure the path contains `SKILL.md` directly, 3. Restart the tool if it caches skill metadata at startup. @@ -64,10 +64,50 @@ cat skills/*/SKILL.md > flowx-skills.md +## Setting up the Python environment + +Flowx's skills invoke Python modules that may depend on third-party packages. The `setup` skill provisions an isolated virtual environment with the required dependencies. + +Run it **once** after installing the skills, before `ingest`, `translate`, `prepare`, or `migrate`. Just ask your agent: + +> Set up the flowx environment + +The setup script can also be run directly from the plugin root: + +```bash +bash /scripts/bootstrap.sh +``` + +Running the setup process will: + +1. Check that `python3`, `pip`, and `venv` are available. +2. Create `/.venv` if it doesn't already exist. +3. Install the `requirements.txt` dependencies into your virtual environment using `pip`. + +The environment is created once and reused. Re-running the script simply confirms the venv exists and its dependencies are satisfied. + + +If `python3`, `pip`, or the `venv` module are missing, the script will print a warning and exit **without** creating anything. +To install Python in your environment, run one of the following commands: + +* **macOS:** `brew install python` +* **Debian/Ubuntu:** `sudo apt-get install python3 python3-venv python3-pip` +* **Windows:** [python.org/downloads](https://www.python.org/downloads/) (enable "Add python.exe to PATH") + + +After the venv exists, every Python command the skills run uses the venv interpreter with `src/` on `PYTHONPATH`: + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" -m flowx.adapter inputs ingest +``` + +On Windows, the interpreter is `\.venv\Scripts\python.exe`. The agent normally runs these commands for you; they are handy for troubleshooting a `ModuleNotFoundError`. + ## Verifying the installation Open your agent and ask: > What flowx skills do you have available? -You should see all four skills listed with their descriptions. If only some appear, double-check the install path your tool watches for skills. +You should see all five skills listed with their descriptions. If only some appear, double-check the install path your tool watches for skills. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..060dab7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,32 @@ +# This file was autogenerated by uv via the following command: +# uv export --frozen --no-dev --no-emit-project --no-hashes --format requirements-txt -o requirements.txt +certifi==2026.5.20 + # via requests +cffi==2.0.0 ; platform_python_implementation != 'PyPy' + # via cryptography +charset-normalizer==3.4.7 + # via requests +cryptography==48.0.0 + # via google-auth +databricks-sdk==0.110.0 + # via flowx +google-auth==2.53.0 + # via databricks-sdk +idna==3.15 + # via requests +protobuf==6.33.6 + # via databricks-sdk +pyasn1==0.6.3 + # via pyasn1-modules +pyasn1-modules==0.4.2 + # via google-auth +pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' + # via cffi +pyyaml==6.0.3 + # via flowx +requests==2.34.2 + # via databricks-sdk +sqlglot==30.8.0 + # via flowx +urllib3==2.7.0 + # via requests diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 0000000..badb2ca --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Bootstraps a Python environment for the flowx plugin. +# +# Creates a virtual environment at /.venv and installs the Python +# dependencies listed in requirements.txt using pip. +# +# If python3, pip, or the venv module are unavailable, the script prints a clear +# warning telling the user what to install and exits non-zero without making changes. +# +# After bootstrapping, run the plugin's Python code with the venv interpreter and +# src/ on PYTHONPATH, e.g.: +# +# PYTHONPATH="/src" "/.venv/bin/python" -m flowx.adapter inputs ingest +# +set -euo pipefail + +# Resolve the plugin root +SOURCE="${BASH_SOURCE[0]}" +while [ -L "$SOURCE" ]; do + DIR="$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)" + SOURCE="$(readlink "$SOURCE")" + [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" +done +SCRIPT_DIR="$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)" +PLUGIN_ROOT="$(cd -P "$SCRIPT_DIR/.." >/dev/null 2>&1 && pwd)" + +VENV_DIR="$PLUGIN_ROOT/.venv" +REQUIREMENTS="$PLUGIN_ROOT/requirements.txt" + +# Verify python3 is available +if ! command -v python3 >/dev/null 2>&1; then + cat >&2 <<'EOF' +WARNING: python3 was not found on your PATH. + +Flowx requires Python 3.12+ to run its translation code. +Please install Python (it bundles pip) before continuing: + + - macOS: brew install python (or https://www.python.org/downloads/) + - Debian/Ubuntu: sudo apt-get install python3 python3-venv python3-pip + - Windows: https://www.python.org/downloads/ (enable "Add python.exe to PATH") + +Re-run this setup step once Python is installed. +EOF + exit 1 +fi + +PYTHON_BIN="$(command -v python3)" + +# Verify pip is available +if ! "$PYTHON_BIN" -m pip --version >/dev/null 2>&1; then + cat >&2 <<'EOF' +WARNING: pip is not available for your python3 installation. + +pip is required to install the flowx plugin's dependencies. Install it with: + + - macOS/Linux: python3 -m ensurepip --upgrade + - Debian/Ubuntu: sudo apt-get install python3-pip + - or follow https://pip.pypa.io/en/stable/installation/ + +Re-run this setup step once pip is installed. +EOF + exit 1 +fi + +# Verify the venv module is available +if ! "$PYTHON_BIN" -m venv --help >/dev/null 2>&1; then + cat >&2 <<'EOF' +WARNING: the Python `venv` module is not available. + +It is required to create the virtual environment. Install it with: + + - Debian/Ubuntu: sudo apt-get install python3-venv + - or reinstall Python from https://www.python.org/downloads/ + +Re-run this setup step once `venv` is available. +EOF + exit 1 +fi + +# Create the virtual environment +if [ ! -x "$VENV_DIR/bin/python" ]; then + echo "Creating virtual environment at $VENV_DIR ..." + "$PYTHON_BIN" -m venv "$VENV_DIR" +else + echo "Using existing virtual environment at $VENV_DIR ..." +fi + +VENV_PYTHON="$VENV_DIR/bin/python" + +# Install dependencies from requirements.txt +if [ ! -f "$REQUIREMENTS" ]; then + echo "ERROR: requirements.txt not found at $REQUIREMENTS" >&2 + exit 1 +fi + +echo "Upgrading pip ..." +"$VENV_PYTHON" -m pip install --quiet --upgrade pip + +echo "Installing dependencies from requirements.txt ..." +"$VENV_PYTHON" -m pip install -r "$REQUIREMENTS" + +cat </scripts/bootstrap.sh +``` + +This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or +pip is missing, the script prints a warning telling the user what to install — relay it and stop +until they have installed Python 3.12+ and pip. + +Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` +(use it anywhere a command below shows `python3`): + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... +``` + ## Workflow Follow these steps in order: diff --git a/skills/migrate/SKILL.md b/skills/migrate/SKILL.md index 3e2b7d0..c0f2b9b 100644 --- a/skills/migrate/SKILL.md +++ b/skills/migrate/SKILL.md @@ -27,6 +27,28 @@ This is the top-level orchestration skill. It runs the full migration pipeline: Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. +## Prerequisite — Python environment + +This skill runs the plugin's Python code, which depends on third-party packages. Before running +any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the +**`setup`** skill, or directly: + +```bash +bash /scripts/bootstrap.sh +``` + +This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or +pip is missing, the script prints a warning telling the user what to install — relay it and stop +until they have installed Python 3.12+ and pip. + +Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` +(use it anywhere a command below shows `python3`): + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... +``` + ## Workflow Follow these steps in order: @@ -180,13 +202,12 @@ The questions the adapter raises: | `consolidate_motif:` | `keep`, `consolidate` | `keep` | DatabricksNotebook and DatabricksSparkPython tasks always inherit the cluster binding derived from -their source linked service; the serverless replacement option was removed because it silently -discarded init scripts and DBR-version pins from the source pipeline. +their source linked service. For each multi-activity motif the detector matches (rest_api_pagination, incremental_load_watermark, metadata_driven_bulk_copy, ...) the adapter emits one -`consolidate_motif:` question. Default is `keep` so motif detection cannot silently -rewrite a pipeline; the user must explicitly opt in to `consolidate` for each pattern. +`consolidate_motif:` question. The user must explicitly opt in to `consolidate` +for each detected pattern. ### Step 6 — Checkpoint: confirm proceed to bundle generation diff --git a/skills/prepare/SKILL.md b/skills/prepare/SKILL.md index 1e6ba9e..b6eed61 100644 --- a/skills/prepare/SKILL.md +++ b/skills/prepare/SKILL.md @@ -26,6 +26,28 @@ The output is a standard DABs project with: - `src/notebooks/` — generated and helper notebooks - `setup/` — infrastructure setup scripts (volumes, secrets, connections) +## Prerequisite — Python environment + +This skill runs the plugin's Python code, which depends on third-party packages. Before running +any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the +**`setup`** skill, or directly: + +```bash +bash /scripts/bootstrap.sh +``` + +This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or +pip is missing, the script prints a warning telling the user what to install — relay it and stop +until they have installed Python 3.12+ and pip. + +Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` +(use it anywhere a command below shows `python3`): + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... +``` + ## Workflow Follow these steps in order: diff --git a/skills/setup/SKILL.md b/skills/setup/SKILL.md new file mode 100644 index 0000000..3d262c8 --- /dev/null +++ b/skills/setup/SKILL.md @@ -0,0 +1,94 @@ +--- +name: setup +description: > + Setup the Python environment for the flowx plugin. Creates a .venv virtual environment + and installs the Python dependencies (from requirements.txt via pip) needed for each phase. + Run this once before any other flowx skill, or whenever dependencies are missing. +triggers: + - "setup flowx" + - "bootstrap flowx" + - "install flowx dependencies" + - "flowx environment" + - "create flowx venv" + - "ModuleNotFoundError flowx" +--- + +# Create the Python Environment + +Create a virtual environment (`.venv`) for the plugin and install its Python dependencies. This +is the prerequisite for the `ingest`, `translate`, `prepare`, and `migrate` skills which run Python +from this environment. + +## Context + +The flowx plugin ships Python code (in `src/flowx/`) that the skills invoke (e.g. +`python -m flowx.adapter ...`, `adf_loader.py`, `engine.py`, `dab_writer.py`). Some code depends +on third-party packages (`pyyaml`, `databricks-sdk`, `sqlglot`). Running it against a bare system +Python fails with `ModuleNotFoundError`. This step provisions an isolated `.venv` with the required +dependencies installed via `pip` from `requirements.txt`. + +The environment is created once and reused. Re-running the bootstrapscript confirms the venv exists +and ensures that dependencies are satisfied. + +## Workflow + +### Step 1 — Run the bootstrap script + +From the plugin root, run: + +```bash +bash /scripts/bootstrap.sh +``` + +Where `` is the root of the flowx plugin (the directory containing `src/`, +`skills/`, and `requirements.txt`). + +The script will: +1. Check that `python3`, `pip`, and the `venv` module are available. +2. Create `/.venv` if it does not already exist. +3. Install dependencies listed in `requirements.txt` into that venv using `pip`. + +### Step 2 — Handle a missing Python or pip + +If Python, pip, or the `venv` module are **not** available, the script prints a `WARNING:` block +explaining what to install and exits non-zero **without** creating anything. + +When this happens, **do not attempt to work around it**. Relay the warning to the user, ask them +to install, and stop: + +> ⚠️ Python must be installed before I can set up the flowx environment. +> +> * On macOS: `brew install python`. +> * On Debian/Ubuntu: `sudo apt-get install python3 python3-venv python3-pip`. +> +> Let me know once it's installed and I'll re-run setup. + +Re-run this setup skill after the user confirms Python and pip are installed. + +### Step 3 — Confirm success and how to run Python code + +On success, the script prints the interpreter path and a usage example. After this, every +Python command in the flowx skills **must** be run with the venv interpreter and `src/` +on `PYTHONPATH`: + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" -m flowx.adapter inputs ingest +``` + +(On Windows the interpreter is `\.venv\Scripts\python.exe`.) + +Use `/.venv/bin/python` anywhere the other skills show `python3`. + +## Output + +| Artifact | Description | +|---|---| +| `/.venv/` | Virtual environment containing the installed dependencies | +| `requirements.txt` | The dependency list installed into the venv | + +## Examples + +- "Set up the flowx environment" +- "Bootstrap flowx so I can run a migration" +- "I got a ModuleNotFoundError running ingest — fix the environment" diff --git a/skills/translate/SKILL.md b/skills/translate/SKILL.md index 75acf1b..0ce95b9 100644 --- a/skills/translate/SKILL.md +++ b/skills/translate/SKILL.md @@ -14,17 +14,37 @@ triggers: # Translate ADF to Databricks IR -Convert the parsed ADF inventory into Databricks intermediate representation (IR) using deterministic translators for known types and agentic fallback for complex/unknown types. +Convert the parsed ADF inventory into Databricks intermediate representation (IR) using deterministic translators for known types and agentic fallback for unknown types. ## Context This is phase 2 of the flowx migration workflow. It consumes the `inventory.json` produced by the `ingest` skill and produces a `translation_report.json` that the `prepare` skill uses to generate Databricks Declarative Automation Bundles. The translation follows a **deterministic-first** strategy: -1. Activities with known, well-defined mappings are translated by built-in Python translators (fast, reliable, no LLM needed) -2. Activities that require interpretation, complex expression conversion, or lack a direct mapping are handled by agentic skills from the `adf-to-databricks-plugin` (LLM-assisted) +1. Activities with known, well-defined mappings are translated by built-in Python translators +2. Activities that require interpretation, expression conversion, or lack a Python translator are handled by agent skills from the `adf-to-databricks-plugin` -This approach maximizes reliability while covering the long tail of ADF activity types. +## Prerequisite — Python environment + +This skill runs the plugin's Python code, which depends on third-party packages. Before running +any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the +**`setup`** skill, or directly: + +```bash +bash /scripts/bootstrap.sh +``` + +This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or +pip is missing, the script prints a warning telling the user what to install — relay it and stop +until they have installed Python 3.12+ and pip. + +Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` +(use it anywhere a command below shows `python3`): + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... +``` ## Workflow @@ -68,12 +88,12 @@ Where: - `` is the root of the flowx plugin - `` is the path to `inventory.json` - `` is the original ADF JSON directory (from the ingest phase) -- `` is where to write translation output (default: `./orchestra_output/translate/`) +- `` is the translation output path (default: `./orchestra_output/translate/`) This produces: - `translation_report.json` — results for deterministic activities + placeholders for agentic gaps -- `ir/` directory — the Databricks IR for each translated activity -- `notebooks/` directory — any generated helper notebooks +- `ir/` directory — Databricks IR for each translated activity +- `notebooks/` directory — generated helper notebooks ### Step 3 — Read the translation report @@ -125,7 +145,7 @@ Invoke `adf-to-databricks:adf-dataflow-converter` with the raw activity JSON and - The raw `typeProperties` from the ADF activity - The data flow JSON definition (if available in the source directory under `dataflow/`) - The linked service configurations for source/sink connections -- Target catalog and schema for the DLT pipeline or PySpark notebook output +- Target catalog and schema for the SDP pipeline or PySpark notebook output **Control flow activities (Switch, Until, Wait, Filter, AppendVariable):** Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: @@ -174,14 +194,14 @@ This updates `translation_report.json` with the agentic results merged in, chang ### Step 6.1 — Gather just-in-time translation preferences The adapter raises several preference questions plus a chained set for -metadata-driven motifs. Drive the loop multi-pass: every time the user -answers a question whose value gates further prompts, re-run `inspect ---answers ` to surface the next batch. +metadata-driven motifs. Every time the user answers a question whose value +gates further prompts, re-run `inspect --answers ` to surface +the next batch. When the metadata-driven flow ends with `metadata_driven_lookup_tool=have` and the agent has a database tool (Genie, MCP SQL, or a workspace SDK), run the lookup query directly and write the rows to -`/lookup_values.json`. When the answer is `none`, prompt +`/lookup_values.json`. When the answer is `none`, prompt the user for a CSV file or comma-separated string and call: ```bash @@ -199,14 +219,13 @@ python3 -m flowx.adapter modify \ --out /translation_report.stamped.json ``` -When no metadata-driven motif is consolidated, `--lookup-values` is -omitted. +When no metadata-driven motif is consolidated, `--lookup-values` is omitted. #### Legacy flow details Before writing the final report, surface any pipeline-modifier questions the IR raises (Copy Data paradigm, non-Databricks task compute, Lakeflow Connect -opt-in, Databricks task compute). Use the adapter CLI bridge: +opt-in, Databricks task compute). Use the adapter CLI bridge: ```bash python3 -m flowx.adapter inspect @@ -236,7 +255,7 @@ The command emits JSON: ``` For each question, prompt the user with the rationale, options, and the -task keys it affects. Use the default when the user defers. Collect the +task keys it affects. Use the default when the user defers. Collect the answers into a JSON file (`/answers.json`) shaped like: ```json @@ -247,8 +266,7 @@ answers into a JSON file (`/answers.json`) shaped like: } ``` -Then apply the answers to produce a stamped report the prepare phase -consumes: +Then apply the answers to produce a stamped report the prepare phase consumes: ```bash python3 -m flowx.adapter modify \ diff --git a/src/orchestra/bundler/dab_writer.py b/src/orchestra/bundler/dab_writer.py index 7d00f0f..05aa8b2 100644 --- a/src/orchestra/bundler/dab_writer.py +++ b/src/orchestra/bundler/dab_writer.py @@ -71,6 +71,12 @@ class _BundleYamlDumper(yaml.SafeDumper): # bundle's ``variables`` block + SETUP.md. _cross_bundle_variables: dict[str, str] = {} +# C-43 (CF5-001 / CF5-002): condition_task operands the dangling-ref safety +# net had to blank. Each entry is {task_key, field, original_ref}. Reset +# per write_bundle call and surfaced as a SETUP.md section so a neutralised +# branch predicate (always-true) is never silent. +_neutralized_conditions: list[dict[str, str]] = [] + _WIDGET_REFERENCE = re.compile(r"""dbutils\.widgets\.get\(\s*["']([^"']+)["']\s*\)""") @@ -98,6 +104,7 @@ def write_bundle( # cross-bundle variables from one bundle into the next. _bundle_warnings.clear() _cross_bundle_variables.clear() + _neutralized_conditions.clear() output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) @@ -250,6 +257,55 @@ def write_bundle( known_bundle_jobs = {resource_key} | {normalize_task_key(inner.name) for inner in workflow.inner_workflows} # ``manual_parameters`` was collected above (before YAML emission) so # the broken values are also stripped from the on-disk YAML. + # VAREX3-003: manual_variable_rollup SetupTasks emitted by + # workflow_preparer surface in SETUP.md so the user knows where to add + # a roll-up notebook. + rollup_configs = [ + st.config + for st in workflow.setup_tasks + if st.type == "manual_variable_rollup" + ] + for inner in workflow.inner_workflows: + rollup_configs.extend( + st.config for st in inner.setup_tasks if st.type == "manual_variable_rollup" + ) + dynamic_dispatch_configs = [ + st.config for st in workflow.setup_tasks if st.type == "dynamic_notebook_dispatch" + ] + unresolved_library_configs = [ + st.config for st in workflow.setup_tasks if st.type == "unresolved_library" + ] + manual_variable_init_configs = [ + st.config for st in workflow.setup_tasks if st.type == "manual_variable_init" + ] + manual_schedule_time_of_day_configs = [ + st.config for st in workflow.setup_tasks if st.type == "manual_schedule_time_of_day" + ] + manual_credential_configs = [ + st.config for st in workflow.setup_tasks if st.type == "manual_credential" + ] + for inner in workflow.inner_workflows: + dynamic_dispatch_configs.extend( + st.config for st in inner.setup_tasks if st.type == "dynamic_notebook_dispatch" + ) + unresolved_library_configs.extend( + st.config for st in inner.setup_tasks if st.type == "unresolved_library" + ) + manual_variable_init_configs.extend( + st.config for st in inner.setup_tasks if st.type == "manual_variable_init" + ) + manual_schedule_time_of_day_configs.extend( + st.config for st in inner.setup_tasks if st.type == "manual_schedule_time_of_day" + ) + manual_credential_configs.extend( + st.config for st in inner.setup_tasks if st.type == "manual_credential" + ) + # LSC3-006: union typed SecretInstructions from the workflow (and + # inner workflows) with the notebook-scanned scopes so SETUP.md and + # create_secrets.py reference the same set of (scope, key) pairs. + all_secret_instructions = list(workflow.secrets) + for inner in workflow.inner_workflows: + all_secret_instructions.extend(inner.secrets) prereqs = build_prereqs( notebooks=all_notebooks, tasks=all_tasks, @@ -257,6 +313,14 @@ def write_bundle( cross_bundle_variables=dict(_cross_bundle_variables), manual_parameters=manual_parameters, parameter_approximations=parameter_approximations, + manual_variable_rollups=rollup_configs, + secret_instructions=all_secret_instructions, + dynamic_notebook_dispatches=dynamic_dispatch_configs, + unresolved_libraries=unresolved_library_configs, + manual_variable_inits=manual_variable_init_configs, + manual_schedule_time_of_day=manual_schedule_time_of_day_configs, + manual_credentials=manual_credential_configs, + neutralized_conditions=list(_neutralized_conditions), ) setup_path = output_dir / "SETUP.md" setup_path.write_text(render_setup_md(prereqs, bundle_name=effective_name), encoding="utf-8") @@ -389,6 +453,38 @@ def _warn(task_key: str, message: str) -> None: _DEFAULT_SPARK_VERSION = "15.4.x-scala2.12" _DEFAULT_NODE_TYPE_ID = "Standard_DS3_v2" +# C-29 (NB-ITER4-002): a real DBR version string matches e.g. +# "15.4.x-scala2.12" / "15.4.x-photon-scala2.12". ADF expressions like +# ``@if(equals(item()?.photon,true),...)`` slip through unfiltered today +# and land in ``databricks.yml`` as the spark_version variable default, +# which bundle deploy rejects. The regex anchors on the canonical +# Databricks Runtime shape so unrecognised strings fall through to the +# safe default. +_DBR_VERSION_RE = re.compile(r"^\d+\.\d+\.x(-[a-z0-9.]+)*$") + + +def _is_valid_spark_version(value: Any) -> bool: + """Return True when *value* parses as a real DBR runtime version string.""" + if not isinstance(value, str) or not value: + return False + return _DBR_VERSION_RE.match(value) is not None + + +def _is_valid_node_type_id(value: Any) -> bool: + """Return True when *value* looks like a real cloud instance type. + + Conservatively rejects anything that starts with ``@`` (an unresolved + ADF expression) or contains spaces; otherwise accepts the value + verbatim so we don't gate out cloud-specific instance families. + """ + if not isinstance(value, str) or not value: + return False + if value.startswith("@"): + return False + if any(ch.isspace() for ch in value): + return False + return True + def _infer_bundle_cluster_defaults(workflow: PreparedWorkflow) -> tuple[str, str]: """Derive ``spark_version`` and ``node_type_id`` defaults from task clusters. @@ -401,14 +497,72 @@ def _infer_bundle_cluster_defaults(workflow: PreparedWorkflow) -> tuple[str, str """ from collections import Counter - spark_versions = [hint["spark_version"] for hint in workflow.cluster_hints if hint.get("spark_version")] - node_types = [hint["node_type_id"] for hint in workflow.cluster_hints if hint.get("node_type_id")] + # C-29 (NB-ITER4-002): filter out unparseable spark_version / + # node_type_id hints before Counter so unresolved ADF expressions + # (e.g. ``@if(equals(item()?.photon,true),...)``) don't land as the + # bundle's default and break ``databricks bundle deploy``. + spark_versions = [ + hint["spark_version"] + for hint in workflow.cluster_hints + if _is_valid_spark_version(hint.get("spark_version")) + ] + node_types = [ + hint["node_type_id"] + for hint in workflow.cluster_hints + if _is_valid_node_type_id(hint.get("node_type_id")) + ] spark_version = Counter(spark_versions).most_common(1)[0][0] if spark_versions else _DEFAULT_SPARK_VERSION node_type_id = Counter(node_types).most_common(1)[0][0] if node_types else _DEFAULT_NODE_TYPE_ID return spark_version, node_type_id +def _infer_bundle_cluster_extras(workflow: PreparedWorkflow) -> dict[str, Any]: + """Surface non-default cluster fields shared across the workflow's tasks. + + Mines :attr:`PreparedWorkflow.cluster_hints` for cluster fields beyond + spark_version / node_type_id (including num_workers) and returns the + consensus values so the default job_cluster reflects ADF settings + end-to-end. + + Args: + workflow: The prepared workflow being written. + + Returns: + Dict of cluster fields ready to merge under ``new_cluster``. Only + the most common value across hints is propagated for each field; + ties are broken by first occurrence. + """ + from collections import Counter + + extras: dict[str, Any] = {} + extra_keys = ( + "num_workers", + "driver_node_type_id", + "data_security_mode", + "spark_env_vars", + "custom_tags", + "init_scripts", + "cluster_log_conf", + "spark_conf", + ) + for key in extra_keys: + values = [hint[key] for hint in workflow.cluster_hints if hint.get(key)] + if not values: + continue + # Use string repr to dedupe non-hashable dict entries while still + # picking the most common. + rep_counter: Counter[str] = Counter() + rep_to_value: dict[str, Any] = {} + for value in values: + rep = repr(value) + rep_counter[rep] += 1 + rep_to_value.setdefault(rep, value) + top_rep, _count = rep_counter.most_common(1)[0] + extras[key] = rep_to_value[top_rep] + return extras + + def _build_databricks_yml( bundle_name: str, catalog: str, @@ -491,40 +645,63 @@ def _build_databricks_yml( } -def _build_default_job_clusters(needed_keys: set[str]) -> list[dict[str, Any]]: +def _build_default_job_clusters( + needed_keys: set[str], + *, + extras: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: """Builds the job_clusters stanza, emitting only the clusters in use. Args: needed_keys: Set of job_cluster_key strings referenced by any task in the workflow. + extras: Optional cluster fields surfaced from per-task hints + (driver_node_type_id, spark_env_vars, custom_tags, ...). When + present these override the corresponding fields of the + multi-purpose default cluster so ADF-derived settings flow + into the emitted YAML. Returns: Ordered list of cluster definitions for inclusion under the job's ``job_clusters`` block. """ - builders = ( - (DEFAULT_JOB_CLUSTER_KEY, _build_default_cluster), + builders: tuple[tuple[str, Any], ...] = ( + (DEFAULT_JOB_CLUSTER_KEY, lambda: _build_default_cluster(extras)), (SINGLE_NODE_JOB_CLUSTER_KEY, _build_single_node_cluster), (MULTI_NODE_JOB_CLUSTER_KEY, _build_multi_node_cluster), ) return [builder() for key, builder in builders if key in needed_keys] -def _build_default_cluster() -> dict[str, Any]: +def _build_default_cluster(extras: dict[str, Any] | None = None) -> dict[str, Any]: """Builds the multi-purpose default job_cluster used for legacy bindings. + Args: + extras: Optional cluster fields lifted from per-task hints to + merge into ``new_cluster`` (num_workers, driver_node_type_id, + spark_env_vars, custom_tags, init_scripts, cluster_log_conf, + spark_conf, data_security_mode). ``num_workers`` overrides the + default single-worker value and ``data_security_mode`` overrides + the default ``SINGLE_USER`` value when supplied. + Returns: - Cluster definition with one worker and bundle-variable knobs for - spark_version and node_type_id. + Cluster definition with the mined (or default single) worker count + and bundle-variable knobs for spark_version and node_type_id, plus + any merged extras. """ + new_cluster: dict[str, Any] = { + "spark_version": "${var.spark_version}", + "node_type_id": "${var.node_type_id}", + "num_workers": 1, + "data_security_mode": "SINGLE_USER", + "single_user_name": "${workspace.current_user.userName}", + } + if extras: + for key, value in extras.items(): + new_cluster[key] = value return { "job_cluster_key": DEFAULT_JOB_CLUSTER_KEY, - "new_cluster": { - "spark_version": "${var.spark_version}", - "node_type_id": "${var.node_type_id}", - "num_workers": 1, - "data_security_mode": "SINGLE_USER", - }, + "new_cluster": new_cluster, } @@ -543,6 +720,7 @@ def _build_single_node_cluster() -> dict[str, Any]: "node_type_id": "${var.node_type_id}", "is_single_node": True, "data_security_mode": "SINGLE_USER", + "single_user_name": "${workspace.current_user.userName}", }, } @@ -561,6 +739,7 @@ def _build_multi_node_cluster() -> dict[str, Any]: "node_type_id": MULTI_NODE_CLUSTER_NODE_TYPE_ID, "num_workers": 2, "data_security_mode": "SINGLE_USER", + "single_user_name": "${workspace.current_user.userName}", }, } @@ -638,16 +817,22 @@ def _value_needs_manual_handling(value: Any) -> bool: def _extract_manual_parameters_from_existing_notebook_tasks( tasks: list[dict[str, Any]], ) -> list[ManualParameter]: - """Finds base_parameters flowx couldn't evaluate for existing-notebook tasks.""" + """Finds base_parameters flowx couldn't evaluate for notebook tasks. + + Previously this scan skipped stub notebooks under ``../src/`` because + their bodies could (in principle) be patched to inline the runtime + computation. In practice stub tasks for activities that have no + deterministic translation are emitted with raw ADF expression values + that ``dbutils.widgets.get`` returns verbatim, which fails at runtime. + Walking both the absolute-path and bundle-relative cases drops the + broken values and surfaces them as a SETUP.md row instead. + """ manual_parameters: list[ManualParameter] = [] for task in _iter_tasks_recursively(tasks): notebook_task = task.get("notebook_task") or {} notebook_path = notebook_task.get("notebook_path", "") base_params = notebook_task.get("base_parameters") - # Bundle-relative paths (``../src/...``) can have their notebook - # bodies patched to inline the runtime computation; absolute paths - # belong to the user's existing notebooks and must be surfaced. - if not notebook_path.startswith("/") or not isinstance(base_params, dict): + if not isinstance(base_params, dict): continue keys_to_drop: list[str] = [] for key, value in base_params.items(): @@ -708,17 +893,45 @@ def _bind_cluster_to_notebook_tasks(tasks: list[dict[str, Any]]) -> None: continue compute_mode = task.get("_compute_mode") if compute_mode == "serverless": + # Serverless cannot host jar/whl libraries. When the task + # ships libraries we must still bind a classic cluster so the + # Jobs API accepts the libraries block. + if _task_has_jar_or_whl_libraries(task): + task["job_cluster_key"] = DEFAULT_JOB_CLUSTER_KEY continue cluster_key = COMPUTE_MODE_TO_CLUSTER_KEY.get(compute_mode or "") if cluster_key is not None: task["job_cluster_key"] = cluster_key continue notebook_path = notebook_task.get("notebook_path", "") + # Stub notebooks (../src/...) are normally left unbound for + # serverless compute. But when libraries are attached we must + # bind to a real cluster (NB-2) -- serverless cannot install + # jar / whl libraries. if notebook_path.startswith("../src/"): + if _task_has_jar_or_whl_libraries(task): + task["job_cluster_key"] = DEFAULT_JOB_CLUSTER_KEY continue task["job_cluster_key"] = DEFAULT_JOB_CLUSTER_KEY +def _task_has_jar_or_whl_libraries(task: dict[str, Any]) -> bool: + """Return True when *task* references a library shape that needs a cluster. + + JAR / EGG / whl / PyPI / Maven / CRAN entries all require a classic + cluster — they cannot be installed on serverless. Requirements files + are treated the same way to be safe. + """ + libs = task.get("libraries") + if not isinstance(libs, list): + return False + cluster_required = {"jar", "egg", "whl", "maven", "pypi", "cran", "requirements"} + for entry in libs: + if isinstance(entry, dict) and any(key in entry for key in cluster_required): + return True + return False + + def _rewrite_post_branch_dependencies(tasks: list[dict[str, Any]]) -> None: """Rewrites ``depends_on`` edges that target a condition_task to target its branches. @@ -783,24 +996,128 @@ def expand_terminals(condition_key: str, seen: set[str]) -> list[str]: _TASK_VALUE_REF = re.compile(r"\{\{tasks\.([^.]+)\.values\.[^}]+\}\}") -def _strip_dangling_task_value_refs(tasks: list[dict[str, Any]], all_task_keys: set[str]) -> None: +def _apply_schedule_to_job(job_def: dict[str, Any], spec: dict[str, Any]) -> None: + """Renders a workflow schedule spec onto a DAB job definition. + + C-10 (SCHED-001): translates the structured schedule dict produced by + ``engine._adf_trigger_to_schedule`` into either ``schedule:`` or + ``trigger:`` keys on the job YAML. Best-effort schedule shapes + (Tumbling / CustomEvents) fall back to a comment-style placeholder + so SETUP.md can capture them. + """ + kind = spec.get("kind") + if kind == "schedule": + if "quartz_cron_expression" in spec: + schedule_block: dict[str, Any] = { + "quartz_cron_expression": spec["quartz_cron_expression"], + "timezone_id": spec.get("timezone_id", "UTC"), + } + if spec.get("pause_status"): + schedule_block["pause_status"] = spec["pause_status"] + job_def["schedule"] = schedule_block + return + # Tumbling fallback -- attach a hint rather than emitting a + # malformed schedule. SETUP.md picks it up downstream. + job_def["schedule_setup_note"] = spec + return + if kind == "periodic": + # SCHED3-002: Day/Week/Month with interval > 1 maps to trigger.periodic. + trigger_block: dict[str, Any] = { + "periodic": { + "interval": spec.get("interval", 1), + "unit": spec.get("unit", "DAYS"), + } + } + if spec.get("pause_status"): + trigger_block["pause_status"] = spec["pause_status"] + job_def["trigger"] = trigger_block + return + if kind == "file_arrival": + trigger_block = { + "file_arrival": {"url": spec.get("url", "")}, + } + if spec.get("pause_status"): + trigger_block["pause_status"] = spec["pause_status"] + job_def["trigger"] = trigger_block + return + if kind == "manual_setup": + # No DAB primitive -- surface the raw spec so SETUP.md can flag it. + job_def["schedule_setup_note"] = spec + return + + +def _strip_dangling_task_value_refs( + tasks: list[dict[str, Any]], + all_task_keys: set[str], +) -> list[dict[str, str]]: """Replaces ``{{tasks.X.values.Y}}`` refs whose ``X`` is not in the bundle. + C-12 (VAREX-005): in addition to ``notebook_task.base_parameters``, + walk ``run_job_task.job_parameters``, ``condition_task.left`` and + ``condition_task.right``, plus the nested for_each body so cross-job + parameter passing surfaces are caught. C-05 fixes most variable + defaults; this safety net catches the residual cases (renames, + scoped-out variables) by emitting an empty string in place of the + dangling ref so SETUP.md §4 can flag it. + + C-43 (CF5-001 / CF5-002): blanking a *condition_task* operand silently + turns ``NOT_EQUAL('', '0')`` into an always-true predicate, so the + branch runs unconditionally with no signal. This function now records + each condition operand it neutralises and returns them so the caller + can surface a 'conditions neutralized — manual re-wiring required' + section in SETUP.md instead of failing silently. + Args: tasks: Top-level tasks for one job (mutated in place). all_task_keys: Task keys that do exist in this job (including those inside ``for_each_task.task`` bodies). + + Returns: + List of ``{task_key, field, original_ref}`` dicts for every + condition operand that was blanked. """ + neutralized: list[dict[str, str]] = [] + + def _is_dangling(value: Any) -> bool: + if not isinstance(value, str): + return False + match = _TASK_VALUE_REF.search(value) + return bool(match and match.group(1) not in all_task_keys) def visit(task: dict[str, Any]) -> None: notebook_task = task.get("notebook_task") or {} base_parameters = notebook_task.get("base_parameters") or {} for widget_name, value in list(base_parameters.items()): - if not isinstance(value, str): - continue - match = _TASK_VALUE_REF.search(value) - if match and match.group(1) not in all_task_keys: + if _is_dangling(value): base_parameters[widget_name] = "" + + # C-12: run_job_task.job_parameters references the parent's task + # values when crossing into an inner job. Strip dangling refs. + run_job_task = task.get("run_job_task") or {} + job_parameters = run_job_task.get("job_parameters") or {} + if isinstance(job_parameters, dict): + for param_name, value in list(job_parameters.items()): + if _is_dangling(value): + job_parameters[param_name] = "" + + # C-12: condition_task operands can also carry dangling refs + # when an upstream renamed task disappeared between rewrite + # passes. C-43: record each neutralised operand for SETUP.md. + condition_task = task.get("condition_task") or {} + if condition_task: + task_key = task.get("task_key", "") + for field_name in ("left", "right"): + operand = condition_task.get(field_name) + if _is_dangling(operand): + neutralized.append( + { + "task_key": str(task_key), + "field": field_name, + "original_ref": str(operand), + } + ) + condition_task[field_name] = "" + for_each = task.get("for_each_task") if for_each and isinstance(for_each.get("task"), dict): visit(for_each["task"]) @@ -808,6 +1125,8 @@ def visit(task: dict[str, Any]) -> None: for task in tasks: visit(task) + return neutralized + def _collect_all_task_keys(tasks: list[dict[str, Any]]) -> set[str]: """Collects every task_key reachable from the job's top-level task list.""" @@ -877,8 +1196,12 @@ def _build_job_resource( _augment_base_parameters(workflow.tasks, augment_scope) # Task values don't cross ``run_job_task`` boundaries; any such # reference in this job resolves to an empty string at runtime. Emit - # the empty string now so SETUP.md §4 flags it. - _strip_dangling_task_value_refs(workflow.tasks, _collect_all_task_keys(workflow.tasks)) + # the empty string now so SETUP.md §4 flags it. C-43: a blanked + # condition operand silently makes the predicate always-true, so record + # each neutralised condition for the SETUP.md re-wiring section. + _neutralized_conditions.extend( + _strip_dangling_task_value_refs(workflow.tasks, _collect_all_task_keys(workflow.tasks)) + ) job_def: dict[str, Any] = { "name": workflow.name, @@ -889,13 +1212,32 @@ def _build_job_resource( _bind_cluster_to_notebook_tasks(workflow.tasks) needed_keys = _collect_required_cluster_keys(workflow.tasks) if needed_keys: - job_def["job_clusters"] = _build_default_job_clusters(needed_keys) + cluster_extras = _infer_bundle_cluster_extras(workflow) + job_def["job_clusters"] = _build_default_job_clusters( + needed_keys, + extras=cluster_extras or None, + ) _strip_compute_mode_markers(workflow.tasks) if workflow.parameters: job_def["parameters"] = workflow.parameters + # C-10 (SCHED-001): render the workflow schedule / trigger spec. + schedule_spec = getattr(workflow, "schedule", None) + if schedule_spec: + _apply_schedule_to_job(job_def, schedule_spec) + # SCHED3-003: trigger-supplied per-pipeline parameter overrides + # update the matching job.parameter defaults so scheduled runs + # receive the trigger's pinned values instead of the bare pipeline + # default. Overrides only mutate existing declared parameters; + # unknown names are silently ignored to keep job_def well-formed. + overrides = schedule_spec.get("parameter_overrides") or {} + if overrides and job_def.get("parameters"): + for entry in job_def["parameters"]: + if entry.get("name") in overrides: + entry["default"] = overrides[entry["name"]] + return { "resources": { "jobs": { @@ -961,6 +1303,8 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]: # format, so secret discovery / setup tasks / control-flow handling # all match. pipelines: dict[str, list[dict]] = {} + pipeline_params: dict[str, list[dict[str, Any]]] = {} + pipeline_schedules: dict[str, dict[str, Any]] = {} for translation in report.get("translations", []): pipeline_name = translation.get("pipeline", "unknown") if translation.get("status") != "translated": @@ -969,9 +1313,27 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]: if not ir: continue pipelines.setdefault(pipeline_name, []).append(ir) + # Round-trip pipeline-level parameters either supplied per- + # translation (newer report shape) or alongside the ir under + # an ``ir.parameters`` key (older single-pipeline serialisations + # roundtripped through this aggregator). + params = translation.get("parameters") or ir.get("parameters") + if params and pipeline_name not in pipeline_params: + pipeline_params[pipeline_name] = list(params) + # Likewise carry pipeline-level ``schedule`` through to the + # rehydrated pipeline_dict so trigger-derived schedule / trigger + # blocks survive the aggregated report shape. + schedule = translation.get("schedule") or ir.get("schedule") + if schedule and pipeline_name not in pipeline_schedules: + pipeline_schedules[pipeline_name] = dict(schedule) for pipeline_name, task_irs in pipelines.items(): - workflow = _pipeline_dict_to_workflow({"name": pipeline_name, "tasks": task_irs}) + pipeline_dict: dict[str, Any] = {"name": pipeline_name, "tasks": task_irs} + if pipeline_params.get(pipeline_name): + pipeline_dict["parameters"] = pipeline_params[pipeline_name] + if pipeline_schedules.get(pipeline_name): + pipeline_dict["schedule"] = pipeline_schedules[pipeline_name] + workflow = _pipeline_dict_to_workflow(pipeline_dict) workflows.append(workflow) return workflows @@ -1014,13 +1376,24 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d for param in pipeline_dict.get("parameters") or []: entry: dict[str, Any] = {"name": param["name"]} if "default" in param and param["default"] is not None: - entry["default"] = normalize_value(str(param["default"])) + default_value = param["default"] + # Bool / int / float defaults must survive the JSON round-trip + # as their declared type so the emitted YAML carries a real + # boolean / number, not a quoted string. String defaults go + # through normalize_value to resolve embedded ADF refs. + if isinstance(default_value, bool): + entry["default"] = default_value + elif isinstance(default_value, (int, float)): + entry["default"] = default_value + else: + entry["default"] = normalize_value(str(default_value)) parameters.append(entry) pipeline = Pipeline( name=pipeline_dict.get("name", "unknown"), tasks=activities, parameters=parameters or None, translation_preferences=_reconstruct_preferences(pipeline_dict.get("translation_preferences")), + schedule=pipeline_dict.get("schedule"), ) return pipeline, parameters @@ -1103,6 +1476,7 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: value_kind=task_ir.get("value_kind", "literal"), notebook_code=task_ir.get("notebook_code"), notebook_imports=task_ir.get("notebook_imports", []), + raw_expression=task_ir.get("raw_expression"), ) if task_type == "WaitActivity": return WaitActivity( @@ -1138,6 +1512,9 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: **base, notebook_path=task_ir.get("notebook_path", ""), base_parameters=task_ir.get("base_parameters"), + notebook_path_unresolved=bool(task_ir.get("notebook_path_unresolved", False)), + notebook_path_expression=task_ir.get("notebook_path_expression"), + unresolved_libraries=list(task_ir.get("unresolved_libraries") or []), ) if task_type == "SparkJarActivity": return SparkJarActivity( @@ -1171,6 +1548,11 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: items_expression=task_ir.get("items_expression", ""), inner_activities=[_reconstruct_ir(child) for child in task_ir.get("inner_activities") or []], concurrency=task_ir.get("concurrency"), + # C-31 (CF4-001): preserve bridge fields so the preparer can + # synthesise the inputs bridge after a JSON roundtrip. + inputs_bridge_notebook_code=task_ir.get("inputs_bridge_notebook_code"), + inputs_bridge_notebook_imports=list(task_ir.get("inputs_bridge_notebook_imports") or []), + inputs_bridge_required_parameters=dict(task_ir.get("inputs_bridge_required_parameters") or {}), ) if task_type == "IfConditionActivity": return IfConditionActivity( @@ -1180,6 +1562,12 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: right=task_ir.get("right", ""), if_true_activities=[_reconstruct_ir(child) for child in task_ir.get("if_true_activities") or []], if_false_activities=[_reconstruct_ir(child) for child in task_ir.get("if_false_activities") or []], + # C-14 (CF3-001 / VAREX3-001): preserve bridge fields so the + # preparer can re-synthesise the hidden _bridge SetVariable task + # after a JSON roundtrip. + bridge_notebook_code=task_ir.get("bridge_notebook_code"), + bridge_notebook_imports=list(task_ir.get("bridge_notebook_imports") or []), + bridge_required_parameters=dict(task_ir.get("bridge_required_parameters") or {}), ) if task_type == "SwitchActivity": return SwitchActivity( @@ -1193,6 +1581,12 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: for case in task_ir.get("cases") or [] ], default_activities=[_reconstruct_ir(child) for child in task_ir.get("default_activities") or []], + # C-14 (CF3-001 / VAREX3-001): preserve bridge fields for Switch + # so the preparer can re-synthesise the bridge task after a + # JSON roundtrip. + bridge_notebook_code=task_ir.get("bridge_notebook_code"), + bridge_notebook_imports=list(task_ir.get("bridge_notebook_imports") or []), + bridge_required_parameters=dict(task_ir.get("bridge_required_parameters") or {}), ) if task_type == "MotifActivity": return MotifActivity( diff --git a/src/orchestra/bundler/inner_job_params.py b/src/orchestra/bundler/inner_job_params.py index 20e4e58..bbb51b3 100644 --- a/src/orchestra/bundler/inner_job_params.py +++ b/src/orchestra/bundler/inner_job_params.py @@ -38,6 +38,7 @@ def collect_inner_job_params( tasks: list[dict[str, Any]], *, raw_ir_tasks: list[dict[str, Any]] | None = None, + variable_task_keys: dict[str, str] | None = None, ) -> tuple[list[dict[str, Any]], dict[str, str]]: """Scans task dicts for parameter references and return declarations + pass-through map. @@ -46,6 +47,15 @@ def collect_inner_job_params( raw_ir_tasks: Optional raw IR dicts (before DAB conversion) to scan for references in fields that are consumed during conversion (e.g. WebActivity ``url``, ``body``). + variable_task_keys: C-06 (VAREX-004): mapping of pipeline-variable + names to the setter task_key on the parent job that owns the + variable. Names that match a variable do NOT get declared as + inner-job parameters; instead the parent passes the variable's + task-value reference (``{{tasks..values.}}``) + through the ``job_parameters`` map. Without this the parent + would emit ``{{job.parameters.}}`` referring to a name + that's never declared on the parent job, so the inner job + would receive an empty string. Returns: Tuple of: @@ -53,19 +63,30 @@ def collect_inner_job_params( the inner job definition. - ``job_parameters``: dict mapping param name -> parent expression, suitable for the ``run_job_task.job_parameters`` block. ``item`` - always maps to ``"{{input}}"``, pipeline/variable params map to - ``"{{job.parameters.}}"``. + always maps to ``"{{input}}"``, pipeline params map to + ``"{{job.parameters.}}"``, variables resolved via the + *variable_task_keys* map route through ``{{tasks.X.values.Y}}``. """ param_names: set[str] = set() item_field_names: set[str] = set() + variable_names: set[str] = set() - _scan_tasks(tasks, param_names, item_field_names=item_field_names) + _scan_tasks(tasks, param_names, item_field_names=item_field_names, variable_names=variable_names) if raw_ir_tasks: - _scan_ir_tasks(raw_ir_tasks, param_names, item_field_names=item_field_names) + _scan_ir_tasks( + raw_ir_tasks, param_names, item_field_names=item_field_names, variable_names=variable_names + ) + + var_task_keys = variable_task_keys or {} parameters: list[dict[str, Any]] = [] for name in sorted(param_names): + # C-06: variables with a known setter task on the parent job route + # via {{tasks.X.values.Y}} -- they must NOT show up as inner-job + # parameter declarations. + if name in variable_names and name in var_task_keys: + continue param: dict[str, Any] = {"name": name} if name != "item": param["default"] = "" @@ -80,6 +101,9 @@ def collect_inner_job_params( job_parameters[name] = "{{input}}" elif name in item_field_names: job_parameters[name] = "{{input." + name + "}}" + elif name in variable_names and name in var_task_keys: + setter = var_task_keys[name] + job_parameters[name] = "{{tasks." + setter + ".values." + name + "}}" else: job_parameters[name] = "{{job.parameters." + name + "}}" @@ -119,6 +143,7 @@ def _scan_tasks( param_names: set[str], *, item_field_names: set[str] | None = None, + variable_names: set[str] | None = None, ) -> None: """Recursively scan task dicts for ADF parameter references. @@ -126,28 +151,31 @@ def _scan_tasks( tasks: List of task dicts to scan. param_names: Accumulator set of discovered parameter names. item_field_names: Optional accumulator for field names from item().field refs. + variable_names: Optional accumulator for names sourced from + ``variables('X')`` references (separate from pipeline params). """ + kw: dict[str, Any] = {"item_field_names": item_field_names, "variable_names": variable_names} for task in tasks: notebook_task = task.get("notebook_task", {}) params = notebook_task.get("base_parameters", {}) for value in params.values(): - _extract_refs(value, param_names, item_field_names=item_field_names) + _extract_refs(value, param_names, **kw) run_job_task = task.get("run_job_task", {}) for value in run_job_task.get("job_parameters", {}).values(): - _extract_refs(value, param_names, item_field_names=item_field_names) + _extract_refs(value, param_names, **kw) condition_task = task.get("condition_task", {}) if condition_task: - _extract_refs(condition_task.get("left", ""), param_names, item_field_names=item_field_names) - _extract_refs(condition_task.get("right", ""), param_names, item_field_names=item_field_names) - _scan_tasks(condition_task.get("if_true", []), param_names, item_field_names=item_field_names) - _scan_tasks(condition_task.get("if_false", []), param_names, item_field_names=item_field_names) + _extract_refs(condition_task.get("left", ""), param_names, **kw) + _extract_refs(condition_task.get("right", ""), param_names, **kw) + _scan_tasks(condition_task.get("if_true", []), param_names, **kw) + _scan_tasks(condition_task.get("if_false", []), param_names, **kw) for_each_task = task.get("for_each_task", {}) body = for_each_task.get("task") if body: - _scan_tasks([body], param_names, item_field_names=item_field_names) + _scan_tasks([body], param_names, **kw) def _scan_ir_tasks( @@ -155,6 +183,7 @@ def _scan_ir_tasks( param_names: set[str], *, item_field_names: set[str] | None = None, + variable_names: set[str] | None = None, ) -> None: """Scans raw IR task dicts for parameter references in all fields. @@ -162,8 +191,9 @@ def _scan_ir_tasks( ir_tasks: Raw serialised IR task dicts. param_names: Accumulator set of discovered parameter names. item_field_names: Optional accumulator for field names from item().field refs. + variable_names: Optional accumulator for variable names. """ - field_name_kwargs = {"item_field_names": item_field_names} + field_name_kwargs = {"item_field_names": item_field_names, "variable_names": variable_names} for task_dict in ir_tasks: _extract_refs(task_dict.get("url", ""), param_names, **field_name_kwargs) _extract_refs(task_dict.get("body"), param_names, **field_name_kwargs) @@ -195,6 +225,7 @@ def _extract_refs( param_names: set[str], *, item_field_names: set[str] | None = None, + variable_names: set[str] | None = None, ) -> None: """Extracts parameter names from a single value that may be a string or ADF expression dict.""" text = "" @@ -211,6 +242,8 @@ def _extract_refs( for match in _VARIABLES_RE.finditer(text): param_names.add(match.group(1)) + if variable_names is not None: + variable_names.add(match.group(1)) for match in _ITEM_FIELD_RE.finditer(text): field_name = match.group(1) diff --git a/src/orchestra/bundler/prereqs_writer.py b/src/orchestra/bundler/prereqs_writer.py index fe344f6..a66f7e5 100644 --- a/src/orchestra/bundler/prereqs_writer.py +++ b/src/orchestra/bundler/prereqs_writer.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from typing import Any -from flowx.models.dab import DabNotebook, ParameterApproximation +from flowx.models.dab import DabNotebook, ParameterApproximation, SecretInstruction # Regexes used to mine the generated artifacts for external dependencies. # Kept as compiled patterns so :func:`build_prereqs` is cheap to call. @@ -111,6 +111,33 @@ class Prereqs: network_endpoints: list[NetworkEndpoint] = field(default_factory=list) manual_parameters: list[ManualParameter] = field(default_factory=list) parameter_approximations: list[ParameterApproximation] = field(default_factory=list) + # VAREX3-003: variables mutated inside a ForEach inner-job that a + # sibling task reads. Each entry is the SetupTask.config dict shape + # ({variable_name, parent_foreach, message}). + manual_variable_rollups: list[dict[str, Any]] = field(default_factory=list) + # C-28 (NB-ITER4-001): notebook activities whose ADF ``notebookPath`` is + # a runtime expression the translator couldn't resolve. Each entry is + # the SetupTask.config dict ({task_key, activity_name, expression, + # widget_name}). + dynamic_notebook_dispatches: list[dict[str, Any]] = field(default_factory=list) + # C-30 (NB-ITER4-003): library descriptor jar/whl paths the translator + # couldn't resolve to a literal/dab_ref. Each entry is the SetupTask + # config dict ({task_key, library_type, expression, missing}). + unresolved_libraries: list[dict[str, Any]] = field(default_factory=list) + # C-33 (VAREX4-001/CF4-003): SetVariable activities whose ADF + # expression couldn't be lowered. Each entry is the SetupTask config + # dict ({task_key, variable_name, expression}). + manual_variable_inits: list[dict[str, Any]] = field(default_factory=list) + # C-36 (SCHED4-001): scheduled jobs whose recurrence carried + # hours/minutes/weekDays the cron emitter could not encode. + manual_schedule_time_of_day: list[dict[str, Any]] = field(default_factory=list) + # C-39 (LSC4-004): MSI / CredentialReference cluster substitutions. + manual_credentials: list[dict[str, Any]] = field(default_factory=list) + # C-43 (CF5-001 / CF5-002): condition_task operands the bundler had to + # blank because they referenced a task in another job. Each entry is + # {task_key, field, original_ref}. A blanked operand makes the + # predicate always-true, so the user must re-wire the condition. + neutralized_conditions: list[dict[str, str]] = field(default_factory=list) def is_empty(self) -> bool: """Return ``True`` when nothing needs to happen before ``bundle run``.""" @@ -123,6 +150,13 @@ def is_empty(self) -> bool: and not self.network_endpoints and not self.manual_parameters and not self.parameter_approximations + and not self.manual_variable_rollups + and not self.dynamic_notebook_dispatches + and not self.unresolved_libraries + and not self.manual_variable_inits + and not self.manual_schedule_time_of_day + and not self.manual_credentials + and not self.neutralized_conditions ) @@ -326,6 +360,14 @@ def build_prereqs( compute_notes: list[str] | None = None, manual_parameters: list[ManualParameter] | None = None, parameter_approximations: list[ParameterApproximation] | None = None, + manual_variable_rollups: list[dict[str, Any]] | None = None, + secret_instructions: list[SecretInstruction] | None = None, + dynamic_notebook_dispatches: list[dict[str, Any]] | None = None, + unresolved_libraries: list[dict[str, Any]] | None = None, + manual_variable_inits: list[dict[str, Any]] | None = None, + manual_schedule_time_of_day: list[dict[str, Any]] | None = None, + manual_credentials: list[dict[str, Any]] | None = None, + neutralized_conditions: list[dict[str, str]] | None = None, ) -> Prereqs: """Assemble a :class:`Prereqs` from the bundle's generated artifacts. @@ -351,8 +393,16 @@ def build_prereqs( # the tasks (in case upstream still emits them). cross_bundle.extend(collect_cross_bundle_refs(tasks, known_bundle_jobs)) + # LSC3-006: union notebook-scanned secrets with the workflow's typed + # SecretInstruction list so SETUP.md Option A (scope/key checklist) and + # Option B (create_secrets.py from workflow.secrets) reference the same + # set of (scope, key) pairs. De-dupe by hash; later additions don't + # overwrite earlier values. + secrets = scan_notebooks_for_secrets(notebooks) + for instruction in secret_instructions or []: + secrets.setdefault(instruction.scope, set()).add(instruction.key) return Prereqs( - secrets=scan_notebooks_for_secrets(notebooks), + secrets=secrets, missing_notebooks=collect_missing_notebooks(notebooks, tasks), cross_bundle_refs=cross_bundle, empty_parameters=collect_empty_parameters(tasks), @@ -360,6 +410,13 @@ def build_prereqs( network_endpoints=collect_network_endpoints(notebooks), manual_parameters=list(manual_parameters or []), parameter_approximations=list(parameter_approximations or []), + manual_variable_rollups=list(manual_variable_rollups or []), + dynamic_notebook_dispatches=list(dynamic_notebook_dispatches or []), + unresolved_libraries=list(unresolved_libraries or []), + manual_variable_inits=list(manual_variable_inits or []), + manual_schedule_time_of_day=list(manual_schedule_time_of_day or []), + manual_credentials=list(manual_credentials or []), + neutralized_conditions=list(neutralized_conditions or []), ) @@ -542,6 +599,162 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: ) lines.append("") + if prereqs.dynamic_notebook_dispatches: + lines.append("## Dynamic notebook dispatch") + lines.append("") + lines.append( + "The ADF activities below carried a runtime expression for " + "`notebookPath`. Flowx emitted a dispatch-stub notebook for " + "each one that reads the resolved path from the listed widget and " + "calls `dbutils.notebook.run()`. Supply the widget value at job " + "runtime (via `--params`, a parent task value, or job parameter " + "default) so the stub can dispatch to the correct notebook." + ) + lines.append("") + lines.append("| Task | Activity | Widget | Original ADF expression |") + lines.append("|---|---|---|---|") + for entry in prereqs.dynamic_notebook_dispatches: + task_key = entry.get("task_key", "") + activity_name = entry.get("activity_name", "") + widget_name = entry.get("widget_name", "") + expression = entry.get("expression", "") + lines.append( + f"| `{task_key}` | `{activity_name}` | `dbutils.widgets.get('{widget_name}')` | `{expression}` |" + ) + lines.append("") + + if prereqs.unresolved_libraries: + lines.append("## Unresolved libraries") + lines.append("") + lines.append( + "The library descriptors below carried ADF expressions that " + "couldn't be reduced to a real path or DAB reference. Without " + "resolution the cluster would try to install a file literally " + "named like the expression and fail at job-run time. Either " + "populate the missing identifiers (see the `Missing` column) " + "or replace the entry with a static path before deploying." + ) + lines.append("") + lines.append("| Task | Library type | Expression | Missing |") + lines.append("|---|---|---|---|") + for entry in prereqs.unresolved_libraries: + task_key = entry.get("task_key", "") + lib_type = entry.get("library_type", "") + expression = entry.get("expression", "") + missing = ", ".join(entry.get("missing") or []) or "*(unknown)*" + lines.append(f"| `{task_key}` | `{lib_type}` | `{expression}` | {missing} |") + lines.append("") + + if prereqs.manual_variable_inits: + lines.append("## Manual variable initialisation") + lines.append("") + lines.append( + "The ADF SetVariable activities below carried expressions the " + "translator couldn't lower. Flowx blanked the variable's " + "initial value to keep the bundle YAML valid. Compute the real " + "value yourself (e.g. via a parent task value or runtime widget) " + "before downstream tasks read the variable." + ) + lines.append("") + lines.append("| Task | Variable | Original ADF expression |") + lines.append("|---|---|---|") + for entry in prereqs.manual_variable_inits: + task_key = entry.get("task_key", "") + variable_name = entry.get("variable_name", "") + expression = entry.get("expression", "") + lines.append(f"| `{task_key}` | `{variable_name}` | `{expression}` |") + lines.append("") + + if prereqs.manual_schedule_time_of_day: + lines.append("## Manual schedule time-of-day") + lines.append("") + lines.append( + "The ADF triggers below declared a `schedule` block (hours / " + "minutes / weekDays) the cron emitter couldn't fully encode. " + "Review the spec and add the desired time-of-day to the job's " + "`schedule.quartz_cron_expression` manually." + ) + lines.append("") + lines.append("| Pipeline | Frequency | Interval | Time-of-day spec |") + lines.append("|---|---|---|---|") + for entry in prereqs.manual_schedule_time_of_day: + pipeline = entry.get("pipeline", "") + frequency = entry.get("frequency", "") + interval = entry.get("interval", "") + tod_spec = entry.get("time_of_day_note", "") + lines.append(f"| `{pipeline}` | `{frequency}` | `{interval}` | `{tod_spec}` |") + lines.append("") + + if prereqs.manual_credentials: + lines.append("## Manual credential setup") + lines.append("") + lines.append( + "The cluster compute backing the tasks below was authenticated in " + "ADF via a managed identity / CredentialReference that has no " + "direct Databricks equivalent. Flowx defaulted the bundle's " + "default_cluster to `single_user_name: ${workspace.current_user.userName}` " + "so deployment works for the deploying user, but production runs " + "should swap that for a service principal." + ) + lines.append("") + lines.append("| Source | Linked service | ADF authentication | Suggested Databricks setup |") + lines.append("|---|---|---|---|") + for entry in prereqs.manual_credentials: + source = entry.get("activity_name") or entry.get("source", "") + linked_service = entry.get("linked_service", "") + auth = entry.get("authentication", "") + note = entry.get( + "note", + "Swap `single_user_name` to the SP application ID or set " + "`run_as.service_principal_name` on the job.", + ) + lines.append(f"| `{source}` | `{linked_service}` | `{auth}` | {note} |") + lines.append("") + + if prereqs.neutralized_conditions: + lines.append("## Conditions neutralized to always-true — manual re-wiring required") + lines.append("") + lines.append( + "The IfCondition tasks below referenced a task value that lives only " + "in another job (typically a parent-job init task hoisted out of a " + "split-out ForEach inner job). Databricks task values cannot cross " + "`run_job_task` boundaries, so Flowx blanked the operand. A blanked " + "operand makes the predicate `NOT_EQUAL('', '0')` **always true**, so the " + "branch now runs unconditionally. Re-wire each condition below — either " + "recompute the operand inside this job or pass it as a job parameter." + ) + lines.append("") + lines.append("| Condition task | Operand | Original reference |") + lines.append("|---|---|---|") + for entry in prereqs.neutralized_conditions: + task_key = entry.get("task_key", "") + field_name = entry.get("field", "") + original = entry.get("original_ref", "") + lines.append(f"| `{task_key}` | `{field_name}` | `{original}` |") + lines.append("") + + if prereqs.manual_variable_rollups: + lines.append("## Manual variable roll-ups") + lines.append("") + lines.append( + "These variables are mutated inside a ForEach inner-job but read by a " + "sibling task in the parent. Databricks task values cannot cross " + "`run_job_task` boundaries, so the sibling reads the stale init value. " + "Add a roll-up notebook task that copies the final value back to a " + "parent-scope task value before the sibling task runs." + ) + lines.append("") + lines.append("| Variable | ForEach task | Workaround |") + lines.append("|---|---|---|") + for rollup in prereqs.manual_variable_rollups: + var_name = rollup.get("variable_name", "") + parent_key = rollup.get("parent_foreach", "") + message = rollup.get("message", "") + lines.append( + f"| `{var_name}` | `{parent_key}` | {message} |" + ) + lines.append("") + if prereqs.network_endpoints: lines.append("## Networking") lines.append("") diff --git a/src/orchestra/bundler/setup_generator.py b/src/orchestra/bundler/setup_generator.py index e637cf5..3bce6f2 100644 --- a/src/orchestra/bundler/setup_generator.py +++ b/src/orchestra/bundler/setup_generator.py @@ -74,11 +74,21 @@ def _generate_secrets_setup_notebook(secrets: list[SecretInstruction]) -> DabNot for s in secrets: scopes.setdefault(s.scope, []).append(s) - body_parts: list[str] = [] + # C-46 (LSC5-002): the ``dbutils.secrets`` submodule is read-only + # (get / getBytes / list / listScopes) — ``createScope`` and ``put`` do + # not exist and raise AttributeError on the first cell. Provision via + # the Databricks SDK ``WorkspaceClient`` instead. + init_cell = textwrap.dedent("""\ + from databricks.sdk import WorkspaceClient + + w = WorkspaceClient() + """).rstrip() + + body_parts: list[str] = [init_cell] for scope_name, scope_secrets in sorted(scopes.items()): lines: list[str] = [f"# Create scope: {scope_name}"] lines.append("try:") - lines.append(f' dbutils.secrets.createScope(scope="{scope_name}")') + lines.append(f' w.secrets.create_scope(scope="{scope_name}")') lines.append(f' print("Created scope: {scope_name}")') lines.append("except Exception as e:") lines.append(' if "RESOURCE_ALREADY_EXISTS" in str(e):') @@ -89,7 +99,10 @@ def _generate_secrets_setup_notebook(secrets: list[SecretInstruction]) -> DabNot for secret in scope_secrets: lines.append(f"# {secret.value_source}") - lines.append(f'dbutils.secrets.put(scope="{scope_name}", key="{secret.key}", string_value="PLACEHOLDER")') + lines.append( + f'w.secrets.put_secret(scope="{scope_name}", key="{secret.key}", ' + 'string_value="PLACEHOLDER")' + ) lines.append(f'print("Created secret: {scope_name}/{secret.key}")') lines.append("") diff --git a/src/orchestra/models/adf_ast.py b/src/orchestra/models/adf_ast.py index e25c90e..9e2d2f9 100644 --- a/src/orchestra/models/adf_ast.py +++ b/src/orchestra/models/adf_ast.py @@ -105,10 +105,14 @@ class AdfLinkedServiceReference: Attributes: reference_name: Logical name of the linked service. type: Reference type (always ``"LinkedServiceReference"``). + parameters: Runtime parameter overrides supplied by the activity, + keyed by parameter name. These flow into the resolver as + ``@linkedService().X`` substitutions. """ reference_name: str type: str = "LinkedServiceReference" + parameters: dict[str, Any] | None = None # --------------------------------------------------------------------------- @@ -241,12 +245,46 @@ class AdfDefinitions: datasets: Dataset definitions keyed by name. linked_services: Linked service definitions keyed by name. triggers: Trigger definitions. + global_parameters: Factory-level ``globalParameters`` keyed by name, + with each value parsed into ``{"type": str, "value": Any}``. """ pipelines: list[AdfPipeline] datasets: dict[str, AdfDataset] = field(default_factory=dict) linked_services: dict[str, AdfLinkedService] = field(default_factory=dict) triggers: list[AdfTrigger] = field(default_factory=list) + global_parameters: dict[str, Any] = field(default_factory=dict) + + def get_dataset(self, name: str | None) -> AdfDataset | None: + """Case-insensitive dataset lookup. + + LSC3-005: ADF identifiers are documented as case-insensitive; pipelines + sometimes reference a dataset by a different casing than the source + JSON file declares. Tolerate the mismatch instead of returning None. + """ + if not name: + return None + found = self.datasets.get(name) + if found is not None: + return found + lowered = name.lower() + for key, value in self.datasets.items(): + if key.lower() == lowered: + return value + return None + + def get_linked_service(self, name: str | None) -> AdfLinkedService | None: + """Case-insensitive linked service lookup; see :meth:`get_dataset`.""" + if not name: + return None + found = self.linked_services.get(name) + if found is not None: + return found + lowered = name.lower() + for key, value in self.linked_services.items(): + if key.lower() == lowered: + return value + return None # --------------------------------------------------------------------------- diff --git a/src/orchestra/models/ir.py b/src/orchestra/models/ir.py index 12a383f..79457ad 100644 --- a/src/orchestra/models/ir.py +++ b/src/orchestra/models/ir.py @@ -12,13 +12,34 @@ @dataclass(slots=True, kw_only=True) class ExpressionResult: - """Result of resolving an ADF expression.""" + """Result of resolving an ADF expression. + + Attributes: + kind: One of ``"literal"`` / ``"dab_ref"`` / ``"notebook_code"``. + value: The resolved value text. + imports: Imports the notebook_code value needs. + required_parameters: Widget name -> DAB ref mapping for + base_parameters threading. + notes: Free-form caveats surfaced in SETUP.md. + was_string_literal: C-34 (VAREX4-002): True when the original + ADF token was a quoted string (``'09'``, ``"12"``) so the + function-call codegen path can ``repr()`` it instead of + emitting a bare numeric token that strips quotedness. + was_bool_literal: C-34 (VAREX4-003): True when the original ADF + token was ``true`` / ``false``. ADF Booleans serialise as + the lowercase strings ``'true'``/``'false'`` on the + SetVariable consumer side (post-C-21), so comparisons must + emit ``'true'`` / ``'false'`` Python strings rather than the + bare Python ``True`` / ``False``. + """ kind: str value: str imports: list[str] = field(default_factory=list) required_parameters: dict[str, str] = field(default_factory=dict) notes: list[str] = field(default_factory=list) + was_string_literal: bool = False + was_bool_literal: bool = False @dataclass(slots=True, kw_only=True) @@ -85,11 +106,27 @@ class NotebookActivity(Activity): notebook_path: Workspace path to the notebook. base_parameters: Parameters passed to the notebook at runtime. linked_service_definition: Raw linked-service dictionary for cluster config. + notebook_path_unresolved: C-28 (NB-ITER4-001): True when the ADF + ``notebookPath`` is a dynamic expression the translator couldn't + reduce to a literal/dab_ref workspace path. The preparer emits + a dispatch-stub notebook that reads ``notebook_path`` from a + widget and ``dbutils.notebook.run()``s the resolved value. + notebook_path_expression: Raw ADF expression text captured when + ``notebook_path_unresolved`` is True, surfaced in SETUP.md. + unresolved_libraries: C-30 (NB-ITER4-003): library descriptor + entries whose value (jar/whl/egg/requirements path) carried an + ADF expression the resolver couldn't reduce to a literal or + dab_ref. Each entry has ``type`` (library shape key), + ``expression`` (raw ADF text), and ``missing`` (referenced + identifier names not bound in the translation context). """ notebook_path: str base_parameters: dict[str, str] | None = None linked_service_definition: dict[str, Any] | None = None + notebook_path_unresolved: bool = False + notebook_path_expression: str | None = None + unresolved_libraries: list[dict[str, Any]] = field(default_factory=list) @dataclass(slots=True, kw_only=True) @@ -142,11 +179,23 @@ class ForEachActivity(Activity): inner_activities: Translated activities executed for each item. concurrency: Maximum parallel iterations (maps to Databricks ``for_each_task.concurrency``). + inputs_bridge_notebook_code: C-31 (CF4-001): when the items + expression resolves to ``notebook_code`` (e.g. + ``@split(variables('fecha'),',')``), the translator captures + the Python code here while the full TranslationContext is + available. The preparer reads it instead of re-resolving + against an empty context (which silently failed before). + inputs_bridge_notebook_imports: Imports the bridge code needs. + inputs_bridge_required_parameters: Widget name → DAB ref mapping + for the bridge notebook's base_parameters. """ items_expression: str inner_activities: list[Activity] = field(default_factory=list) concurrency: int | None = None + inputs_bridge_notebook_code: str | None = None + inputs_bridge_notebook_imports: list[str] = field(default_factory=list) + inputs_bridge_required_parameters: dict[str, str] = field(default_factory=dict) @dataclass(slots=True, kw_only=True) @@ -159,6 +208,16 @@ class IfConditionActivity(Activity): right: Right-hand operand expression. if_true_activities: Activities for the true branch. if_false_activities: Activities for the false branch. + bridge_notebook_code: C-07 (CF-iter2-001 / VAREX-003): when the + ADF condition expression contained a function call that + couldn't be lowered to a literal/dab_ref operand, + ``bridge_notebook_code`` carries the Python code that + evaluates it. The preparer synthesises a hidden SetVariable + task that runs this code and points ``left`` at the + resulting task value. + bridge_notebook_imports: Imports the bridge notebook code needs. + bridge_required_parameters: Widget name -> DAB ref mapping for + the bridge notebook's base_parameters. """ op: str @@ -166,6 +225,9 @@ class IfConditionActivity(Activity): right: str if_true_activities: list[Activity] = field(default_factory=list) if_false_activities: list[Activity] = field(default_factory=list) + bridge_notebook_code: str | None = None + bridge_notebook_imports: list[str] = field(default_factory=list) + bridge_required_parameters: dict[str, str] = field(default_factory=dict) @dataclass(slots=True, kw_only=True) @@ -175,16 +237,23 @@ class SetVariableActivity(Activity): Attributes: variable_name: Name of the variable being set. variable_value: Expression string that evaluates to the value. - value_kind: Kind of the resolved expression ("literal", "dab_ref", "notebook_code"). + value_kind: Kind of the resolved expression ("literal", "dab_ref", + "notebook_code", "unresolved"). notebook_code: Python code for notebook_code kind values. notebook_imports: Import statements needed for notebook_code. + raw_expression: C-33 (VAREX4-001 / CF4-003): when ``value_kind`` is + ``"unresolved"`` (the resolver returned None for an ADF + ``@``-prefixed value), this carries the original ADF + expression text so SETUP.md can surface the manual + initialisation step. """ variable_name: str variable_value: str - value_kind: str = "literal" # "literal", "dab_ref", "notebook_code" + value_kind: str = "literal" # "literal", "dab_ref", "notebook_code", "unresolved" notebook_code: str | None = None notebook_imports: list[str] = field(default_factory=list) + raw_expression: str | None = None @dataclass(slots=True, kw_only=True) @@ -319,11 +388,21 @@ class SwitchActivity(Activity): on_expression: The ADF expression to evaluate. cases: Ordered list of case branches. default_activities: Activities to run when no case matches. + bridge_notebook_code: C-07 (CF-iter2-001 / CF-iter2-003): when + ``on_expression`` cannot be lowered to a literal/dab_ref, this + field carries the Python code the preparer runs in a bridge + task so the resolved value drives ``condition_task.left``. + bridge_notebook_imports: Imports for the bridge notebook code. + bridge_required_parameters: Widget name -> DAB ref mapping for + the bridge notebook's base_parameters. """ on_expression: str cases: list[SwitchCase] = field(default_factory=list) default_activities: list[Activity] = field(default_factory=list) + bridge_notebook_code: str | None = None + bridge_notebook_imports: list[str] = field(default_factory=list) + bridge_required_parameters: dict[str, str] = field(default_factory=dict) @dataclass(slots=True, kw_only=True) @@ -488,6 +567,10 @@ class TranslationContext: registry: MappingProxyType[str, Any] = field(default_factory=lambda: MappingProxyType({})) variable_cache: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) variable_value_cache: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) + variable_types: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) + variable_default_literals: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) + global_parameters: MappingProxyType[str, Any] = field(default_factory=lambda: MappingProxyType({})) + linked_service_parameters: MappingProxyType[str, Any] = field(default_factory=lambda: MappingProxyType({})) def with_activity(self, name: str, activity: Activity) -> TranslationContext: """Return a new context with *activity* added to the cache. @@ -504,6 +587,10 @@ def with_activity(self, name: str, activity: Activity) -> TranslationContext: registry=self.registry, variable_cache=self.variable_cache, variable_value_cache=self.variable_value_cache, + variable_types=self.variable_types, + variable_default_literals=self.variable_default_literals, + global_parameters=self.global_parameters, + linked_service_parameters=self.linked_service_parameters, ) def get_activity(self, activity_name: str) -> Activity | None: @@ -545,16 +632,104 @@ def with_variable( registry=self.registry, variable_cache=MappingProxyType({**self.variable_cache, variable_name: task_key}), variable_value_cache=new_variable_value_cache, + variable_types=self.variable_types, + variable_default_literals=self.variable_default_literals, + global_parameters=self.global_parameters, + linked_service_parameters=self.linked_service_parameters, + ) + + def with_variable_types( + self, + types: dict[str, str], + *, + default_literals: dict[str, str] | None = None, + ) -> TranslationContext: + """Return a new context seeded with declared variable types. + + Args: + types: Mapping of variable name -> ADF declared type + (``"String"``, ``"Boolean"``, ``"Array"``, ...). Used by + the IfCondition fallback to recognise Boolean variables + whose value is seeded only by a literal init task (and + therefore absent from ``variable_value_cache``). + default_literals: Optional mapping of variable name -> seeded + literal default (e.g. ``"true"``/``"false"``). The + IfCondition bridge (C-43) recomputes a Boolean operand + locally from this literal so an inner-ForEach condition does + not dangle to a parent-job task value. + + Returns: + New context carrying the variable type / default-literal maps. + """ + return TranslationContext( + activity_cache=self.activity_cache, + registry=self.registry, + variable_cache=self.variable_cache, + variable_value_cache=self.variable_value_cache, + variable_types=MappingProxyType({**self.variable_types, **types}), + variable_default_literals=MappingProxyType( + {**self.variable_default_literals, **(default_literals or {})} + ), + global_parameters=self.global_parameters, + linked_service_parameters=self.linked_service_parameters, ) def get_variable_task_key(self, variable_name: str) -> str | None: """Look up the task key that sets a variable.""" return self.variable_cache.get(variable_name) + def get_variable_type(self, variable_name: str) -> str | None: + """Look up a variable's declared ADF type, if known.""" + return self.variable_types.get(variable_name) + + def get_variable_default_literal(self, variable_name: str) -> str | None: + """Look up a variable's seeded literal default value, if known.""" + return self.variable_default_literals.get(variable_name) + def get_variable_dab_ref(self, variable_name: str) -> str | None: """Look up the inlined DAB ref value for a variable, if available.""" return self.variable_value_cache.get(variable_name) + def with_linked_service_parameters(self, params: dict[str, Any]) -> TranslationContext: + """Return a new context with linked-service-scoped parameters applied. + + Args: + params: Mapping of LS parameter name -> resolved value. Used + by ``@linkedService().X`` references in LS typeProperties. + + Returns: + New context with the parameters bound for the current activity. + """ + return TranslationContext( + activity_cache=self.activity_cache, + registry=self.registry, + variable_cache=self.variable_cache, + variable_value_cache=self.variable_value_cache, + variable_types=self.variable_types, + variable_default_literals=self.variable_default_literals, + global_parameters=self.global_parameters, + linked_service_parameters=MappingProxyType(dict(params)), + ) + + def get_global_parameter(self, name: str) -> Any: + """Look up a factory-level global parameter value. + + Args: + name: Global parameter name (e.g. ``"env_variable"``). + + Returns: + The parameter value if present, else ``None``. Values may be + scalar or dict-typed (e.g. ``{"type": "string", "value": "t"}``). + """ + raw = self.global_parameters.get(name) + if isinstance(raw, dict) and "value" in raw: + return raw["value"] + return raw + + def get_linked_service_parameter(self, name: str) -> Any: + """Look up an activity-supplied linked-service parameter value.""" + return self.linked_service_parameters.get(name) + TranslationResult: TypeAlias = Activity | UnsupportedActivity diff --git a/src/orchestra/parser/adf_loader.py b/src/orchestra/parser/adf_loader.py index 03e6374..acc3ed9 100644 --- a/src/orchestra/parser/adf_loader.py +++ b/src/orchestra/parser/adf_loader.py @@ -93,6 +93,17 @@ def load_adf_definitions(source_dir: Path) -> AdfDefinitions: datasets: dict[str, AdfDataset] = {} linked_services: dict[str, AdfLinkedService] = {} triggers: list[AdfTrigger] = [] + global_parameters: dict[str, Any] = {} + + factory_dir = _find_json_dir(source_dir, "factory", "factories") + if factory_dir is not None: + for json_file in sorted(factory_dir.glob("*.json")): + try: + data = json.loads(json_file.read_text(encoding="utf-8")) + factory_params = _parse_factory_global_parameters(data) + global_parameters.update(factory_params) + except Exception: + logger.exception("Failed to parse factory file %s", json_file) pipeline_dir = _find_json_dir(source_dir, "pipelines", "pipeline") if pipeline_dir is not None: @@ -137,9 +148,36 @@ def load_adf_definitions(source_dir: Path) -> AdfDefinitions: datasets=datasets, linked_services=linked_services, triggers=triggers, + global_parameters=global_parameters, ) +def _parse_factory_global_parameters(data: dict[str, Any]) -> dict[str, Any]: + """Extracts ``globalParameters`` from a factory JSON payload. + + Args: + data: Raw JSON dictionary loaded from ``factory/.json``. + + Returns: + Mapping of global parameter name -> value. Each value is either + the scalar default (when ADF stores it bare) or the original + ``{"type": ..., "value": ...}`` dict. + """ + data = _normalize_arm(data) + props = data.get("properties", data) + raw = props.get("globalParameters") or {} + if not isinstance(raw, dict): + return {} + + result: dict[str, Any] = {} + for name, value in raw.items(): + if isinstance(value, dict) and "value" in value: + result[name] = value + else: + result[name] = value + return result + + def classify_activity(activity_type: str) -> tuple[TranslationStrategy, str | None]: """Classify an ADF activity type into a translation strategy. @@ -324,6 +362,7 @@ def parse_activity(data: dict[str, Any]) -> AdfActivity: linked_service_name = AdfLinkedServiceReference( reference_name=raw_ls.get("referenceName", ""), type=raw_ls.get("type", "LinkedServiceReference"), + parameters=raw_ls.get("parameters"), ) if_true_activities: list[AdfActivity] | None = None @@ -527,6 +566,7 @@ def _load_arm_template(template_path: Path) -> AdfDefinitions: datasets: dict[str, AdfDataset] = {} linked_services: dict[str, AdfLinkedService] = {} triggers: list[AdfTrigger] = [] + global_parameters: dict[str, Any] = {} for resource in resources: rtype = resource.get("type", "") @@ -561,12 +601,18 @@ def _load_arm_template(template_path: Path) -> AdfDefinitions: triggers.append(_parse_trigger_json(wrapped, fallback_name=name)) except Exception: logger.exception("Failed to parse ARM trigger resource %s", name) + elif rtype.endswith("/factories"): + try: + global_parameters.update(_parse_factory_global_parameters(wrapped)) + except Exception: + logger.exception("Failed to parse ARM factory resource %s", name) return AdfDefinitions( pipelines=pipelines, datasets=datasets, linked_services=linked_services, triggers=triggers, + global_parameters=global_parameters, ) diff --git a/src/orchestra/parser/expression_parser.py b/src/orchestra/parser/expression_parser.py index a325cd0..af09506 100644 --- a/src/orchestra/parser/expression_parser.py +++ b/src/orchestra/parser/expression_parser.py @@ -10,7 +10,10 @@ _ITEM_RE = re.compile(r"item\(\s*\)$", re.IGNORECASE) -_ITEM_FIELD_RE = re.compile(r"item\(\s*\)\.(\w+)", re.IGNORECASE) +# C-35 (CF4-004): anchor the end-of-string so multi-segment chains like +# ``item().condition.name`` don't match here and silently drop the trailing +# ``.name`` (the previous behaviour mapped to ``{{input.condition}}``). +_ITEM_FIELD_RE = re.compile(r"item\(\s*\)\.(\w+)\s*$", re.IGNORECASE) _ACTIVITY_OUTPUT_RE = re.compile( r"""activity\(\s*'([^']+)'\s*\)\.output(?:\.(.+))?""", @@ -22,11 +25,26 @@ re.IGNORECASE, ) +_PIPELINE_GLOBAL_PARAM_RE = re.compile( + r"""pipeline\(\s*\)\.globalParameters\.(\w+)""", + re.IGNORECASE, +) + _PIPELINE_PROPERTY_RE = re.compile( r"""pipeline\(\s*\)\.(\w+)""", re.IGNORECASE, ) +_LINKED_SERVICE_PARAM_RE = re.compile( + r"""linkedService\(\s*\)\.(\w+)""", + re.IGNORECASE, +) + +_ITEM_SAFE_NAV_RE = re.compile( + r"item\(\s*\)((?:\??\.\w+)+)$", + re.IGNORECASE, +) + _VARIABLE_RE = re.compile( r"""variables\(\s*'([^']+)'\s*\)""", re.IGNORECASE, @@ -67,10 +85,17 @@ _INTERPOLATION_RE = re.compile(r"@\{(.+?)\}") _FUNCTION_CALL_RE = re.compile( - r"([a-zA-Z_]\w*)\((.*)?\)$", + r"([a-zA-Z_]\w*)\((.*)?\)\s*$", re.IGNORECASE | re.DOTALL, ) +# Function names that are no-op wrappers when they appear at the outermost +# position around a single deterministic parameter / variable reference. +# Stripping these lets resolve_expression reach the underlying ref instead +# of falling through to notebook_code for trivial @json(pipeline().parameters.X) +# style wrappers commonly used in ADF for type coercion. +_NOOP_WRAPPER_NAMES: frozenset[str] = frozenset({"json", "string", "array"}) + _DATETIME_IMPORTS = ["from datetime import datetime, timezone, timedelta"] _TIME_UNIT_MAP: dict[str, str] = { @@ -109,7 +134,10 @@ def resolve_expression( return None if isinstance(value, bool): - return ExpressionResult(kind="literal", value=str(value)) + # VAREX3-002: render Python bool as lowercase 'true'/'false' so + # downstream ADF comparisons like @equals(variables('X'), true) + # match ADF's lowercase boolean tokens. + return ExpressionResult(kind="literal", value="true" if value else "false") if isinstance(value, (int, float)): return ExpressionResult(kind="literal", value=str(value)) @@ -119,7 +147,15 @@ def resolve_expression( if not value.startswith("@"): return ExpressionResult(kind="literal", value=value) - expr = value[1:] # strip leading @ + expr = value[1:].rstrip() # strip leading @ and trailing whitespace/newlines + + # Strip no-op wrappers like @json(pipeline().parameters.X) so the inner + # ref resolves to its DAB dynamic value. We only unwrap when the inner + # expression itself resolves cleanly (literal / dab_ref) so we don't + # eat the wrapper's semantics where it actually matters. + unwrapped = _unwrap_noop_call(expr, context, variable_task_keys=variable_task_keys) + if unwrapped is not None: + return unwrapped if _ITEM_RE.match(expr): return ExpressionResult(kind="dab_ref", value="{{input}}") @@ -129,6 +165,18 @@ def resolve_expression( field_name = match.group(1) return ExpressionResult(kind="dab_ref", value="{{input." + field_name + "}}") + result = _resolve_item_safe_nav(expr) + if result is not None: + return result + + result = _resolve_pipeline_global_param(expr, context) + if result is not None: + return result + + result = _resolve_linked_service_param(expr, context) + if result is not None: + return result + result = _resolve_pipeline_param(expr) if result is not None: return result @@ -157,6 +205,25 @@ def resolve_expression( if result is not None: return result + # CF3-004 / fix-attribute-access-on-function-results: handle + # ``....`` chains like + # ``json(pipeline().parameters.items).type`` by resolving the function + # call first then chaining `.get('attr')` onto the resulting code. + result = _resolve_function_call_with_attribute( + expr, context, variable_task_keys=variable_task_keys + ) + if result is not None: + return result + + # C-33 (VAREX4-001): handle ``[N]`` chains so e.g. + # ``@split(pipeline().parameters.referenceDate,'/')[0]`` lowers to + # notebook_code. + result = _resolve_function_call_with_index( + expr, context, variable_task_keys=variable_task_keys + ) + if result is not None: + return result + return None @@ -295,6 +362,114 @@ def _resolve_pipeline_param(expr: str) -> ExpressionResult | None: return ExpressionResult(kind="dab_ref", value="{{" + f"job.parameters.{param_name}" + "}}") +def _resolve_pipeline_global_param(expr: str, context: TranslationContext) -> ExpressionResult | None: + """Resolves ``pipeline().globalParameters.X`` against factory globals. + + When ``context.global_parameters`` carries a concrete value for *X* + the expression collapses to a literal so downstream callers (notably + ``concat`` reductions) get the actual factory value baked in. When + no factory value is available we fall back to a job-parameter DAB + ref so the bundle YAML can supply it. + """ + match = _PIPELINE_GLOBAL_PARAM_RE.match(expr) + if match is None: + return None + param_name = match.group(1) + value = context.get_global_parameter(param_name) + if value is None: + return ExpressionResult(kind="dab_ref", value="{{" + f"job.parameters.{param_name}" + "}}") + return ExpressionResult(kind="literal", value=str(value)) + + +def _resolve_linked_service_param(expr: str, context: TranslationContext) -> ExpressionResult | None: + """Resolves ``linkedService().X`` against activity-supplied LS parameters.""" + match = _LINKED_SERVICE_PARAM_RE.match(expr) + if match is None: + return None + param_name = match.group(1) + if param_name in context.linked_service_parameters: + value = context.get_linked_service_parameter(param_name) + if value is None: + return None + return ExpressionResult(kind="literal", value=str(value)) + return None + + +def _resolve_item_safe_nav(expr: str) -> ExpressionResult | None: + """Resolves ``item()?.X``, ``item().a?.b?.c`` and ``item().a.b`` chains. + + Any chain that uses the ADF safe-navigation operator ``?.`` — even a + single segment ``item()?.X`` — must lower to notebook_code so the bridge + lowering can fire (the trivial ``item().X`` case stays a DAB ref via + ``_ITEM_FIELD_RE``). The returned ``notebook_code`` walks the chain + using ``.get()`` so missing keys do not raise. + + C-16 (CF3-005 / VAREX3-005): the previous ``len(parts) < 2`` guard + blocked Switch on-expressions and SetVariable expressions wrapping + ``item()?.X`` from triggering bridge lowering. + """ + match = _ITEM_SAFE_NAV_RE.match(expr) + if match is None: + return None + chain = match.group(1) + # Collect (operator, field) tuples so single-segment item()?.X still + # lowers to notebook_code (was: skipped when len(parts) < 2). + segments: list[tuple[str, str]] = re.findall(r"(\??\.)(\w+)", chain) + if not segments: + return None + # If the chain has no safe-nav operator at all (purely ``item().a.b``) + # AND only one segment, defer to _ITEM_FIELD_RE's dab_ref path. + # C-35 (CF4-004): multi-segment pure-dotted chains like + # ``item().condition.name`` must lower to notebook_code so the + # downstream consumers can walk both segments instead of mapping to + # ``{{input.condition}}`` and silently dropping ``.name``. + has_safe_nav = any(op == "?." for op, _ in segments) + if not has_safe_nav and len(segments) < 2: + return None + expr_code = "__import__('json').loads(dbutils.widgets.get('item'))" + for _, part in segments: + expr_code = f"({expr_code} or {{}}).get('{part}')" + return ExpressionResult(kind="notebook_code", value=expr_code) + + +def _unwrap_noop_call( + expr: str, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> ExpressionResult | None: + """If *expr* is a no-op wrapper around a deterministic ref, return inner. + + Handles patterns like ``@json(pipeline().parameters.items)``, + ``@string(pipeline().parameters.X)``, and ``@array(...)`` where the + sole argument resolves to a clean literal/dab_ref. When the inner + cannot be deterministically resolved we return ``None`` so the + regular function dispatcher takes over. + """ + match = _FUNCTION_CALL_RE.match(expr) + if match is None: + return None + func_name = match.group(1) + if func_name.lower() not in _NOOP_WRAPPER_NAMES: + return None + inner = (match.group(2) or "").strip() + if not inner: + return None + args = _split_args(inner) + if len(args) != 1: + return None + sole_arg = args[0].strip() + if sole_arg.startswith("'") and sole_arg.endswith("'"): + return None # bare literal, let upstream string() handler decide + sub_expr = sole_arg if sole_arg.startswith("@") else "@" + sole_arg + inner_result = resolve_expression(sub_expr, context, variable_task_keys=variable_task_keys) + if inner_result is None: + return None + if inner_result.kind in ("literal", "dab_ref"): + return inner_result + return None + + def _resolve_pipeline_property(expr: str) -> ExpressionResult | None: """Resolves ``pipeline().PropertyName`` -> DAB ref.""" match = _PIPELINE_PROPERTY_RE.match(expr) @@ -336,19 +511,27 @@ def _resolve_variable( *, variable_task_keys: dict[str, str] | None = None, ) -> ExpressionResult | None: - """Resolves ``variables('name')`` -> task value DAB ref.""" + """Resolves ``variables('name')`` -> task value DAB ref. + + C-05 (VAREX-002): when neither the explicit mapping nor the context's + variable_cache knows a setter for *var_name*, return ``None`` instead + of falling back to ``{{tasks..values.}}`` — that + self-referential placeholder is never satisfied at runtime and + pollutes the bundle with hundreds of dangling refs. Init tasks + synthesised in :func:`engine._build_variable_init_activities` seed + the cache for default-valued variables, so this path now triggers + only for genuinely unset variables (caller logs / surfaces a setup + note). + """ match = _VARIABLE_RE.match(expr) if match is None: return None var_name = match.group(1) - # Always resolve to the task value reference. This preserves the - # explicit task dependency chain — downstream tasks must depend on the - # setter task. Even when the variable was set to a DAB built-in like - # {{job.start_time.iso_datetime}}, the task value is the canonical - # source since the setter notebook may transform the value. variable_task_keys_map = variable_task_keys or {} - setter_key = variable_task_keys_map.get(var_name) or context.get_variable_task_key(var_name) or var_name + setter_key = variable_task_keys_map.get(var_name) or context.get_variable_task_key(var_name) + if setter_key is None: + return None return ExpressionResult(kind="dab_ref", value="{{" + f"tasks.{setter_key}.values.{var_name}" + "}}") @@ -436,6 +619,8 @@ def _resolve_concat( all_imports: list[str] = [] code_parts: list[str] = [] + literal_parts: list[str] = [] + all_literal = True all_required_parameters: dict[str, str] = {} for part in parts: @@ -443,25 +628,36 @@ def _resolve_concat( if not part: continue if part.startswith("'") and part.endswith("'"): - code_parts.append(repr(part[1:-1])) + value_text = part[1:-1] + code_parts.append(repr(value_text)) + literal_parts.append(value_text) else: sub_result = resolve_expression("@" + part, context, variable_task_keys=variable_task_keys) if sub_result is None: return None if sub_result.kind == "literal": code_parts.append(repr(sub_result.value)) + literal_parts.append(sub_result.value) elif sub_result.kind == "dab_ref": code_parts.append(_dab_ref_to_widget_code(sub_result.value)) widget_name, dab_ref = _required_parameter_for_ref(sub_result.value) all_required_parameters.setdefault(widget_name, dab_ref) + all_literal = False elif sub_result.kind == "notebook_code": code_parts.append(f"str({sub_result.value})") all_imports.extend(sub_result.imports) all_required_parameters.update(sub_result.required_parameters) + all_literal = False if not code_parts: return None + # If every part collapsed to a literal value, fold the whole concat into + # a single literal so downstream consumers (notebook library install, + # cluster fields, etc.) get a plain string instead of Python source. + if all_literal: + return ExpressionResult(kind="literal", value="".join(literal_parts)) + value = " + ".join(code_parts) return ExpressionResult( kind="notebook_code", @@ -534,6 +730,115 @@ def _split_args(inner: str) -> list[str]: return parts +_FUNCTION_CALL_WITH_ATTRIBUTE_RE = re.compile( + # Captures `funcName(args).attr.attr...` -- the trailing attribute chain + # must end with a word character so we don't accidentally swallow other + # closing parens / spaces. Used to lower + # ``json(pipeline().parameters.items).type`` to a notebook_code expression + # since the bare function dispatcher requires the function call to be the + # outermost token. + r"^([a-zA-Z_]\w*)\((.*)\)((?:\.\w+)+)\s*$", + re.IGNORECASE | re.DOTALL, +) + +_FUNCTION_CALL_WITH_INDEX_RE = re.compile( + # C-33 (VAREX4-001): ``funcName(args)[N]`` — captures a trailing + # integer subscript so ``split(...)[0]`` and similar ADF expressions + # lower to notebook_code (the bare dispatcher only matched when the + # function call was the outermost token). We support a single + # numeric subscript for now; nested chains (``...[0][1]``) fall + # through to the legacy unsupported path. + r"^([a-zA-Z_]\w*)\((.*)\)\[\s*(-?\d+)\s*\]\s*$", + re.IGNORECASE | re.DOTALL, +) + + +def _resolve_function_call_with_index( + expr: str, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> ExpressionResult | None: + """Lowers ``[N]`` to notebook_code. + + C-33 (VAREX4-001): ADF SetVariable expressions like + ``@split(pipeline().parameters.referenceDate,'/')[0]`` were + previously rejected because the bare function dispatcher matched + only when the call was the outermost token. Resolve the function + call as usual, then append ``[N]`` to the resulting Python code. + """ + match = _FUNCTION_CALL_WITH_INDEX_RE.match(expr) + if match is None: + return None + func_name = match.group(1) + inner = match.group(2) + index = match.group(3) + func_expr = f"@{func_name}({inner})" + base_result = resolve_expression(func_expr, context, variable_task_keys=variable_task_keys) + if base_result is None: + return None + if base_result.kind == "literal": + base_code = repr(base_result.value) + elif base_result.kind == "dab_ref": + base_code = _dab_ref_to_widget_code(base_result.value) + else: + base_code = base_result.value + code = f"({base_code})[{index}]" + return ExpressionResult( + kind="notebook_code", + value=code, + imports=list(base_result.imports), + required_parameters=dict(base_result.required_parameters), + ) + + +def _resolve_function_call_with_attribute( + expr: str, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> ExpressionResult | None: + """Lowers `....` chains to a notebook_code expression. + + CF3-004 / fix-attribute-access-on-function-results: the bare function + dispatcher only matches when the function call is the outermost token, + so an expression like ``json(pipeline().parameters.items).type`` falls + through with ``None`` and ships unmodified into ``condition_task.left``. + We resolve the function call, render it as Python code, then chain + ``.get('attr')`` for each segment of the trailing attribute path so the + bridge lowering picks it up. + """ + match = _FUNCTION_CALL_WITH_ATTRIBUTE_RE.match(expr) + if match is None: + return None + func_name = match.group(1) + inner = match.group(2) + attr_chain = match.group(3) + func_expr = f"@{func_name}({inner})" + base_result = resolve_expression( + func_expr, context, variable_task_keys=variable_task_keys + ) + if base_result is None: + return None + if base_result.kind == "literal": + # The result is a known literal -- render it as a Python expression + # then chain `.get(...)` so the resulting notebook_code is valid. + base_code = repr(base_result.value) + elif base_result.kind == "dab_ref": + base_code = _dab_ref_to_widget_code(base_result.value) + else: + base_code = base_result.value + code = base_code + for segment in attr_chain.strip(".").split("."): + code = f"({code}).get('{segment}')" + return ExpressionResult( + kind="notebook_code", + value=code, + imports=list(base_result.imports), + required_parameters=dict(base_result.required_parameters), + ) + + def _resolve_function_call( expr: str, context: TranslationContext, @@ -566,12 +871,28 @@ def _resolve_function_call( continue if (raw_arg.startswith("'") and raw_arg.endswith("'")) or (raw_arg.startswith('"') and raw_arg.endswith('"')): - resolved_args.append(ExpressionResult(kind="literal", value=raw_arg[1:-1])) + # C-34 (VAREX4-002): preserve the quotedness so the codegen + # downstream emits ``repr(value)`` rather than a bare token — + # otherwise quoted ``'09'`` / ``'12'`` collapse to a bare + # numeric and either raise a SyntaxError (leading zero) or + # silently compare against the wrong value. + resolved_args.append( + ExpressionResult(kind="literal", value=raw_arg[1:-1], was_string_literal=True) + ) elif _is_numeric(raw_arg): resolved_args.append(ExpressionResult(kind="literal", value=raw_arg)) elif raw_arg.lower() in ("true", "false"): + # C-34 (VAREX4-003): ADF Booleans (``true`` / ``false``) match + # lowercase strings on the SetVariable consumer side (C-21). + # Mark the literal so ``_arg_to_code`` emits ``'true'`` / + # ``'false'`` strings rather than the bare Python ``True`` / + # ``False`` (whose ``str()`` is title-case and never matches). resolved_args.append( - ExpressionResult(kind="literal", value="True" if raw_arg.lower() == "true" else "False") + ExpressionResult( + kind="literal", + value="true" if raw_arg.lower() == "true" else "false", + was_bool_literal=True, + ) ) elif raw_arg.lower() == "null": resolved_args.append(ExpressionResult(kind="literal", value="None")) @@ -610,8 +931,18 @@ def _is_numeric(text: str) -> bool: def _arg_to_code(arg: ExpressionResult) -> str: - """Converts a resolved argument to a Python code snippet.""" + """Converts a resolved argument to a Python code snippet. + + C-34 (VAREX4-002/003): quoted-string and Boolean-literal arguments + must emit ``repr()`` of the value (e.g. ``'09'`` rather than the + bare token ``09``) so the resulting code (a) parses (leading-zero + integers are SyntaxErrors in modern Python) and (b) compares against + the right concrete value (Booleans on the SetVariable consumer side + serialise as lowercase strings, not Python bools). + """ if arg.kind == "literal": + if arg.was_string_literal or arg.was_bool_literal: + return repr(arg.value) if arg.value in ("True", "False", "None") or _is_numeric(arg.value): return arg.value return repr(arg.value) @@ -690,9 +1021,17 @@ def _result_from_args( def _handle_concat(args: list[ExpressionResult]) -> ExpressionResult | None: - """concat(a, b, ...) -> str(a) + str(b) + ...""" + """concat(a, b, ...) -> str(a) + str(b) + ... + + When every argument resolved to a ``literal`` kind, collapse the whole + expression to a single literal so downstream consumers (cluster fields, + library paths, notebook-install jar refs, etc.) get a plain string + instead of Python source. + """ if not args: return None + if all(a.kind == "literal" for a in args): + return ExpressionResult(kind="literal", value="".join(a.value for a in args)) parts = [f"str({_arg_to_code(a)})" for a in args] return _result_from_args(" + ".join(parts), args) @@ -762,7 +1101,17 @@ def _handle_starts_with(args: list[ExpressionResult]) -> ExpressionResult | None def _handle_substring(args: list[ExpressionResult]) -> ExpressionResult | None: - """substring(text, start, length) -> str(text)[int(start):int(start)+int(length)]""" + """substring(text, start[, length]) -> Python slice. + + C-33 (VAREX4-001): ADF accepts the 2-arg form ``substring(x, start)`` + in addition to the documented 3-arg form. Treat the 2-arg case as + ``str(text)[int(start):]`` so SetVariable activities that wrap it can + actually resolve. + """ + if len(args) == 2: + text = _arg_to_code(args[0]) + start = _arg_to_code(args[1]) + return _result_from_args(f"str({text})[int({start}):]", args) if len(args) != 3: return None text = _arg_to_code(args[0]) diff --git a/src/orchestra/preparer/activity_preparers/for_each.py b/src/orchestra/preparer/activity_preparers/for_each.py index a59c117..95acb6d 100644 --- a/src/orchestra/preparer/activity_preparers/for_each.py +++ b/src/orchestra/preparer/activity_preparers/for_each.py @@ -19,6 +19,7 @@ from flowx.preparer.workflow_preparer import ( PreparedActivity, PreparedWorkflow, + _iter_activity_with_descendants, build_common_task_fields, prepare_activity, ) @@ -28,31 +29,116 @@ from flowx.models.ir import ForEachActivity -def _resolve_for_each_inputs(items_expression: str) -> str: - """Converts an ADF items expression to a DAB dynamic value reference. - - Args: - items_expression: The raw ADF expression for ForEach items. - - Returns: - A DAB dynamic value reference string, or the original expression if - it cannot be resolved. +def _resolve_for_each_inputs_with_bridge( + activity: ForEachActivity, +) -> tuple[str, dict[str, Any] | None, list[DabNotebook]]: + """Resolves the ForEach items expression and emits a bridge task when needed. + + C-08 (CF-iter2-002): per Databricks docs, ``for_each_task.inputs`` + accepts a literal JSON array, ``{{tasks.X.values.Y}}``, or + ``{{job.parameters.X}}``. Function calls like ``@split(, ',')`` + are rejected. When the expression resolves to ``notebook_code`` we + synthesise a hidden seed task that computes the array and publishes + it as a task value the ForEach inputs reference. + + C-31 (CF4-001): the translator now stashes the resolved bridge code + on the IR (``inputs_bridge_notebook_code`` and friends) while the + full TranslationContext is available. The preparer reads those + fields rather than re-resolving against an empty TranslationContext + — the latter silently failed for any expression that needed + variable_cache lookups (e.g. ``@split(variables('fecha'),',')``). """ + items_expression = activity.items_expression + task_key = activity.task_key + + # IR-supplied bridge wins (C-31). Falls through to the legacy + # re-resolution path only when no bridge code was captured. + if activity.inputs_bridge_notebook_code: + bridge_key = f"{task_key}_inputs_bridge" + value_key = "items" + notebook_relative_path = f"notebooks/{bridge_key}.py" + base_parameters: dict[str, str] = dict(activity.inputs_bridge_required_parameters) + notebook_source = _render_for_each_inputs_bridge( + activity.inputs_bridge_notebook_code, + list(activity.inputs_bridge_notebook_imports), + list(base_parameters.keys()), + value_key, + ) + bridge_task: dict[str, Any] = { + "task_key": bridge_key, + "notebook_task": { + "notebook_path": f"../src/{notebook_relative_path}", + "base_parameters": base_parameters, + }, + } + bridge_value_ref = f"{{{{tasks.{bridge_key}.values.{value_key}}}}}" + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=notebook_source)] + return bridge_value_ref, bridge_task, notebooks + if items_expression.startswith("{{"): - return items_expression + return items_expression, None, [] context = TranslationContext() result = resolve_expression(items_expression, context) + if result is None and not items_expression.startswith("@"): + result = resolve_expression("@" + items_expression, context) + if result is not None and result.kind in ("dab_ref", "literal"): - return result.value + return result.value, None, [] + + if result is not None and result.kind == "notebook_code": + bridge_key = f"{task_key}_inputs_bridge" + value_key = "items" + notebook_relative_path = f"notebooks/{bridge_key}.py" + base_parameters = dict(result.required_parameters) + notebook_source = _render_for_each_inputs_bridge( + result.value, + result.imports, + list(base_parameters.keys()), + value_key, + ) + bridge_task = { + "task_key": bridge_key, + "notebook_task": { + "notebook_path": f"../src/{notebook_relative_path}", + "base_parameters": base_parameters, + }, + } + bridge_value_ref = f"{{{{tasks.{bridge_key}.values.{value_key}}}}}" + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=notebook_source)] + return bridge_value_ref, bridge_task, notebooks + + return items_expression, None, [] - # Also try with @ prefix if not present - if not items_expression.startswith("@"): - result = resolve_expression("@" + items_expression, context) - if result is not None and result.kind in ("dab_ref", "literal"): - return result.value - return items_expression +def _render_for_each_inputs_bridge( + notebook_code: str, + imports: list[str], + widget_names: list[str], + value_key: str, +) -> str: + """Generates the Python source for a ForEach-inputs bridge notebook. + + The notebook computes the array value and publishes it via + ``dbutils.jobs.taskValues.set`` so the parent ForEach task can + reference it via ``{{tasks..values.items}}``. + """ + lines: list[str] = [] + seen_imports: set[str] = set() + for imp in imports: + if imp in seen_imports: + continue + seen_imports.add(imp) + lines.append(imp) + if seen_imports: + lines.append("") + for widget in widget_names: + lines.append(f"dbutils.widgets.text('{widget}', '')") + if widget_names: + lines.append("") + lines.append(f"_bridge_value = {notebook_code}") + lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") + return "\n".join(lines) + "\n" def _inject_input_parameter(inner_task: dict) -> dict: @@ -73,12 +159,22 @@ def _inject_input_parameter(inner_task: dict) -> dict: return inner_task -def prepare(activity: ForEachActivity, *, scope: str = "") -> PreparedActivity: +def prepare( + activity: ForEachActivity, + *, + scope: str = "", + variable_task_keys: dict[str, str] | None = None, +) -> PreparedActivity: """Converts a ForEachActivity into a DAB for_each_task definition. Args: activity: The translated for-each activity from the IR. scope: Secret scope name (typically the pipeline/job name). + variable_task_keys: C-06 (VAREX-004): parent-job variable->setter + mapping threaded into ``collect_inner_job_params`` so + ``@variables('X')`` references in the inner-job body route + through the variable's task-value rather than fabricating an + undeclared inner-job parameter. Returns: A PreparedActivity with the for_each_task, plus any notebooks, secrets, @@ -86,35 +182,93 @@ def prepare(activity: ForEachActivity, *, scope: str = "") -> PreparedActivity: """ task = build_common_task_fields(activity) concurrency = activity.concurrency if activity.concurrency is not None else 20 - inputs = _resolve_for_each_inputs(activity.items_expression) + inputs, inputs_bridge_task, inputs_bridge_notebooks = _resolve_for_each_inputs_with_bridge(activity) + if inputs_bridge_task is not None: + existing_deps = list(task.get("depends_on") or []) + task["depends_on"] = [*existing_deps, {"task_key": inputs_bridge_task["task_key"]}] inner_activities = activity.inner_activities - all_notebooks: list[DabNotebook] = [] + all_notebooks: list[DabNotebook] = list(inputs_bridge_notebooks) all_secrets: list[SecretInstruction] = [] all_setup_tasks: list[SetupTask] = [] inner_workflows: list[PreparedWorkflow] = [] + extra_tasks: list[dict[str, Any]] = [] + if inputs_bridge_task is not None: + extra_tasks.append(inputs_bridge_task) if len(inner_activities) == 1: inner_prepared = prepare_activity(inner_activities[0], scope=scope) - inner_task = _inject_input_parameter(inner_prepared.task) all_notebooks.extend(inner_prepared.notebooks) all_secrets.extend(inner_prepared.secrets) all_setup_tasks.extend(inner_prepared.setup_tasks) inner_workflows.extend(inner_prepared.inner_workflows) - task["for_each_task"] = { - "inputs": inputs, - "task": inner_task, - "concurrency": concurrency, - } + # If the single child contributed extra_tasks (e.g. IfCondition or + # Switch branch bodies) we cannot inline as for_each_task.task — + # for_each only accepts a single task. Escalate to the sub-job + # path so the entire branch body survives (CF-001). + if inner_prepared.extra_tasks: + inner_job_name = f"{activity.task_key}_inner_tasks" + inner_tasks: list[dict[str, Any]] = [ + inner_prepared.task, + *inner_prepared.extra_tasks, + ] + normalize_inner_task_params(inner_tasks) + parameters, job_parameters = collect_inner_job_params( + inner_tasks, variable_task_keys=variable_task_keys + ) + + # LSC3-001: gather cluster hints from inner activities so the + # inner-job default cluster lifts spark_env_vars / custom_tags / + # driver_node_type_id etc. from the LS-derived cluster spec. + inner_cluster_hints: list[dict[str, Any]] = [] + for nested_activity in _iter_activity_with_descendants(inner_activities[0]): + if nested_activity.cluster: + inner_cluster_hints.append(dict(nested_activity.cluster)) + + inner_workflow = PreparedWorkflow( + name=inner_job_name, + tasks=inner_tasks, + notebooks=[], + secrets=[], + setup_tasks=[], + parameters=parameters, + cluster_hints=inner_cluster_hints, + ) + inner_workflows.append(inner_workflow) + + inner_job_key = normalize_task_key(inner_job_name) + body_task: dict[str, Any] = { + "task_key": f"{activity.task_key}_iteration", + "run_job_task": { + "job_id": f"${{resources.jobs.{inner_job_key}.id}}", + "job_parameters": job_parameters, + }, + } + + task["for_each_task"] = { + "inputs": inputs, + "task": body_task, + "concurrency": concurrency, + } + else: + inner_task = _inject_input_parameter(inner_prepared.task) + task["for_each_task"] = { + "inputs": inputs, + "task": inner_task, + "concurrency": concurrency, + } elif len(inner_activities) > 1: inner_job_name = f"{activity.task_key}_inner_tasks" - inner_tasks: list[dict[str, Any]] = [] + inner_tasks = [] for child in inner_activities: child_prepared = prepare_activity(child, scope=scope) inner_tasks.append(child_prepared.task) + # Carry IfCondition / Switch branch bodies through so the + # nested control flow survives the ForEach wrap (CF-001). + inner_tasks.extend(child_prepared.extra_tasks) all_notebooks.extend(child_prepared.notebooks) all_secrets.extend(child_prepared.secrets) all_setup_tasks.extend(child_prepared.setup_tasks) @@ -122,7 +276,18 @@ def prepare(activity: ForEachActivity, *, scope: str = "") -> PreparedActivity: normalize_inner_task_params(inner_tasks) - parameters, job_parameters = collect_inner_job_params(inner_tasks) + parameters, job_parameters = collect_inner_job_params( + inner_tasks, variable_task_keys=variable_task_keys + ) + + # LSC3-001: gather cluster hints from every nested inner activity + # so the inner-job default cluster picks up LS-derived + # spark_env_vars / custom_tags / driver_node_type_id. + inner_cluster_hints = [] + for child in inner_activities: + for nested_activity in _iter_activity_with_descendants(child): + if nested_activity.cluster: + inner_cluster_hints.append(dict(nested_activity.cluster)) inner_workflow = PreparedWorkflow( name=inner_job_name, @@ -131,11 +296,12 @@ def prepare(activity: ForEachActivity, *, scope: str = "") -> PreparedActivity: secrets=[], setup_tasks=[], parameters=parameters, + cluster_hints=inner_cluster_hints, ) inner_workflows.append(inner_workflow) inner_job_key = normalize_task_key(inner_job_name) - body_task: dict[str, Any] = { + body_task = { "task_key": f"{activity.task_key}_iteration", "run_job_task": { "job_id": f"${{resources.jobs.{inner_job_key}.id}}", @@ -158,6 +324,7 @@ def prepare(activity: ForEachActivity, *, scope: str = "") -> PreparedActivity: return PreparedActivity( task=task, + extra_tasks=extra_tasks, notebooks=all_notebooks, secrets=all_secrets, setup_tasks=all_setup_tasks, diff --git a/src/orchestra/preparer/activity_preparers/if_condition.py b/src/orchestra/preparer/activity_preparers/if_condition.py index 85473d3..f69b8b3 100644 --- a/src/orchestra/preparer/activity_preparers/if_condition.py +++ b/src/orchestra/preparer/activity_preparers/if_condition.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any +from flowx.models.dab import DabNotebook from flowx.preparer.workflow_preparer import ( PreparedActivity, PreparedArtifacts, @@ -20,6 +21,9 @@ if TYPE_CHECKING: from flowx.models.ir import IfConditionActivity +# Placeholder emitted by the translator when an operand requires a bridge. +_BRIDGE_PLACEHOLDER_PREFIX = "__BRIDGE__::" + def inject_outcome_dependency(tasks: list[dict[str, Any]], condition_key: str, outcome: str) -> None: """Gates branch-root tasks on the condition's outcome. @@ -59,12 +63,24 @@ def prepare(activity: IfConditionActivity, *, scope: str = "") -> PreparedActivi and aggregated artifacts from both branches. """ task = build_common_task_fields(activity) + + bridge_task, bridge_value_ref, bridge_notebooks = _build_bridge_task(activity) + left = _rewrite_bridge_placeholder(activity.left, bridge_value_ref) + right = _rewrite_bridge_placeholder(activity.right, bridge_value_ref) + task["condition_task"] = { "op": activity.op, - "left": activity.left, - "right": activity.right, + "left": left, + "right": right, } + # Bridge task runs ahead of the condition task -- the condition must + # depend on the bridge succeeding. + if bridge_task is not None: + bridge_dep = {"task_key": bridge_task["task_key"]} + existing_deps = list(task.get("depends_on") or []) + task["depends_on"] = [*existing_deps, bridge_dep] + artifacts = PreparedArtifacts() if_true_tasks: list[dict[str, Any]] = [] @@ -83,11 +99,96 @@ def prepare(activity: IfConditionActivity, *, scope: str = "") -> PreparedActivi artifacts = merge_prepared_artifacts(artifacts, prepared) inject_outcome_dependency(if_false_tasks, activity.task_key, "false") + extras: list[dict[str, Any]] = [] + if bridge_task is not None: + extras.append(bridge_task) + extras.extend(if_true_tasks + if_false_tasks) + + notebooks = list(artifacts.notebooks) + notebooks.extend(bridge_notebooks) + return PreparedActivity( task=task, - extra_tasks=if_true_tasks + if_false_tasks, - notebooks=list(artifacts.notebooks), + extra_tasks=extras, + notebooks=notebooks, secrets=list(artifacts.secrets), setup_tasks=list(artifacts.setup_tasks), inner_workflows=list(artifacts.inner_workflows), ) + + +def _build_bridge_task( + activity: IfConditionActivity, +) -> tuple[dict[str, Any] | None, str | None, list[DabNotebook]]: + """Synthesises a hidden SetVariable-like task that evaluates a bridged + notebook_code expression for an IfCondition operand. + + C-07 (CF-iter2-001 / CF-iter2-003 / VAREX-003): when the translator + surfaces ``bridge_notebook_code``, the preparer wires it into the job + graph as a Python notebook task that writes a single task value the + condition operand can reference. + """ + if not activity.bridge_notebook_code: + return None, None, [] + + bridge_key = f"{activity.task_key}_bridge" + value_key = "result" + notebook_relative_path = f"notebooks/{bridge_key}.py" + + # Build the bridge notebook source. base_parameters can include widget + # bindings the bridge expression depends on. + base_parameters: dict[str, str] = dict(activity.bridge_required_parameters) + notebook_source = _render_bridge_notebook( + activity.bridge_notebook_code, + activity.bridge_notebook_imports, + list(base_parameters.keys()), + value_key, + ) + + bridge_task: dict[str, Any] = { + "task_key": bridge_key, + "notebook_task": { + "notebook_path": f"../src/{notebook_relative_path}", + "base_parameters": base_parameters, + }, + } + bridge_value_ref = f"{{{{tasks.{bridge_key}.values.{value_key}}}}}" + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=notebook_source)] + return bridge_task, bridge_value_ref, notebooks + + +def _rewrite_bridge_placeholder(operand: str, bridge_value_ref: str | None) -> str: + """Rewrites a translator-side bridge placeholder to the real task value.""" + if not isinstance(operand, str): + return operand + if not operand.startswith(_BRIDGE_PLACEHOLDER_PREFIX): + return operand + if bridge_value_ref is None: + # Defensive: translator surfaced a placeholder but no bridge code. + return operand + return bridge_value_ref + + +def _render_bridge_notebook( + notebook_code: str, + imports: list[str], + widget_names: list[str], + value_key: str, +) -> str: + """Generates the Python source for a condition bridge notebook.""" + lines: list[str] = [] + seen_imports: set[str] = set() + for imp in imports: + if imp in seen_imports: + continue + seen_imports.add(imp) + lines.append(imp) + if seen_imports: + lines.append("") + for widget in widget_names: + lines.append(f"dbutils.widgets.text('{widget}', '')") + if widget_names: + lines.append("") + lines.append(f"_bridge_value = {notebook_code}") + lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") + return "\n".join(lines) + "\n" diff --git a/src/orchestra/preparer/activity_preparers/notebook.py b/src/orchestra/preparer/activity_preparers/notebook.py index c131fde..1bc8839 100644 --- a/src/orchestra/preparer/activity_preparers/notebook.py +++ b/src/orchestra/preparer/activity_preparers/notebook.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING -from flowx.models.dab import DabNotebook, ParameterApproximation +from flowx.models.dab import DabNotebook, ParameterApproximation, SetupTask from flowx.models.ir import TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string from flowx.preparer.activity_preparers.naming import notebook_filename, workspace_notebook_filename @@ -94,6 +94,51 @@ def _resolve_notebook_path(path: str) -> str: return path +_DISPATCH_STUB_WIDGET = "notebook_path" + + +def _dispatch_stub_notebook(activity: NotebookActivity, filename: str) -> str: + """Returns the body of a dynamic-dispatch stub notebook. + + C-28 (NB-ITER4-001): when the ADF ``notebookPath`` is a runtime + expression the translator couldn't reduce (e.g. ``@trim(json(...))``), + the bundle ships a stub that reads ``notebook_path`` from a widget and + calls ``dbutils.notebook.run()`` to dispatch to whatever the workflow + resolved at runtime. The base_parameters dict ferries the rest of the + widgets through. + """ + expression = activity.notebook_path_expression or "(unspecified)" + return ( + "# Databricks notebook source\n" + "# MAGIC %md\n" + f"# MAGIC # Dispatch stub: {activity.name}\n" + "# MAGIC\n" + "# MAGIC The ADF activity's `notebookPath` is a runtime expression that\n" + "# MAGIC flowx could not resolve at translation time.\n" + "# MAGIC\n" + f"# MAGIC **Original expression**: `{expression}`\n" + "# MAGIC\n" + "# MAGIC This stub reads the resolved notebook path from the\n" + f"# MAGIC `{_DISPATCH_STUB_WIDGET}` widget and dispatches via\n" + "# MAGIC `dbutils.notebook.run`. See SETUP.md → *Dynamic notebook dispatch*.\n" + "\n# COMMAND ----------\n\n" + f"dbutils.widgets.text('{_DISPATCH_STUB_WIDGET}', '')\n" + f"target_notebook = dbutils.widgets.get('{_DISPATCH_STUB_WIDGET}')\n" + "if not target_notebook:\n" + " raise ValueError(\n" + f" \"Dispatch stub for activity {activity.name!r} requires a runtime \"\n" + f" \"value for the '{_DISPATCH_STUB_WIDGET}' widget. See SETUP.md.\"\n" + " )\n" + "\n" + "# Forward every other widget through to the resolved notebook so it\n" + "# receives the same base_parameters the workflow declared.\n" + f"_passthrough_widgets = [w for w in dbutils.widgets.getAll() if w != '{_DISPATCH_STUB_WIDGET}']\n" + "arguments = {name: dbutils.widgets.get(name) for name in _passthrough_widgets}\n" + "\n" + "dbutils.notebook.run(target_notebook, timeout_seconds=0, arguments=arguments)\n" + ) + + def prepare( activity: NotebookActivity, *, @@ -101,6 +146,10 @@ def prepare( variable_task_keys: dict[str, str] | None = None, ) -> PreparedActivity: """Converts a NotebookActivity into a DAB notebook_task definition.""" + # C-28 (NB-ITER4-001): dynamic notebookPath -> emit a dispatch stub. + if activity.notebook_path_unresolved: + return _prepare_dispatch_stub(activity, variable_task_keys=variable_task_keys) + resolved_path = _resolve_notebook_path(activity.notebook_path) task = build_common_task_fields(activity) is_existing_notebook = resolved_path.startswith("/") @@ -137,14 +186,23 @@ def prepare( if activity.libraries: task["libraries"] = activity.libraries notebooks = [DabNotebook(relative_path=notebook_relative_path, content=downloaded)] - return PreparedActivity(task=task, notebooks=notebooks, parameter_approximations=approximations) + return PreparedActivity( + task=task, + notebooks=notebooks, + parameter_approximations=approximations, + setup_tasks=_unresolved_library_setup_tasks(activity), + ) task["notebook_task"] = {"notebook_path": resolved_path} if base_parameters is not None: task["notebook_task"]["base_parameters"] = base_parameters if activity.libraries: task["libraries"] = activity.libraries - return PreparedActivity(task=task, parameter_approximations=approximations) + return PreparedActivity( + task=task, + parameter_approximations=approximations, + setup_tasks=_unresolved_library_setup_tasks(activity), + ) placeholder_filename = notebook_filename(activity.task_key, activity.name) notebook_relative_path = f"notebooks/{placeholder_filename}" @@ -159,4 +217,102 @@ def prepare( task["libraries"] = activity.libraries notebooks = [DabNotebook(relative_path=notebook_relative_path, content=content)] - return PreparedActivity(task=task, notebooks=notebooks, parameter_approximations=approximations) + setup_tasks = _unresolved_library_setup_tasks(activity) + return PreparedActivity( + task=task, + notebooks=notebooks, + parameter_approximations=approximations, + setup_tasks=setup_tasks, + ) + + +def _unresolved_library_setup_tasks(activity: NotebookActivity) -> list[SetupTask]: + """Builds ``unresolved_library`` setup tasks for SETUP.md. + + C-30 (NB-ITER4-003): library entries whose jar/whl path didn't resolve + surface as a SETUP.md section so the user can fix the missing identifier + rather than discovering the failure when the cluster tries to install + a file called ``@concat(...)`` at job-run time. + """ + tasks: list[SetupTask] = [] + for entry in activity.unresolved_libraries: + tasks.append( + SetupTask( + type="unresolved_library", + config={ + "task_key": activity.task_key, + "library_type": entry.get("type", ""), + "expression": entry.get("expression", ""), + "missing": list(entry.get("missing") or []), + }, + ) + ) + return tasks + + +def _prepare_dispatch_stub( + activity: NotebookActivity, + *, + variable_task_keys: dict[str, str] | None = None, +) -> PreparedActivity: + """Builds the bundle artifacts for a dynamic-notebookPath activity. + + C-28 (NB-ITER4-001): emits a dispatch-stub notebook (reads + ``notebook_path`` widget and ``dbutils.notebook.run()``s it), a + SetupTask of kind ``dynamic_notebook_dispatch`` for SETUP.md, and + threads the original base_parameters through. + """ + task = build_common_task_fields(activity) + filename = notebook_filename(activity.task_key, activity.name) + notebook_relative_path = f"notebooks/{filename}" + + base_parameters: dict[str, str] = {} + base_parameters[_DISPATCH_STUB_WIDGET] = "" + if activity.base_parameters: + for key, value in _resolve_base_parameters( + dict(activity.base_parameters), + variable_task_keys=variable_task_keys, + existing_notebook=False, + ).items(): + base_parameters.setdefault(key, value) + + content = _dispatch_stub_notebook(activity, filename) + + task["notebook_task"] = { + "notebook_path": f"../src/{notebook_relative_path}", + "base_parameters": base_parameters, + } + if activity.libraries: + task["libraries"] = activity.libraries + + setup_tasks: list[SetupTask] = [ + SetupTask( + type="dynamic_notebook_dispatch", + config={ + "task_key": activity.task_key, + "activity_name": activity.name, + "expression": activity.notebook_path_expression or "", + "widget_name": _DISPATCH_STUB_WIDGET, + }, + ) + ] + setup_tasks.extend(_unresolved_library_setup_tasks(activity)) + + approximations = [ + ParameterApproximation( + task_key=activity.task_key, + widget_name=entry["widget_name"], + raw_expression=entry["raw_expression"], + replacement=entry["replacement"], + note=entry["note"], + ) + for entry in activity.parameter_approximations + ] + + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=content)] + return PreparedActivity( + task=task, + notebooks=notebooks, + setup_tasks=setup_tasks, + parameter_approximations=approximations, + ) diff --git a/src/orchestra/preparer/activity_preparers/set_variable.py b/src/orchestra/preparer/activity_preparers/set_variable.py index 5820929..e84e5fe 100644 --- a/src/orchestra/preparer/activity_preparers/set_variable.py +++ b/src/orchestra/preparer/activity_preparers/set_variable.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING +from flowx.models.dab import SetupTask from flowx.preparer.activity_preparers.helpers import build_notebook_activity_task from flowx.preparer.activity_preparers.naming import notebook_filename from flowx.preparer.code_generator import generate_set_variable_notebook @@ -18,6 +19,8 @@ def prepare(activity: SetVariableActivity, *, scope: str = "") -> PreparedActivi base_parameters: dict[str, str] = {"variable_name": activity.variable_name} if activity.value_kind in ("literal", "dab_ref"): base_parameters["value"] = activity.variable_value + elif activity.value_kind == "unresolved": + base_parameters["value"] = "" for widget_name, dab_ref in activity.required_parameters.items(): base_parameters.setdefault(widget_name, dab_ref) @@ -27,4 +30,20 @@ def prepare(activity: SetVariableActivity, *, scope: str = "") -> PreparedActivi notebook_content=generate_set_variable_notebook(activity), base_parameters=base_parameters, ) - return PreparedActivity(task=task, notebooks=notebooks) + + setup_tasks: list[SetupTask] = [] + # C-33 (VAREX4-001 / CF4-003): emit a manual_variable_init SetupTask so + # SETUP.md flags the variable as needing a runtime value. + if activity.value_kind == "unresolved" and activity.raw_expression: + setup_tasks.append( + SetupTask( + type="manual_variable_init", + config={ + "task_key": activity.task_key, + "variable_name": activity.variable_name, + "expression": activity.raw_expression, + }, + ) + ) + + return PreparedActivity(task=task, notebooks=notebooks, setup_tasks=setup_tasks) diff --git a/src/orchestra/preparer/activity_preparers/switch.py b/src/orchestra/preparer/activity_preparers/switch.py index a442f0c..bfec286 100644 --- a/src/orchestra/preparer/activity_preparers/switch.py +++ b/src/orchestra/preparer/activity_preparers/switch.py @@ -10,6 +10,7 @@ import re from typing import TYPE_CHECKING, Any +from flowx.models.dab import DabNotebook from flowx.models.ir import TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string from flowx.preparer.activity_preparers.if_condition import inject_outcome_dependency @@ -24,6 +25,8 @@ if TYPE_CHECKING: from flowx.models.ir import SwitchActivity +_BRIDGE_PLACEHOLDER_PREFIX = "__BRIDGE__::" + def sanitize_case_key(value: str) -> str: """Returns a task-key-safe form of a switch case value. @@ -44,14 +47,33 @@ def resolve_switch_on_expression(on_expression: str) -> str: plain literal passes through unchanged. Both the in-process preparer and the JSON-reload path call this so a hand-edited IR with a raw ``@variables(...)`` is still resolved before being written to YAML. + + C-13 (CF-iter2-004): when the input already contains a ``{{...}}`` + DAB dynamic value reference or is not an ``@``-prefixed ADF + expression, return it unchanged. Constructing a bare + :class:`TranslationContext` from this side strips global parameters + and the variable_cache, so re-resolving a previously-lowered ref + would discard the data the translator already populated. The + translator-side bridge placeholder (``__BRIDGE__::``) is likewise + preserved so the bridge rewrite step downstream can fill it. """ + if not isinstance(on_expression, str): + return on_expression + if not on_expression: + return on_expression + # Already a DAB ref / translator placeholder: pass through unchanged. + if "{{" in on_expression or on_expression.startswith(_BRIDGE_PLACEHOLDER_PREFIX): + return on_expression + # Only attempt resolution for bare ADF expressions. Other strings + # (raw literals) pass through. + if not on_expression.startswith("@"): + return on_expression context = TranslationContext() if "@{" in on_expression: return resolve_interpolated_string(on_expression, context) - if on_expression.startswith("@"): - result = resolve_expression(on_expression, context) - if result is not None and result.kind in ("dab_ref", "literal"): - return result.value + result = resolve_expression(on_expression, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value return on_expression @@ -70,6 +92,11 @@ def prepare(activity: SwitchActivity, *, scope: str = "") -> PreparedActivity: extra_tasks: list[dict[str, Any]] = [] resolved_expr = resolve_switch_on_expression(activity.on_expression) + # C-07: if the translator produced a bridge_notebook_code, synthesise + # the bridge task and rewrite the on-expression to its task value. + bridge_task, bridge_value_ref, bridge_notebooks = _build_switch_bridge_task(activity) + if bridge_value_ref is not None and resolved_expr.startswith(_BRIDGE_PLACEHOLDER_PREFIX): + resolved_expr = bridge_value_ref if not activity.cases: task = build_common_task_fields(activity) @@ -83,10 +110,19 @@ def prepare(activity: SwitchActivity, *, scope: str = "") -> PreparedActivity: artifacts = merge_prepared_artifacts(artifacts, prepared) inject_outcome_dependency(default_tasks, activity.task_key, "true") + extras_no_cases: list[dict[str, Any]] = [] + notebooks_no_cases = list(artifacts.notebooks) + if bridge_task is not None: + extras_no_cases.append(bridge_task) + notebooks_no_cases.extend(bridge_notebooks) + existing_deps = list(task.get("depends_on") or []) + task["depends_on"] = [*existing_deps, {"task_key": bridge_task["task_key"]}] + extras_no_cases.extend(default_tasks) + return PreparedActivity( task=task, - extra_tasks=default_tasks, - notebooks=list(artifacts.notebooks), + extra_tasks=extras_no_cases, + notebooks=notebooks_no_cases, secrets=list(artifacts.secrets), setup_tasks=list(artifacts.setup_tasks), inner_workflows=list(artifacts.inner_workflows), @@ -145,12 +181,87 @@ def prepare(activity: SwitchActivity, *, scope: str = "") -> PreparedActivity: # original Switch task_key onto the renamed first case. remap = {activity.task_key: case_keys[0]} + notebooks_out = list(artifacts.notebooks) + extras_out = extra_tasks + if bridge_task is not None: + # Bridge task runs first; the first condition task depends on it. + extras_out = [bridge_task, *extra_tasks] + notebooks_out.extend(bridge_notebooks) + existing_deps = list(first_condition_task.get("depends_on") or []) + first_condition_task["depends_on"] = [ + *existing_deps, + {"task_key": bridge_task["task_key"]}, + ] + return PreparedActivity( task=first_condition_task, - extra_tasks=extra_tasks, - notebooks=list(artifacts.notebooks), + extra_tasks=extras_out, + notebooks=notebooks_out, secrets=list(artifacts.secrets), setup_tasks=list(artifacts.setup_tasks), inner_workflows=list(artifacts.inner_workflows), task_key_remap=remap, ) + + +def _build_switch_bridge_task( + activity: SwitchActivity, +) -> tuple[dict[str, Any] | None, str | None, list[DabNotebook]]: + """Synthesises a bridge SetVariable-style task for a Switch on-expression. + + C-07 (CF-iter2-001 / CF-iter2-003): when ``on_expression`` contains an + ADF function call (e.g. ``@toUpper(coalesce(item()?.type, 'default'))``) + we route the value through a hidden notebook task so the + ``condition_task.left`` operand is a real task-value reference and not + a raw ADF expression string. + """ + if not activity.bridge_notebook_code: + return None, None, [] + + bridge_key = f"{activity.task_key}_bridge" + value_key = "result" + notebook_relative_path = f"notebooks/{bridge_key}.py" + + base_parameters: dict[str, str] = dict(activity.bridge_required_parameters) + notebook_source = _render_bridge_notebook( + activity.bridge_notebook_code, + activity.bridge_notebook_imports, + list(base_parameters.keys()), + value_key, + ) + + bridge_task: dict[str, Any] = { + "task_key": bridge_key, + "notebook_task": { + "notebook_path": f"../src/{notebook_relative_path}", + "base_parameters": base_parameters, + }, + } + bridge_value_ref = f"{{{{tasks.{bridge_key}.values.{value_key}}}}}" + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=notebook_source)] + return bridge_task, bridge_value_ref, notebooks + + +def _render_bridge_notebook( + notebook_code: str, + imports: list[str], + widget_names: list[str], + value_key: str, +) -> str: + """Generates the Python source for a Switch on-expression bridge notebook.""" + lines: list[str] = [] + seen_imports: set[str] = set() + for imp in imports: + if imp in seen_imports: + continue + seen_imports.add(imp) + lines.append(imp) + if seen_imports: + lines.append("") + for widget in widget_names: + lines.append(f"dbutils.widgets.text('{widget}', '')") + if widget_names: + lines.append("") + lines.append(f"_bridge_value = {notebook_code}") + lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") + return "\n".join(lines) + "\n" diff --git a/src/orchestra/preparer/activity_preparers/web_activity.py b/src/orchestra/preparer/activity_preparers/web_activity.py index b1d72f4..08bd1f2 100644 --- a/src/orchestra/preparer/activity_preparers/web_activity.py +++ b/src/orchestra/preparer/activity_preparers/web_activity.py @@ -2,9 +2,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any -from flowx.models.dab import SecretInstruction +from flowx.models.dab import SecretInstruction, SetupTask from flowx.preparer.activity_preparers.helpers import ( build_notebook_activity_task, resolve_param_value, @@ -19,25 +19,137 @@ def prepare(activity: WebActivity, *, scope: str = "") -> PreparedActivity: """Converts a WebActivity into a notebook_task with a generated HTTP notebook.""" + secrets, setup_tasks = _extract_secrets_and_setup(activity, scope=scope) + + # C-38 (LSC4-002): when the preparer resolved an AzureKeyVaultSecret + # payload to a real (scope, key) pair, thread it into the notebook + # generator so the rendered ``dbutils.secrets.get`` references the + # real values rather than the hard-coded ``scope=task_key, + # key='auth-credential'`` fallback. + credential_scope: str | None = None + credential_key: str | None = None + if secrets: + first = secrets[0] + credential_scope = first.scope + credential_key = first.key + task, notebooks = build_notebook_activity_task( activity, notebook_relative_path=f"notebooks/{notebook_filename(activity.task_key, activity.name)}", - notebook_content=generate_web_activity_notebook(activity, scope=scope), + notebook_content=generate_web_activity_notebook( + activity, + scope=scope, + credential_scope=credential_scope, + credential_key=credential_key, + ), base_parameters={ "url": resolve_param_value(activity.url), "method": resolve_param_value(activity.method), }, ) + return PreparedActivity( + task=task, + notebooks=notebooks, + secrets=secrets, + setup_tasks=setup_tasks, + ) + + +def _extract_secrets_and_setup( + activity: WebActivity, *, scope: str = "" +) -> tuple[list[SecretInstruction], list[SetupTask]]: + """Inspects the Web activity's authentication payload and emits per-secret refs. + + C-11 (LSC2-005): the legacy implementation always emitted a single + static ``auth-credential`` SecretInstruction regardless of the + underlying ADF auth shape, so AzureKeyVaultSecret payloads lost their + Key Vault scope/secret name and CredentialReference (MSI) payloads + surfaced a placeholder secret that never matches a real secret in + the workspace. + """ secrets: list[SecretInstruction] = [] - if activity.authentication: - auth_type = activity.authentication.get("type", "unknown") + setup_tasks: list[SetupTask] = [] + + auth = activity.authentication or {} + if not auth: + return secrets, setup_tasks + + auth_type = auth.get("type", "unknown") + default_scope = scope or activity.task_key + + # Common nested fields per ADF auth shapes. + for field_name in ("password", "secret", "clientSecret", "pfx", "key"): + field_value = auth.get(field_name) + secret = _materialise_secret(field_value, default_scope=default_scope, role=field_name) + if secret is not None: + secrets.append(secret) + + if auth_type == "MSI" or auth.get("credential"): + # CredentialReference (managed identity) has no static secret -- emit + # a SETUP.md note instead of a fake placeholder secret. + cred = auth.get("credential") or {} + cred_name = cred.get("referenceName") if isinstance(cred, dict) else None + setup_tasks.append( + SetupTask( + type="manual_credential", + config={ + "activity_name": activity.name, + "credential_reference": cred_name or "", + "note": ( + "Web activity uses an Azure managed-identity credential. " + "Configure equivalent OAuth or service-principal auth in Databricks " + "and update the generated notebook." + ), + }, + ) + ) + + if not secrets and auth_type not in ("MSI",) and not auth.get("credential"): + # Fallback for shapes the per-field probe didn't recognise -- preserve + # the legacy behaviour so callers depending on it still get something. secrets.append( SecretInstruction( - scope=scope or activity.task_key, + scope=default_scope, key="auth-credential", value_source=f"Authentication credential ({auth_type}) for web activity '{activity.name}'", ) ) - return PreparedActivity(task=task, notebooks=notebooks, secrets=secrets) + return secrets, setup_tasks + + +def _materialise_secret( + value: Any, *, default_scope: str, role: str +) -> SecretInstruction | None: + """Builds a :class:`SecretInstruction` from an ADF secret payload. + + Handles the two common shapes: + - ``{"type": "AzureKeyVaultSecret", "store": {"referenceName": ...}, "secretName": ...}`` + - ``{"type": "SecureString", "value": ...}`` + + Returns ``None`` for shapes we cannot map. + """ + if not isinstance(value, dict): + return None + payload_type = value.get("type") + if payload_type == "AzureKeyVaultSecret": + store = value.get("store") or {} + scope_name = store.get("referenceName") or default_scope + secret_name = value.get("secretName") or role + base_url = (value.get("typeProperties") or {}).get("baseUrl", "") + value_source = f"Azure Key Vault secret '{secret_name}'" + if base_url: + value_source += f" at {base_url}" + return SecretInstruction( + scope=str(scope_name), + key=str(secret_name), + value_source=value_source, + ) + if payload_type == "SecureString": + return SecretInstruction( + scope=default_scope, + key=role, + value_source=f"SecureString carried inline in the ADF activity (role={role})", + ) + return None diff --git a/src/orchestra/preparer/code_generator.py b/src/orchestra/preparer/code_generator.py index 4bb12fa..2f2e43a 100644 --- a/src/orchestra/preparer/code_generator.py +++ b/src/orchestra/preparer/code_generator.py @@ -88,6 +88,8 @@ def generate_lookup_notebook(activity: LookupActivity, *, scope: str = "") -> st for col_name, col_value in output.items(): dbutils.jobs.taskValues.set(key=col_name, value=col_value) """) + elif _is_file_lookup(activity): + body = _file_lookup_body(activity) else: body = textwrap.dedent(f"""\ import json @@ -116,12 +118,189 @@ def generate_lookup_notebook(activity: LookupActivity, *, scope: str = "") -> st return header + _command_separator() + body -def generate_web_activity_notebook(activity: WebActivity, *, scope: str = "") -> str: +_DATASET_TYPE_TO_SPARK_FORMAT: dict[str, str] = { + "Json": "json", + "Parquet": "parquet", + "DelimitedText": "csv", + "Avro": "avro", + "Orc": "orc", + "Excel": "com.crealytics.spark.excel", + "Xml": "xml", + "Binary": "binaryFile", +} + + +def _is_file_lookup(activity: LookupActivity) -> bool: + """Return True when the LookupActivity carries a file-source dataset.""" + props = activity.source_properties or {} + dataset_type = props.get("dataset_type") + return bool(dataset_type) and dataset_type in _DATASET_TYPE_TO_SPARK_FORMAT + + +def _coerce_to_str(value: Any) -> str: + """Defensive coercion for file-Lookup path components. + + C-37 (LSC4-001): folder_path / file_name occasionally arrive as ADF + expression dicts (``{"value": ..., "type": "Expression"}``) when the + translator's unwrap pass missed them. ``.strip('/')`` on a dict + crashes the bundler. Coerce to a string so the worst-case outcome + is a missing path component instead of a stack trace that aborts + bundle generation. + """ + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, dict) and "value" in value: + inner = value["value"] + return str(inner) if inner is not None else "" + return str(value) + + +_ADLS_HTTPS_RE = re.compile( + r"^https?://(?P[A-Za-z0-9\-]+)\.(?:dfs|blob)\.core\.windows\.net(?P/.*)?$", + re.IGNORECASE, +) + + +def _rewrite_abfss(url: str, container: str) -> str: + """Rewrites an ``https://.dfs.core.windows.net`` URL to abfss://. + + C-37 (LSC4-003): AzureBlobFS linked services often surface the + HTTPS endpoint instead of the abfss:// form Databricks expects on a + cluster. When the container is known, rewrite to + ``abfss://@.dfs.core.windows.net`` so the + generated lookup notebook can actually read the path. + """ + if not container: + return url + match = _ADLS_HTTPS_RE.match(url) + if match is None: + return url + account = match.group("account") + rest = (match.group("path") or "").lstrip("/") + suffix = f"/{rest}" if rest else "" + return f"abfss://{container}@{account}.dfs.core.windows.net{suffix}" + + +def _assemble_file_lookup_source_path(props: dict[str, Any]) -> str: + """Compose a fully-qualified default source path for a file-source Lookup. + + LSC3-005: when the bound linked service exposes a URL like + ``abfss://container@account.dfs.core.windows.net``, append the dataset's + folder + filename onto the URL so the lookup notebook ships a real + default rather than ``''``. Returns an empty string when no URL is + available -- callers can still override via the widget at runtime. + + C-37 (LSC4-001 + LSC4-003): defensively coerce expression-dict values + to strings before ``.strip('/')`` so 4 pipelines that previously + crashed with AttributeError now emit bundles. Also rewrites + ``https://.dfs.core.windows.net`` URLs to ``abfss://`` when + a container is known so the lookup notebook reads the real ADLS + path rather than the HTTPS REST endpoint. + """ + url = _coerce_to_str(props.get("linked_service_url")) + folder = _coerce_to_str(props.get("folder_path")) + filename = _coerce_to_str(props.get("file_name")) + container = _coerce_to_str(props.get("container")) + # C-47 (LSC5-001): never join a raw ``dataset()`` reference into the + # baked default path. lookup.translate substitutes these from the + # dataset reference's parameter bindings; if one still leaks through + # (e.g. an unbound dataset parameter) drop it so spark.read does not get + # a literal broken ``abfss://.../@dataset().fileName`` path. + if "dataset(" in folder: + folder = "" + if "dataset(" in filename: + filename = "" + if url: + url = _rewrite_abfss(url, container) + if not (url or folder or filename): + return "" + parts: list[str] = [] + if url: + parts.append(url.rstrip("/")) + if folder: + parts.append(folder.strip("/")) + if filename: + parts.append(filename.strip("/")) + return "/".join(p for p in parts if p) + + +def _file_lookup_body(activity: LookupActivity) -> str: + """Render the notebook body for a file-source Lookup. + + Builds a ``spark.read.format(...).option(...).load()`` call + with multiline JSON handling for ``firstRowOnly=False`` over arrayOfObjects. + The source_path is read from a widget so callers can override per run. + + LSC3-005: when the bound linked service supplies a URL (e.g. abfss:// + container@account.dfs.core.windows.net), the default widget value is + pre-populated with the fully-assembled URI so the notebook reads from + the right place without manual SETUP.md fixups. + """ + props = activity.source_properties or {} + dataset_type = props.get("dataset_type", "Json") + spark_format = _DATASET_TYPE_TO_SPARK_FORMAT.get(dataset_type, "json") + + options: list[str] = [] + if dataset_type == "Json": + # ADF Lookup with firstRowOnly=False over a JSON file typically + # walks an array-of-objects, which requires multiline. + if not activity.first_row_only or props.get("multiLineJson"): + options.append('.option("multiline", "true")') + options_block = "\n ".join(options) + options_section = ("\n " + options_block) if options_block else "" + + default_source_path = _assemble_file_lookup_source_path(props) + default_path_literal = repr(default_source_path) if default_source_path else "''" + + body = textwrap.dedent(f"""\ + import json + + # Parameters + first_row_only = dbutils.widgets.get("first_row_only") == "true" + source_path = dbutils.widgets.get("source_path") or {default_path_literal} + + # File-source Lookup + df = ( + spark.read.format({spark_format!r})__OPTIONS__ + .load(source_path) + ) + + if first_row_only: + result = df.first() + output = result.asDict() if result else {{}} + else: + output = [row.asDict() for row in df.collect()] + + dbutils.jobs.taskValues.set(key="result", value=json.dumps(output)) + if first_row_only and isinstance(output, dict): + for col_name, col_value in output.items(): + dbutils.jobs.taskValues.set(key=col_name, value=col_value) + """) + return body.replace("__OPTIONS__", options_section) + + +def generate_web_activity_notebook( + activity: WebActivity, + *, + scope: str = "", + credential_scope: str | None = None, + credential_key: str | None = None, +) -> str: """Generates a Python notebook that makes an HTTP request. Args: activity: The WebActivity IR node. scope: Secret scope name (defaults to task_key if empty). + credential_scope: C-38 (LSC4-002): when the preparer resolved the + auth payload to a real (scope, key) pair (e.g. an + AzureKeyVaultSecret with ``lakeh_ls_keyvault`` / + ``adapp-...-secret``), pass them through so the rendered + ``dbutils.secrets.get`` call references the real values + rather than the hard-coded ``scope=task_key, + key='auth-credential'`` fallback that never matches. + credential_key: See ``credential_scope``. Returns: Complete notebook source code as a string. @@ -137,10 +316,32 @@ def generate_web_activity_notebook(activity: WebActivity, *, scope: str = "") -> if auth: scope = scope or activity.task_key auth_type = auth.get("type", "") - if auth_type in ("ServicePrincipal", "MSI", "ManagedServiceIdentity"): + # C-38 (LSC4-002): prefer the resolved (scope, key) tuple from the + # preparer when supplied. Fall back to the legacy + # ``(task_key, 'auth-credential')`` shape only when the preparer + # didn't (or couldn't) compute one. + resolved_scope = credential_scope or scope + resolved_key = credential_key or "auth-credential" + if auth_type in ("MSI", "ManagedServiceIdentity"): + # LSC3-002: MSI / Managed Identity auth carries no static secret, + # so reading ``auth-credential`` from a secret scope is a fake + # placeholder that fails at runtime. Surface a NotImplementedError + # so the user can implement the credential exchange manually -- + # the manual_credential SetupTask emitted by web_activity preparer + # already flags this in SETUP.md. + auth_block = textwrap.dedent(f"""\ + # Authentication ({auth_type}) - manual implementation required + raise NotImplementedError( + "WebActivity authentication type '{auth_type}' has no static " + "secret to read. See SETUP.md (Manual credential setup) for " + "the Databricks equivalent (e.g. workspace OAuth M2M, " + "service principal token exchange)." + ) + """) + elif auth_type == "ServicePrincipal": auth_block = textwrap.dedent(f"""\ - # Authentication ({auth_type}) - auth_token = dbutils.secrets.get(scope="{scope}", key="auth-credential") + # Authentication (ServicePrincipal) + auth_token = dbutils.secrets.get(scope="{resolved_scope}", key="{resolved_key}") headers["Authorization"] = f"Bearer {{auth_token}}" """) elif auth_type == "Basic": @@ -148,14 +349,14 @@ def generate_web_activity_notebook(activity: WebActivity, *, scope: str = "") -> # Authentication (Basic) import base64 username = dbutils.secrets.get(scope="{scope}", key="auth-username") - password = dbutils.secrets.get(scope="{scope}", key="auth-credential") + password = dbutils.secrets.get(scope="{resolved_scope}", key="{resolved_key}") token = base64.b64encode(f"{{username}}:{{password}}".encode()).decode() headers["Authorization"] = f"Basic {{token}}" """) else: auth_block = textwrap.dedent(f"""\ # Authentication - auth_credential = dbutils.secrets.get(scope="{scope}", key="auth-credential") + auth_credential = dbutils.secrets.get(scope="{resolved_scope}", key="{resolved_key}") headers["Authorization"] = f"Bearer {{auth_credential}}" """) diff --git a/src/orchestra/preparer/workflow_preparer.py b/src/orchestra/preparer/workflow_preparer.py index bd8680a..6c99dea 100644 --- a/src/orchestra/preparer/workflow_preparer.py +++ b/src/orchestra/preparer/workflow_preparer.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Iterable from dataclasses import dataclass, field from typing import Any @@ -67,6 +68,9 @@ class PreparedWorkflow: cluster_hints: list[dict[str, Any]] = field(default_factory=list) pipeline_resources: list[dict[str, Any]] = field(default_factory=list) parameter_approximations: list[ParameterApproximation] = field(default_factory=list) + # C-10 (SCHED-001): serialised schedule / trigger spec the bundler + # renders as ``schedule:`` / ``trigger:`` on the emitted DAB job. + schedule: dict[str, Any] | None = None def run_if_from_adf_outcomes(outcomes: list[str | None]) -> str | None: @@ -173,6 +177,12 @@ def prepare_activity( prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) elif type(activity) is AppendVariableActivity: prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) + elif type(activity) is ForEachActivity: + # C-06 (VAREX-004): inner-job parameter collector needs the parent's + # variable -> setter mapping so @variables('X') references inside the + # ForEach body route through {{tasks.X.values.Y}} rather than an + # undeclared {{job.parameters.X}}. + prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) else: prepared = preparer_fn(activity, scope=scope) @@ -284,8 +294,14 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: all_tasks.extend(prepared.extra_tasks) artifacts = merge_prepared_artifacts(artifacts, prepared) task_key_remap.update(prepared.task_key_remap) - if activity.cluster: - cluster_hints.append(dict(activity.cluster)) + # C-04 (NB-ITER2-4 / LSC2-001): walk into IfCondition / Switch / + # ForEach branches so the workflow's cluster_hints aggregation + # picks up cluster config on activities nested inside compound + # activities. Without this the default Standard_DS3_v2 / 15.4.x + # fallback ships even when the inner notebook has an explicit LS. + for nested_activity in _iter_activity_with_descendants(activity): + if nested_activity.cluster: + cluster_hints.append(dict(nested_activity.cluster)) if isinstance(activity, (SetVariableActivity, AppendVariableActivity)): variable_task_keys_map[activity.variable_name] = activity.task_key @@ -305,19 +321,199 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: seen_secrets.add(secret_id) unique_secrets.append(secret) + # VAREX3-003: emit a manual_variable_rollup SetupTask whenever a sibling + # IfCondition / Switch / SetVariable reads a variable that is only + # mutated inside a ForEach inner job. ADF semantics treat the post- + # ForEach read as "latest committed value" but that value is unreachable + # across the run_job_task boundary in DAB. Surfacing the warning lets + # the user add a roll-up notebook before the dependent activity runs. + cross_scope_rollups = _detect_cross_foreach_variable_reads(pipeline.tasks) + setup_tasks_out = _dedupe_setup_tasks(artifacts.setup_tasks) + setup_tasks_out.extend(cross_scope_rollups) + # C-36 (SCHED4-001): emit a manual_schedule_time_of_day SetupTask + # whenever the trigger.periodic schedule carries hours/minutes/weekDays + # that the periodic primitive can't encode. SETUP.md picks it up so + # the user can manually add the time-of-day to the cron expression. + if pipeline.schedule and pipeline.schedule.get("time_of_day_note"): + setup_tasks_out.append( + SetupTask( + type="manual_schedule_time_of_day", + config={ + "pipeline": pipeline.name, + "frequency": pipeline.schedule.get("unit", ""), + "interval": pipeline.schedule.get("interval", ""), + "time_of_day_note": pipeline.schedule.get("time_of_day_note"), + }, + ) + ) + + # C-39 (LSC4-004): when any cluster hint references an ADF + # authentication mode that has no direct Databricks equivalent (MSI, + # CredentialReference) the bundle's default_cluster silently uses + # ``single_user_name: ${workspace.current_user.userName}``. Surface + # a manual_credential SetupTask so SETUP.md flags the substitution. + seen_auth: set[tuple[str, str]] = set() + for hint in cluster_hints: + auth = hint.get("_adf_authentication") or "" + cred = hint.get("_adf_credential_reference") or "" + if not auth and not cred: + continue + key = (str(auth), str(cred)) + if key in seen_auth: + continue + seen_auth.add(key) + setup_tasks_out.append( + SetupTask( + type="manual_credential", + config={ + "source": pipeline.name, + "linked_service": cred or "", + "authentication": auth or "CredentialReference", + "note": ( + "ADF cluster auth has no Databricks equivalent. The " + "default_cluster runs as ${workspace.current_user.userName}; " + "swap to a service principal via single_user_name or " + "set run_as.service_principal_name on the job." + ), + }, + ) + ) + return PreparedWorkflow( name=pipeline.name, tasks=all_tasks, notebooks=list(artifacts.notebooks), secrets=unique_secrets, - setup_tasks=_dedupe_setup_tasks(artifacts.setup_tasks), + setup_tasks=setup_tasks_out, inner_workflows=list(artifacts.inner_workflows), cluster_hints=cluster_hints, pipeline_resources=list(artifacts.pipeline_resources), parameter_approximations=list(artifacts.parameter_approximations), + schedule=pipeline.schedule, ) +def _detect_cross_foreach_variable_reads(activities: list[Activity]) -> list[SetupTask]: + """Returns SetupTasks for variables mutated inside a ForEach but read outside. + + VAREX3-003: when a SetVariable for `X` lives only inside a ForEach + inner-job, a sibling task reading @variables('X') gets the stale init + value (post-ForEach reads cannot cross the run_job_task boundary in + DAB). Surfacing this as a manual_variable_rollup SetupTask gives the + user a documented workaround (add a roll-up notebook that copies the + final value to a parent-scope task value). + """ + import re + + var_ref_pattern = re.compile(r"@?variables\(\s*'([^']+)'\s*\)", re.IGNORECASE) + + # Index variable -> set of ForEach activity names that contain the setter + # so we can name the parent in the warning message. + var_set_inside_foreach: dict[str, list[str]] = {} + for activity in activities: + if isinstance(activity, ForEachActivity): + for inner in activity.inner_activities: + if isinstance(inner, SetVariableActivity): + var_set_inside_foreach.setdefault(inner.variable_name, []).append( + activity.task_key + ) + + if not var_set_inside_foreach: + return [] + + # Identify variables that are also set OUTSIDE any ForEach -- those are + # not cross-scope dangers because the parent always has a fresh setter + # to point at. + set_outside: set[str] = set() + for activity in activities: + if isinstance(activity, SetVariableActivity): + set_outside.add(activity.variable_name) + + dangerous_vars = { + name: parents for name, parents in var_set_inside_foreach.items() + if name not in set_outside + } + if not dangerous_vars: + return [] + + # Now find sibling reads of those variables. Walk every top-level + # activity that is NOT the originating ForEach and collect refs. + flagged: dict[str, str] = {} # variable -> parent task_key + + def _read_refs(text: str) -> set[str]: + if not isinstance(text, str): + return set() + return {m.group(1) for m in var_ref_pattern.finditer(text)} + + def _walk_activity_strings(activity: Activity) -> Iterable[str]: + # Yield every string-like field the variable might appear in. + if isinstance(activity, IfConditionActivity): + yield activity.left or "" + yield activity.right or "" + if isinstance(activity, SwitchActivity): + yield activity.on_expression or "" + if isinstance(activity, SetVariableActivity): + yield activity.variable_value or "" + if isinstance(activity, NotebookActivity): + for value in (activity.base_parameters or {}).values(): + if isinstance(value, str): + yield value + if isinstance(activity, WebActivity): + yield activity.url or "" + if isinstance(activity.body, str): + yield activity.body + + for activity in activities: + # Skip ForEach themselves (siblings only). + if isinstance(activity, ForEachActivity): + continue + for text in _walk_activity_strings(activity): + for var_name in _read_refs(text): + if var_name in dangerous_vars and var_name not in flagged: + flagged[var_name] = dangerous_vars[var_name][0] + + return [ + SetupTask( + type="manual_variable_rollup", + config={ + "variable_name": var_name, + "parent_foreach": parent_key, + "message": ( + f"Variable '{var_name}' is mutated inside ForEach " + f"'{parent_key}' but read in a sibling task. Task " + f"values cannot cross run_job_task boundaries; add a " + f"roll-up notebook that copies the final value to a " + f"parent-scope task value before the sibling runs." + ), + }, + ) + for var_name, parent_key in sorted(flagged.items()) + ] + + +def _iter_activity_with_descendants(activity: Activity) -> Iterable[Activity]: + """Yields *activity* and every nested branch activity (BFS). + + C-04 (NB-ITER2-4 / LSC2-001): IfCondition / Switch / ForEach activities + nest sub-activities in branch fields (``if_true_activities``, + ``if_false_activities``, ``inner_activities``, ``cases[].activities``, + ``default_activities``). ``prepare_workflow`` previously only saw the + top-level tasks, so cluster hints carried by a deeply nested + NotebookActivity were dropped. + """ + queue: list[Activity] = [activity] + while queue: + current = queue.pop(0) + yield current + for attr in ("inner_activities", "if_true_activities", "if_false_activities"): + nested = getattr(current, attr, None) or [] + queue.extend(nested) + if isinstance(current, SwitchActivity): + for case_item in current.cases: + queue.extend(case_item.activities) + queue.extend(current.default_activities) + + def _dedupe_setup_tasks(setup_tasks: tuple[SetupTask, ...]) -> list[SetupTask]: """Returns the setup-task list with duplicates collapsed by identifying config. diff --git a/src/orchestra/translator/activity_translators/execute_pipeline.py b/src/orchestra/translator/activity_translators/execute_pipeline.py index 6c2eef1..89276c9 100644 --- a/src/orchestra/translator/activity_translators/execute_pipeline.py +++ b/src/orchestra/translator/activity_translators/execute_pipeline.py @@ -6,7 +6,8 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, ExecutePipelineActivity, TranslationContext -from flowx.translator.activity_translators.resolve import resolve_dict_values, resolve_field +from flowx.parser.expression_parser import resolve_expression +from flowx.translator.activity_translators.resolve import resolve_field def translate( @@ -35,12 +36,67 @@ def translate( else str(pipeline_ref) ) - parameters = resolve_dict_values(type_properties.get("parameters"), context) or {} + raw_parameters = type_properties.get("parameters") or {} + parameters: dict[str, str] = {} + approximations: list[dict[str, str]] = list(base_kwargs.get("parameter_approximations") or []) + for name, value in raw_parameters.items(): + resolved_value = _resolve_execute_pipeline_parameter(name, value, context, approximations) + if resolved_value is not None: + parameters[name] = resolved_value + wait_on_completion = type_properties.get("waitOnCompletion", True) + if approximations: + # Stamp the approximations onto the activity so the bundler can + # surface them in SETUP.md. + base_kwargs = {**base_kwargs, "parameter_approximations": approximations} + return ExecutePipelineActivity( **base_kwargs, pipeline_name=pipeline_name, parameters=parameters, wait_on_completion=wait_on_completion, ) + + +def _resolve_execute_pipeline_parameter( + name: str, + value: Any, + context: TranslationContext, + approximations: list[dict[str, str]], +) -> str | None: + """Resolves a single ExecutePipeline parameter or drops it with a SETUP note. + + C-09 (VAREX-001): when the value resolves to ``notebook_code`` the + parameter cannot ride through ``job_parameters`` as a literal Python + source string -- the sub-job's widget would receive the code text. + Drop the parameter and record an approximation so the bundler surfaces + it in SETUP.md for the user to supply manually. + """ + if value is None: + return "" + result = resolve_expression(value, context) + if result is None: + # Fallback to the legacy resolve_field for plain strings / dicts. + return resolve_field(value, context) + if result.kind in ("literal", "dab_ref"): + return result.value + if result.kind == "notebook_code": + raw = value + if isinstance(value, dict) and "value" in value: + raw = value["value"] + approximations.append( + { + "widget_name": name, + "raw_expression": str(raw), + "replacement": "", + "note": ( + "ExecutePipeline parameter dropped: the value resolves to a " + "notebook_code expression which cannot ride through DAB " + "job_parameters as a literal. Supply manually in SETUP.md or " + "synthesise a generator task that publishes a task value." + ), + } + ) + return None + return None diff --git a/src/orchestra/translator/activity_translators/for_each.py b/src/orchestra/translator/activity_translators/for_each.py index 54146b3..f13b655 100644 --- a/src/orchestra/translator/activity_translators/for_each.py +++ b/src/orchestra/translator/activity_translators/for_each.py @@ -35,8 +35,27 @@ def translate( items_raw = type_properties.get("items") expr_result = resolve_expression(items_raw, context) if items_raw is not None else None + inputs_bridge_notebook_code: str | None = None + inputs_bridge_notebook_imports: list[str] = [] + inputs_bridge_required_parameters: dict[str, str] = {} if expr_result is not None and expr_result.kind in ("dab_ref", "literal"): items_expression = expr_result.value + elif expr_result is not None and expr_result.kind == "notebook_code": + # C-31 (CF4-001): the preparer used to construct a bare + # TranslationContext() and re-resolve the items expression on the + # JSON-reload path, but ``variable_cache`` is empty there so the + # bridge never fired and DAB rejected the raw @split(...) call. + # Capture the resolved notebook_code here while the full context + # is available; the preparer reads it from these IR fields. + if isinstance(items_raw, dict) and items_raw.get("type") == "Expression": + items_expression = items_raw.get("value", "") + elif isinstance(items_raw, str): + items_expression = items_raw + else: + items_expression = "" + inputs_bridge_notebook_code = expr_result.value + inputs_bridge_notebook_imports = list(expr_result.imports) + inputs_bridge_required_parameters = dict(expr_result.required_parameters) else: # Fallback: extract raw string if isinstance(items_raw, dict) and items_raw.get("type") == "Expression": @@ -64,6 +83,8 @@ def translate( registry=context.registry, variable_cache=context.variable_cache, variable_value_cache=context.variable_value_cache, + global_parameters=context.global_parameters, + linked_service_parameters=context.linked_service_parameters, ) inner_activities, _ = translate_activities_fn(child_adf_activities, child_context, definitions) @@ -72,6 +93,9 @@ def translate( items_expression=items_expression, inner_activities=inner_activities, concurrency=batch_count, + inputs_bridge_notebook_code=inputs_bridge_notebook_code, + inputs_bridge_notebook_imports=inputs_bridge_notebook_imports, + inputs_bridge_required_parameters=inputs_bridge_required_parameters, ) return foreach_activity, context diff --git a/src/orchestra/translator/activity_translators/if_condition.py b/src/orchestra/translator/activity_translators/if_condition.py index c450f1d..c7db2dc 100644 --- a/src/orchestra/translator/activity_translators/if_condition.py +++ b/src/orchestra/translator/activity_translators/if_condition.py @@ -11,7 +11,11 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, IfConditionActivity, TranslationContext -from flowx.parser.expression_parser import resolve_expression +from flowx.translator.activity_translators.resolve import ( + BridgeRequest, + lower_to_bridge, + merge_bridge_requests, +) # --------------------------------------------------------------------------- # ADF comparison function -> Databricks condition_task op mapping @@ -69,7 +73,7 @@ def translate( type_properties = activity.type_properties or {} expression_raw = type_properties.get("expression", {}) - op, left, right = _parse_condition(expression_raw, context) + op, left, right, bridge = _parse_condition(expression_raw, context) if_true_activities: list[Activity] = [] if_true_adf = activity.if_true_activities or [] @@ -81,6 +85,14 @@ def translate( if translate_activities_fn and if_false_adf: if_false_activities, _ = translate_activities_fn(if_false_adf, context, definitions) + bridge_kwargs: dict[str, Any] = {} + if bridge is not None: + bridge_kwargs = { + "bridge_notebook_code": bridge.notebook_code, + "bridge_notebook_imports": list(bridge.notebook_imports), + "bridge_required_parameters": dict(bridge.required_parameters), + } + if_activity = IfConditionActivity( **base_kwargs, op=op, @@ -88,20 +100,35 @@ def translate( right=right, if_true_activities=if_true_activities, if_false_activities=if_false_activities, + **bridge_kwargs, ) return if_activity, context -def _parse_condition(expression: dict[str, Any] | str, context: TranslationContext) -> tuple[str, str, str]: - """Parses an ADF IfCondition expression into ``(op, left, right)``. +_BRIDGE_TASK_VALUE_KEY = "result" + + +def _parse_condition( + expression: dict[str, Any] | str, context: TranslationContext +) -> tuple[str, str, str, BridgeRequest | None]: + """Parses an ADF IfCondition expression into ``(op, left, right, bridge)``. + + C-07 (CF-iter2-001 / CF-iter2-003 / VAREX-003): when an operand + resolves to ``notebook_code`` (e.g. ``@empty(X)``, ``@toUpper(...)``), + we package the code into a :class:`BridgeRequest` so the preparer can + emit a hidden SetVariable task whose value drives the + condition_task. The condition operand becomes the bridge's task + value reference. Args: expression: Raw ADF expression dict or string. context: Translation context for resolving variables. Returns: - Tuple of ``(databricks_op, left_operand, right_operand)``. + ``(databricks_op, left_operand, right_operand, bridge_request)`` + where ``bridge_request`` is non-None when the condition required + lowering to a notebook task. """ expr_str = "" if isinstance(expression, dict): @@ -117,9 +144,9 @@ def _parse_condition(expression: dict[str, Any] | str, context: TranslationConte inner_op_name = m_not.group(1).lower() op = _NEGATE_OP_MAP.get(inner_op_name, "NOT_EQUAL") args = _split_args(m_not.group(2).strip()) - left = _resolve_operand(args[0], context) if len(args) > 0 else "" - right = _resolve_operand(args[1], context) if len(args) > 1 else "" - return op, left, right + left, left_bridge = _resolve_operand(args[0], context) if len(args) > 0 else ("", None) + right, right_bridge = _resolve_operand(args[1], context) if len(args) > 1 else ("", None) + return op, left, right, merge_bridge_requests(left_bridge, right_bridge) m = _COMPARISON_RE.match(expr_str.strip()) if m: @@ -128,65 +155,163 @@ def _parse_condition(expression: dict[str, Any] | str, context: TranslationConte if adf_op == "not": inner = m.group(2).strip() - resolved = _resolve_operand(inner, context) - return "NOT_EQUAL", resolved, "" + resolved, bridge = _resolve_operand(inner, context) + # C-15 (CF3-003 / VAREX3-004): when the operand bridges to a + # Python bool task value, compare against 'False' (not '') so + # the IfCondition can actually evaluate to FALSE. + right_operand = "False" if bridge is not None else "" + return "NOT_EQUAL", resolved, right_operand, bridge args = _split_args(m.group(2).strip()) - left = _resolve_operand(args[0], context) if len(args) > 0 else "" - right = _resolve_operand(args[1], context) if len(args) > 1 else "" - return op, left, right - - # Fallback: treat the whole expression as a truthy check - resolved = _resolve_operand(expr_str, context) - return "NOT_EQUAL", resolved, "0" - - -def _resolve_operand(operand: str, context: TranslationContext) -> str: - """Converts an ADF expression operand to a Databricks task value reference. - - Examples:: - - activity('Lookup').output.firstRow.cnt - -> {{tasks.Lookup.values.cnt}} - - activity('Lookup').output.value - -> {{tasks.Lookup.values.result}} - - 0 -> 0 (literal) - 'active' -> active (string literal) - null -> "" (null literal) + left, left_bridge = _resolve_operand(args[0], context) if len(args) > 0 else ("", None) + right, right_bridge = _resolve_operand(args[1], context) if len(args) > 1 else ("", None) + return op, left, right, merge_bridge_requests(left_bridge, right_bridge) + + # Fallback: treat the whole expression as a truthy check. C-07: route + # through the bridge path when the expression is an ADF function call + # so the operand ends up as a real task-value reference rather than + # the legacy NOT_EQUAL '0' against a raw expression string. + resolved, bridge = _resolve_operand(expr_str, context) + if bridge is not None: + return "NOT_EQUAL", _bridge_task_value_placeholder(), "False", bridge + # C-15 (CF3-003 / VAREX3-004): when the truthy operand resolves to a + # task-value ref backed by a SetVariable that writes a Python bool + # (e.g. a previously-cached @variables('continue') with bridge-set + # value), compare against 'False' so the legacy truthy path doesn't + # silently invert behaviour. Detected by the presence of a + # __BRIDGE__:: placeholder or the lowercase 'true'/'false' literal + # body of the upstream SetVariable. + if isinstance(resolved, str) and "__BRIDGE__" in resolved: + return "NOT_EQUAL", resolved, "False", None + # C-43 (CF5-001 / LSC5-001): when the operand is a known-Boolean + # variable that resolves to a parent-job task-value ref + # (``{{tasks._init_X.values.X}}``), prefer recomputing the boolean + # locally via a BridgeRequest, mirroring the Switch path. Without this + # an inner-ForEach IfCondition references a task that lives only in the + # parent job; the bundler then blanks the operand to '' and + # NOT_EQUAL('', '0') is always TRUE, running the true branch + # unconditionally with no SETUP.md signal. The bridge keeps the + # operand local so it survives the dangling-ref safety net. + if _operand_is_known_boolean(expr_str, context): + bridge = _boolean_variable_bridge(expr_str, resolved, context) + if bridge is not None: + return "NOT_EQUAL", _bridge_task_value_placeholder(), "False", bridge + # C-32 (CF4-002): compare against lowercase ``'false'`` (matching + # C-21 SetVariable rendering) instead of the legacy ``'0'`` — the + # latter is always true for a Boolean-string operand so the false + # branch becomes dead code. + return "NOT_EQUAL", resolved, "false", None + return "NOT_EQUAL", resolved, "0", None + + +def _boolean_variable_bridge( + expr: str, resolved: str, context: TranslationContext +) -> BridgeRequest | None: + """Builds a local-recompute BridgeRequest for a Boolean-variable operand. + + C-43 (CF5-001): the bridge re-derives the boolean inside whatever job + the IfCondition lands in (parent or split-out inner ForEach job), so + the condition operand is a *local* task value rather than a parent-job + ref the bundler would blank. The recomputed value is the variable's + seeded literal default (``true``/``false``); when no literal default is + cached the caller falls back to the in-place ``'false'`` comparison. + + Returns ``None`` when there is no literal default to recompute from + (e.g. the variable is set dynamically), leaving the legacy path intact. + """ + expr = expr.strip() + if expr.startswith("@"): + expr = expr[1:] + var_match = re.match(r"variables\(\s*'([^']+)'\s*\)\s*$", expr, re.IGNORECASE) + if not var_match: + return None + var_name = var_match.group(1) + literal = context.get_variable_default_literal(var_name) + if literal is None or literal.lower() not in ("true", "false"): + return None + python_bool = "True" if literal.lower() == "true" else "False" + return BridgeRequest(notebook_code=python_bool) + + +def _operand_is_known_boolean(expr: str, context: TranslationContext) -> bool: + """Returns True when *expr* references a Boolean-typed variable / parameter. + + Inspects ``context.variable_value_cache`` (populated by C-05 init + SetVariable activities with lowercase ``'true'/'false'`` defaults), the + declared ``context.variable_types`` map (C-41), and bare + ``@pipeline().parameters.`` references when the context carries + Boolean type hints. When the type is known to be Boolean we return + True so the IfCondition fallback uses ``'false'`` as the right operand. + """ + expr = expr.strip() + if expr.startswith("@"): + expr = expr[1:] + # @variables('X') -> look up the cached lowercase value + var_match = re.match(r"variables\(\s*'([^']+)'\s*\)\s*$", expr, re.IGNORECASE) + if var_match: + var_name = var_match.group(1) + cached = context.get_variable_dab_ref(var_name) + if isinstance(cached, str) and cached.lower() in ("true", "false"): + return True + # C-41 (CF5-001): a Boolean variable seeded only by a literal + # default init task never populates variable_value_cache as a + # dab_ref, so fall back to its declared ADF type. + declared = context.get_variable_type(var_name) + if isinstance(declared, str) and declared.lower() in ("boolean", "bool"): + return True + # @pipeline().parameters.X -- without parameter type metadata we + # cannot prove Booleanness; return False conservatively. + return False + + +def _bridge_task_value_placeholder() -> str: + """Sentinel left-operand the preparer rewrites to the bridge task value.""" + return f"__BRIDGE__::{_BRIDGE_TASK_VALUE_KEY}" + + +def _resolve_operand( + operand: str, context: TranslationContext +) -> tuple[str, BridgeRequest | None]: + """Converts an ADF expression operand to a Databricks task value reference + or a :class:`BridgeRequest` when the operand requires a bridge task. Args: operand: A single operand string from the parsed condition. context: Translation context for resolving variables. Returns: - A DAB dynamic value reference or literal string. + ``(operand, bridge_request)`` where ``bridge_request`` is None for + literals / DAB refs. """ operand = operand.strip() if operand.lower() == "null": - return "" + return "", None if operand.startswith("'") and operand.endswith("'"): - return operand[1:-1] + return operand[1:-1], None if operand.lstrip("-").replace(".", "", 1).isdigit(): - return operand + return operand, None inner = _unwrap_functions(operand) - # Try unified expression resolution with @ prefix - result = resolve_expression("@" + inner, context) - if result is not None and result.kind in ("dab_ref", "literal"): - return result.value + sub_expr = inner if inner.startswith("@") else "@" + inner + operand_value, bridge = lower_to_bridge(sub_expr, context) + if operand_value is not None: + return operand_value, None + if bridge is not None: + return _bridge_task_value_placeholder(), bridge if inner != operand: - result = resolve_expression("@" + operand, context) - if result is not None and result.kind in ("dab_ref", "literal"): - return result.value - - return operand + sub_expr_full = operand if operand.startswith("@") else "@" + operand + operand_value, bridge = lower_to_bridge(sub_expr_full, context) + if operand_value is not None: + return operand_value, None + if bridge is not None: + return _bridge_task_value_placeholder(), bridge + + return operand, None def _unwrap_functions(expr: str) -> str: diff --git a/src/orchestra/translator/activity_translators/lookup.py b/src/orchestra/translator/activity_translators/lookup.py index 3620df8..7515250 100644 --- a/src/orchestra/translator/activity_translators/lookup.py +++ b/src/orchestra/translator/activity_translators/lookup.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from typing import Any from flowx.models.adf_ast import AdfActivity, AdfDefinitions @@ -9,6 +10,78 @@ from flowx.translator.activity_translators.resolve import resolve_field +def _dataset_parameter_scope( + activity: AdfActivity, context: TranslationContext +) -> dict[str, str]: + """Resolve the Lookup dataset reference's ``parameters`` binding. + + C-47 (LSC5-001): a file-source dataset's ``folderPath`` / ``fileName`` + can reference its own parameters via ``dataset().X``. The Lookup's + ``typeProperties.dataset.parameters`` block binds each dataset parameter + (e.g. ``digitalCase``) to a pipeline-scoped value (e.g. + ``@pipeline().parameters.digitalCaseCode``). Resolve each binding + against the pipeline ``TranslationContext`` so ``dataset().digitalCase`` + can be substituted with the resolved ``{{job.parameters.X}}`` ref / literal. + + Returns a ``{dataset_param_name: resolved_value}`` map (empty when the + reference carries no parameters). + """ + type_props = activity.type_properties or {} + ref = type_props.get("dataset") + if not isinstance(ref, dict): + return {} + params = ref.get("parameters") + if not isinstance(params, dict): + return {} + return {name: resolve_field(value, context) for name, value in params.items()} + + +def _substitute_dataset_refs(value: Any, scope: dict[str, str]) -> Any: + """Replace ``dataset().X`` occurrences in *value* with the resolved binding. + + C-47 (LSC5-001): without this, ``folderPath`` of + ``@toLower(dataset().digitalCase)`` and ``fileName`` of + ``@dataset().fileName`` pass through verbatim and the code generator + bakes a literal broken ``abfss://.../@toLower(dataset().digitalCase)`` + default path that ``spark.read`` cannot load. + + Only string values carrying a ``dataset().`` reference are rewritten; + everything else is returned unchanged. + """ + if not isinstance(value, str) or "dataset(" not in value: + return value + result = value + for name, resolved in scope.items(): + # dataset().X and dataset()['X'] / dataset()["X"] forms. + result = re.sub( + r"dataset\(\)\s*(?:\.\s*" + re.escape(name) + r"\b|\[\s*['\"]" + + re.escape(name) + r"['\"]\s*\])", + resolved, + result, + ) + return result + + +def _unwrap_expression(value: Any) -> Any: + """Unwrap a ``{"value": X, "type": "Expression"}`` dict-wrapper. + + C-37 (LSC4-001): folder_path / file_name on file-source datasets + sometimes ship as the ADF expression dict shape. Without unwrapping, + downstream code (notably ``_assemble_file_lookup_source_path``) calls + ``.strip('/')`` on the dict and crashes with AttributeError, which + has bundled 4 pipelines into "empty bundle directory" outcomes. + """ + if isinstance(value, dict) and "value" in value and value.get("type") == "Expression": + return value["value"] + return value + +# File-source dataset types that a Lookup can read directly. Keeping +# this list local avoids tugging the broader copy translator in. +_FILE_DATASET_TYPES: frozenset[str] = frozenset( + {"Json", "Parquet", "DelimitedText", "Avro", "Orc", "Excel", "Xml", "Binary"} +) + + def translate( activity: AdfActivity, base_kwargs: dict[str, Any], @@ -39,6 +112,59 @@ def translate( first_row_only = type_properties.get("firstRowOnly", True) + # Resolve the Lookup's dataset reference (lookup-translator-ignores-dataset-reference): + # typeProperties.dataset is the canonical place for ADF; activity.inputs + # is the legacy fall-back used by the loader for flattened activity shapes. + dataset_ref = _resolve_lookup_dataset(activity, definitions) + if dataset_ref is not None: + dataset_props = dataset_ref["properties"] + dataset_type = dataset_ref["type"] + type_props = dataset_props.get("typeProperties") or {} + location = type_props.get("location") or {} + if dataset_type in _FILE_DATASET_TYPES: + source_properties.setdefault("dataset_type", dataset_type) + # Stash the dataset path components so the code generator can + # build the right spark.read call. Avoid pulling in the full + # copy translator dataset-path machinery — we only need the + # raw container + folder + filename to surface to the user. + # C-37 (LSC4-001): unwrap any ADF expression dict shapes so + # downstream code can treat these as plain strings. + container = _unwrap_expression( + location.get("container") or location.get("fileSystem") or location.get("bucketName") + ) + folder = _unwrap_expression(location.get("folderPath")) + filename = _unwrap_expression(location.get("fileName")) + # C-47 (LSC5-001): substitute dataset().X param refs using the + # Lookup dataset reference's parameter bindings, then resolve the + # result so the path default is a real literal / interpolated + # {{job.parameters.X}} string rather than a verbatim dataset() + # expression the code generator would bake into a broken path. + ds_scope = _dataset_parameter_scope(activity, context) + if ds_scope: + if isinstance(folder, str) and folder: + folder = resolve_field(_substitute_dataset_refs(folder, ds_scope), context) + if isinstance(filename, str) and filename: + filename = resolve_field(_substitute_dataset_refs(filename, ds_scope), context) + if container: + source_properties.setdefault("container", container) + if folder: + source_properties.setdefault("folder_path", folder) + if filename: + source_properties.setdefault("file_name", filename) + # Forward type-specific format options (multiline, encoding, etc.) + # so the generator can pass them as spark.read.option(...). + format_settings = type_props.get("formatSettings") or {} + if isinstance(format_settings, dict): + for key in ("multiLineJson", "filePattern"): + if key in format_settings: + source_properties.setdefault(key, format_settings[key]) + # LSC3-005: surface the linked service URL when present so the + # generator can assemble the abfss:// path for AzureBlobFS / ADLS + # backed file datasets. + ls_url = dataset_props.get("linked_service_url") + if ls_url: + source_properties.setdefault("linked_service_url", ls_url) + return LookupActivity( **base_kwargs, source_type=source_type, @@ -46,3 +172,44 @@ def translate( first_row_only=first_row_only, source_query=source_query, ) + + +def _resolve_lookup_dataset( + activity: AdfActivity, + definitions: AdfDefinitions, +) -> dict[str, Any] | None: + """Resolves the Lookup's dataset reference to its full dataset record. + + Args: + activity: The ADF Lookup activity AST node. + definitions: Full ADF definitions for dataset lookup. + + Returns: + Dict with keys ``type`` and ``properties`` describing the bound + dataset, or ``None`` when no dataset is referenced. + """ + type_props = activity.type_properties or {} + ref = type_props.get("dataset") + dataset_name: str | None = None + if isinstance(ref, dict): + dataset_name = ref.get("referenceName") or ref.get("dataset", {}).get("referenceName") + if dataset_name is None and activity.inputs: + dataset_name = activity.inputs[0].reference_name + if dataset_name is None: + return None + # LSC3-005: ADF identifiers are case-insensitive; tolerate casing drift + # between the pipeline's dataset reference and the source JSON filename. + dataset = definitions.get_dataset(dataset_name) + if dataset is None: + return None + properties = dict(dataset.properties or {}) + # Thread linkedService typeProperties.url through onto the properties so + # the lookup notebook can assemble the abfss:// file path for file-source + # datasets where the URL is only known on the linked service. + linked_service = definitions.get_linked_service(dataset.linked_service_name) + if linked_service is not None: + ls_props = linked_service.properties or {} + ls_type_props = ls_props.get("typeProperties") if isinstance(ls_props, dict) else None + if isinstance(ls_type_props, dict) and "url" in ls_type_props: + properties.setdefault("linked_service_url", ls_type_props["url"]) + return {"type": dataset.type, "properties": properties} diff --git a/src/orchestra/translator/activity_translators/notebook.py b/src/orchestra/translator/activity_translators/notebook.py index 438c001..6bb10a3 100644 --- a/src/orchestra/translator/activity_translators/notebook.py +++ b/src/orchestra/translator/activity_translators/notebook.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from typing import Any from flowx.models.adf_ast import AdfActivity, AdfDefinitions @@ -29,9 +30,16 @@ def translate( """ type_properties = activity.type_properties or {} - notebook_path = resolve_field(type_properties.get("notebookPath", ""), context) + # C-28 (NB-ITER4-001): when notebookPath is an ADF expression that lowers + # to notebook_code (e.g. @trim(json(...).notebook_path)), preserve the raw + # expression and mark the activity so the preparer emits a dispatch stub + # rather than inlining Python source as the workspace path. + notebook_path_raw = type_properties.get("notebookPath", "") + notebook_path, notebook_path_unresolved, notebook_path_expression = _resolve_notebook_path_field( + notebook_path_raw, context + ) raw_params = type_properties.get("baseParameters") or {} - libraries = type_properties.get("libraries") + libraries, unresolved_libraries = _resolve_libraries(type_properties.get("libraries"), context) # Resolve base_parameters at translate time so ADF expressions like # @variables('runTimestamp') are inlined to DAB refs while the full @@ -61,14 +69,153 @@ def translate( return NotebookActivity( **base_kwargs, notebook_path=notebook_path, + notebook_path_unresolved=notebook_path_unresolved, + notebook_path_expression=notebook_path_expression, base_parameters=resolved_params, libraries=libraries, + unresolved_libraries=unresolved_libraries, parameter_approximations=approximations, ) +def _resolve_notebook_path_field( + value: Any, + context: TranslationContext, +) -> tuple[str, bool, str | None]: + """Resolves the ADF ``notebookPath`` field, preserving dynamic dispatch shapes. + + C-28 (NB-ITER4-001): the legacy ``resolve_field`` returns ``result.value`` + for every kind including ``notebook_code``, which means an ADF expression + like ``@trim(json(activity('cfg').output.firstRow).notebook_path)`` ends up + as Python source text in ``notebook_path``. Bundle SETUP.md then + mis-documents the source as a workspace path. + + Returns ``(notebook_path, notebook_path_unresolved, raw_expression)``. + When the expression cannot be reduced to a workspace path we set + ``notebook_path_unresolved=True`` so the preparer emits a dispatch-stub + notebook and SETUP.md flags the dynamic dispatch. + """ + if value is None: + return "", False, None + if isinstance(value, dict): + if value.get("type") == "Expression" and "value" in value: + raw_text = str(value["value"]) + result = resolve_expression(value, context) + if result is not None and result.kind in ("literal", "dab_ref"): + return result.value, False, None + return "", True, raw_text + return resolve_field(value, context), False, None + if isinstance(value, str): + if value.startswith("@"): + result = resolve_expression(value, context) + if result is not None and result.kind in ("literal", "dab_ref"): + return result.value, False, None + return "", True, value + return value, False, None + return resolve_field(value, context), False, None + + def _raw_expression_text(value: Any) -> str: """Returns the original ADF expression text from a base_parameter value.""" if isinstance(value, dict) and value.get("type") == "Expression" and "value" in value: return str(value["value"]) return str(value) + + +# Library entry keys that may carry ADF expressions (jar/whl paths, +# maven coordinates with @concat, etc). PyPI uses ``package`` and CRAN +# uses ``package``; we walk all of them through the resolver and only +# emit the entry when every expression resolves to a clean literal/dab_ref. +_LIBRARY_VALUE_KEYS: tuple[str, ...] = ("jar", "whl", "egg", "requirements") + + +_GLOBAL_PARAM_REF_RE = re.compile( + r"pipeline\(\s*\)\.globalParameters\.(\w+)", re.IGNORECASE +) +_PIPELINE_PARAM_REF_RE = re.compile( + r"pipeline\(\s*\)\.parameters\.(\w+)", re.IGNORECASE +) +_VARIABLE_REF_RE = re.compile( + r"variables\(\s*'([^']+)'\s*\)", re.IGNORECASE +) + + +def _extract_missing_identifiers(expression_text: str, context: TranslationContext) -> list[str]: + """Returns identifier names referenced by *expression_text* that aren't + bound in *context*. + + Helps surface concrete root causes in SETUP.md when a library expression + fails to resolve (e.g. ``@concat(...proj4jLibFileName)`` referencing a + global parameter that the factory doesn't declare). + """ + missing: list[str] = [] + for match in _GLOBAL_PARAM_REF_RE.finditer(expression_text): + name = match.group(1) + if context.get_global_parameter(name) is None and name not in missing: + missing.append(name) + for match in _PIPELINE_PARAM_REF_RE.finditer(expression_text): + name = match.group(1) + if name not in missing: + missing.append(name) + for match in _VARIABLE_REF_RE.finditer(expression_text): + name = match.group(1) + if context.get_variable_task_key(name) is None and name not in missing: + missing.append(name) + return missing + + +def _resolve_libraries( + libraries: list[dict[str, Any]] | None, + context: TranslationContext, +) -> tuple[list[dict[str, Any]] | None, list[dict[str, Any]]]: + """Pipes library descriptor values through the expression resolver. + + Library entries whose ``jar``/``whl``/``egg``/``requirements`` value is + an ADF expression that resolves cleanly to a literal get the literal + substituted in place. Entries whose expression is unresolved (e.g. + references a missing globalParameter) are passed through unchanged + so downstream bundler tooling can flag them in SETUP.md. + + C-30 (NB-ITER4-003): also returns a list of unresolved library entries + so the preparer can render an ``Unresolved libraries`` section in + SETUP.md instead of shipping a broken ``@concat(...)`` literal jar path + that the cluster cannot install. + """ + unresolved: list[dict[str, Any]] = [] + if not libraries: + return libraries, unresolved + + resolved: list[dict[str, Any]] = [] + for lib in libraries: + if not isinstance(lib, dict): + resolved.append(lib) + continue + resolved_entry: dict[str, Any] = {} + for key, value in lib.items(): + if key in _LIBRARY_VALUE_KEYS and isinstance(value, (str, dict)): + result = resolve_expression(value, context) + # C-13 (NB-ITER3-004): accept both literal and dab_ref so a + # jar path like @pipeline().parameters.libName collapses to + # {{job.parameters.libName}} (symmetric with custom_tags + # resolution in _resolve_ls_parameters). + if result is not None and result.kind in ("literal", "dab_ref"): + resolved_entry[key] = result.value + else: + expression_text = _raw_expression_text(value) + resolved_entry[key] = value + # Only surface library entries whose value carried an + # ADF expression (starts with ``@``). Bare literal + # paths that already resolved successfully don't need a + # SETUP.md callout. + if isinstance(expression_text, str) and expression_text.startswith("@"): + unresolved.append( + { + "type": key, + "expression": expression_text, + "missing": _extract_missing_identifiers(expression_text, context), + } + ) + else: + resolved_entry[key] = value + resolved.append(resolved_entry) + return resolved, unresolved diff --git a/src/orchestra/translator/activity_translators/resolve.py b/src/orchestra/translator/activity_translators/resolve.py index fd6f496..7f8d434 100644 --- a/src/orchestra/translator/activity_translators/resolve.py +++ b/src/orchestra/translator/activity_translators/resolve.py @@ -2,12 +2,92 @@ from __future__ import annotations +from dataclasses import dataclass, field from typing import Any -from flowx.models.ir import TranslationContext +from flowx.models.ir import ExpressionResult, TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string +@dataclass(slots=True) +class BridgeRequest: + """Carrier for a notebook_code expression that must run as a bridge task. + + C-07 (CF-iter2-001 / CF-iter2-003 / VAREX-003): when an operand of an + IfCondition / Switch condition_task resolves to ``notebook_code`` (e.g. + ``@empty(...)``, ``@toUpper(coalesce(...))``), the preparer must + synthesise a hidden SetVariable task ahead of the condition and + rewrite the operand to point at the bridge task's value. Returning a + structured request keeps the translator free of preparer-side + concerns. + """ + + notebook_code: str + notebook_imports: list[str] = field(default_factory=list) + required_parameters: dict[str, str] = field(default_factory=dict) + + +def lower_to_bridge(value: Any, context: TranslationContext) -> tuple[str | None, BridgeRequest | None]: + """Lowers *value* to a condition-task-safe operand or a BridgeRequest. + + Returns: + ``(operand_value, bridge_request)`` -- exactly one of which is + populated. When ``operand_value`` is a string, it's a literal + or ``{{...}}`` DAB ref that can be dropped directly into + ``condition_task.left`` / ``.right``. When ``bridge_request`` is + populated, the caller must emit a hidden SetVariable task that + runs the notebook code and reference its task value in the + operand. When both are None the expression is unresolvable. + """ + if value is None: + return None, None + result = resolve_expression(value, context) + if result is None: + return None, None + if result.kind in ("literal", "dab_ref"): + return result.value, None + if result.kind == "notebook_code": + return None, BridgeRequest( + notebook_code=result.value, + notebook_imports=list(result.imports), + required_parameters=dict(result.required_parameters), + ) + return None, None + + +def merge_bridge_requests(*requests: BridgeRequest | None) -> BridgeRequest | None: + """Combines several bridge requests into one merged Python expression. + + The bridges are joined with ``and`` so call-sites that compose two + operand-level bridges (e.g. ``@and(empty(X), empty(Y))``) produce a + single bridge task with a boolean truthiness result. Returns ``None`` + when no non-None requests are supplied. + """ + populated = [r for r in requests if r is not None] + if not populated: + return None + if len(populated) == 1: + return populated[0] + expression = " and ".join(f"({r.notebook_code})" for r in populated) + imports: list[str] = [] + required: dict[str, str] = {} + for req in populated: + for imp in req.notebook_imports: + if imp not in imports: + imports.append(imp) + required.update(req.required_parameters) + return BridgeRequest( + notebook_code=expression, + notebook_imports=imports, + required_parameters=required, + ) + + +def expression_kind(value: Any, context: TranslationContext) -> ExpressionResult | None: + """Convenience wrapper that returns the raw ExpressionResult for *value*.""" + return resolve_expression(value, context) + + def resolve_field(value: Any, context: TranslationContext) -> str: """Resolves a field value that may contain an ADF expression. diff --git a/src/orchestra/translator/activity_translators/set_variable.py b/src/orchestra/translator/activity_translators/set_variable.py index 6ddc3e1..6f127ea 100644 --- a/src/orchestra/translator/activity_translators/set_variable.py +++ b/src/orchestra/translator/activity_translators/set_variable.py @@ -9,6 +9,51 @@ from flowx.parser.expression_parser import resolve_expression +def _unwrap_return_value_pairs(value: Any) -> Any: + """Unwrap a Set Pipeline Return Value list-of-pairs to a resolvable value. + + A ``pipelineReturnValue`` value is shaped as a list of + ``{'key': ..., 'value': }`` + entries. ADF's expression dicts here use the ``content`` key (not + ``value``). We normalise a single pair's inner value into the + ``{'type': 'Expression', 'value': ...}`` shape (or a bare literal) that + :func:`resolve_expression` understands, so the inner ``@variables('X')`` + reference is preserved instead of stringifying the whole list. + + When the list is empty or carries more than one pair (no single + canonical result), the original value is returned unchanged so the + legacy unresolved/blanking path still applies. + """ + if not isinstance(value, list) or len(value) != 1: + return value + entry = value[0] + if not isinstance(entry, dict) or "value" not in entry: + return value + inner = entry["value"] + if isinstance(inner, dict) and inner.get("type") == "Expression" and "content" in inner: + return {"type": "Expression", "value": inner["content"]} + if isinstance(inner, dict) and inner.get("type") == "Expression" and "value" in inner: + return inner + return inner + + +def _raw_expression_text(value: Any) -> str: + """Returns the original ADF expression text for *value*.""" + if isinstance(value, dict) and value.get("type") == "Expression" and "value" in value: + return str(value["value"]) + return "" if value is None else str(value) + + +def _is_adf_expression(value: Any) -> bool: + """Returns True when *value* is an ADF expression we can't pass through verbatim.""" + if isinstance(value, dict) and value.get("type") == "Expression" and "value" in value: + inner = value["value"] + return isinstance(inner, str) and inner.startswith("@") + if isinstance(value, str): + return value.startswith("@") + return False + + def translate( activity: AdfActivity, base_kwargs: dict[str, Any], @@ -32,21 +77,49 @@ def translate( variable_name = type_properties.get("variableName", "") value_raw = type_properties.get("value", "") + # C-42 (VAREX5-001): a Set Pipeline Return Value activity carries a + # list of {key, value} pairs (e.g. + # [{'key': 'result', 'value': {'type': 'Expression', + # 'content': "@variables('executionOutputs')"}}]). The legacy path + # fails _is_adf_expression and stringifies the whole list, which the + # bundler then blanks. The inner expression is resolvable, so unwrap a + # single pair's value and route it through the normal resolution + # pipeline instead of losing the reference. + value_raw = _unwrap_return_value_pairs(value_raw) + expr_result = resolve_expression(value_raw, context) required_parameters: dict[str, str] = {} + raw_expression_text = _raw_expression_text(value_raw) if expr_result is not None: variable_value = expr_result.value value_kind = expr_result.kind notebook_code = expr_result.value if expr_result.kind == "notebook_code" else None notebook_imports = list(expr_result.imports) if expr_result.kind == "notebook_code" else [] required_parameters = dict(expr_result.required_parameters) + elif _is_adf_expression(value_raw): + # C-33 (VAREX4-001 / CF4-003): when the value is an ADF expression + # the resolver couldn't handle (e.g. a nested function call we + # don't model), do NOT stamp value_kind='literal' with the raw + # @concat text — that ships uninterpretable Python source through + # SETUP.md. Blank the value and mark it unresolved so the bundler + # emits a manual_variable_init SetupTask the user can act on. + variable_value = "" + value_kind = "unresolved" + notebook_code = None + notebook_imports = [] else: # Fallback: unwrap expression-type dicts to at least preserve the string if isinstance(value_raw, dict) and value_raw.get("type") == "Expression": variable_value = value_raw.get("value", "") elif isinstance(value_raw, str): variable_value = value_raw + elif isinstance(value_raw, bool): + # VAREX3-002: render Python bool as lowercase 'true'/'false' so + # downstream ADF comparisons like @equals(variables('X'), true) + # match consistently. ``str(True)`` would emit 'True' and silently + # invert the comparison. + variable_value = "true" if value_raw else "false" else: variable_value = str(value_raw) value_kind = "literal" @@ -61,6 +134,7 @@ def translate( notebook_code=notebook_code, notebook_imports=notebook_imports, required_parameters=required_parameters, + raw_expression=raw_expression_text if value_kind == "unresolved" else None, ) # Register variable -> task_key mapping in context. diff --git a/src/orchestra/translator/activity_translators/switch.py b/src/orchestra/translator/activity_translators/switch.py index 095840d..1ce80db 100644 --- a/src/orchestra/translator/activity_translators/switch.py +++ b/src/orchestra/translator/activity_translators/switch.py @@ -7,29 +7,47 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, SwitchActivity, SwitchCase, TranslationContext from flowx.parser.adf_loader import parse_activity -from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string -from flowx.translator.activity_translators.resolve import resolve_field +from flowx.parser.expression_parser import resolve_interpolated_string +from flowx.translator.activity_translators.resolve import ( + BridgeRequest, + lower_to_bridge, + resolve_field, +) +_BRIDGE_PLACEHOLDER = "__BRIDGE__::result" -def _resolve_on_expression(on_expression: str, context: TranslationContext) -> str: - """Resolves the ``on`` expression to a DAB dynamic value ref. + +def _resolve_on_expression( + on_expression: str, context: TranslationContext +) -> tuple[str, BridgeRequest | None]: + """Resolves the ``on`` expression to a DAB dynamic value ref or a bridge request. + + C-07 (CF-iter2-001 / CF-iter2-003): when the expression involves an + ADF function call (e.g. ``@toUpper(coalesce(...))``), lower it to a + bridge SetVariable task instead of shipping the raw ADF string into + the condition_task operand. Args: on_expression: Raw ADF on-expression string. context: Translation context for resolving variables. Returns: - Resolved DAB ref string, or the original if unresolvable. + Tuple of ``(resolved_value_or_placeholder, bridge_request_or_None)``. """ if "@{" in on_expression: - return resolve_interpolated_string(on_expression, context) + return resolve_interpolated_string(on_expression, context), None if on_expression.startswith("@"): - result = resolve_expression(on_expression, context) - if result is not None and result.kind in ("dab_ref", "literal"): - return result.value + operand, bridge = lower_to_bridge(on_expression, context) + if operand is not None: + return operand, None + if bridge is not None: + return _BRIDGE_PLACEHOLDER, bridge + # Resolution failed entirely -- preserve the raw string so the + # preparer can flag it via SETUP.md. + return on_expression, None - return on_expression + return on_expression, None def translate( @@ -63,7 +81,7 @@ def translate( else: on_expression_raw = str(on_raw) if on_raw else "" - on_expression = _resolve_on_expression(on_expression_raw, context) + on_expression, bridge = _resolve_on_expression(on_expression_raw, context) cases: list[SwitchCase] = [] raw_cases = type_properties.get("cases", []) @@ -93,11 +111,20 @@ def translate( definitions, ) + bridge_kwargs: dict[str, Any] = {} + if bridge is not None: + bridge_kwargs = { + "bridge_notebook_code": bridge.notebook_code, + "bridge_notebook_imports": list(bridge.notebook_imports), + "bridge_required_parameters": dict(bridge.required_parameters), + } + switch_activity = SwitchActivity( **base_kwargs, on_expression=on_expression, cases=cases, default_activities=default_activities, + **bridge_kwargs, ) return switch_activity, context diff --git a/src/orchestra/translator/engine.py b/src/orchestra/translator/engine.py index aa217cb..69981f3 100644 --- a/src/orchestra/translator/engine.py +++ b/src/orchestra/translator/engine.py @@ -8,6 +8,7 @@ import re from collections import defaultdict from dataclasses import asdict +from datetime import datetime from pathlib import Path from types import MappingProxyType from typing import Any, Callable @@ -122,13 +123,40 @@ def translate_pipeline( activity_cache=MappingProxyType({}), registry=MappingProxyType(TRANSLATOR_REGISTRY), variable_cache=MappingProxyType({}), + global_parameters=MappingProxyType(dict(definitions.global_parameters)), ) + # C-41 (CF5-001): seed declared variable types so the IfCondition + # fallback can recognise Boolean variables that are backed only by a + # literal default init task (and thus never populate + # variable_value_cache). Without this a `continue`-style Boolean + # condition emits NOT_EQUAL(left, '0'), always true for a + # 'true'/'false' string, making the false branch dead code. + if pipeline.variables: + default_literals: dict[str, str] = {} + for name, var in pipeline.variables.items(): + default = var.default_value + if isinstance(default, bool): + default_literals[name] = "true" if default else "false" + elif isinstance(default, str) and default.lower() in ("true", "false"): + default_literals[name] = default.lower() + context = context.with_variable_types( + {name: var.type for name, var in pipeline.variables.items()}, + default_literals=default_literals, + ) + gaps: list[AgenticGap] = [] warnings: list[str] = [] + # C-05 (VAREX-002): synthesise init SetVariable tasks for pipeline + # variables carrying a defaultValue. This seeds variable_cache so + # downstream @variables('X') references resolve to the init task's + # value reference instead of falling back to a self-referential + # {{tasks.X.values.X}} dangler. + init_variable_activities, context = _build_variable_init_activities(pipeline, context) + translated_activities: list[Activity] = list(init_variable_activities) + sorted_activities = _topological_visit(pipeline.activities) - translated_activities: list[Activity] = [] deterministic_count = 0 agentic_count = 0 unsupported_count = 0 @@ -162,18 +190,22 @@ def translate_pipeline( ) warnings.append(f"Activity '{adf_activity.name}' (type={adf_activity.type}) has no translation path.") - parameters: dict[str, Any] = {} + parameter_entries: list[dict[str, Any]] = [] if pipeline.parameters: for param_name, param_def in pipeline.parameters.items(): - parameters[param_name] = param_def.default_value + entry: dict[str, Any] = {"name": param_name, "type": param_def.type} + if param_def.default_value is not None: + entry["default"] = _coerce_parameter_default(param_def.default_value, param_def.type) + parameter_entries.append(entry) + + schedule = _compile_pipeline_schedule(pipeline, definitions) pipeline_ir = Pipeline( name=pipeline.name, - parameters=[{"name": param_name, "default": param_value} for param_name, param_value in parameters.items()] - if parameters - else None, + parameters=parameter_entries or None, tasks=translated_activities, tags={"source": "adf", "pipeline": pipeline.name}, + schedule=schedule, ) # Whole-IR expression rewrite: catches @{...} tokens the per-activity @@ -254,7 +286,7 @@ def _dispatch_activity( Returns: Tuple of ``(translated_activity, updated_context)``. """ - base_kwargs = _build_base_kwargs(activity, definitions) + base_kwargs = _build_base_kwargs(activity, definitions, context=context) match activity.type: case "ForEach": @@ -353,6 +385,424 @@ def _translate_activity_list( return results, context +# C-10 (SCHED-001): map Windows timezone names ADF emits onto IANA names +# the Databricks DAB ``schedule.timezone_id`` field expects. Only the +# ones observed in the corpus are mapped explicitly; anything else passes +# through unchanged (Databricks accepts any IANA zone). +_ADF_TIMEZONE_TO_IANA: dict[str, str] = { + "UTC": "UTC", + "Coordinated Universal Time": "UTC", + "Romance Standard Time": "Europe/Madrid", + "Central Europe Standard Time": "Europe/Budapest", + "Central European Standard Time": "Europe/Warsaw", + "W. Europe Standard Time": "Europe/Berlin", + "GMT Standard Time": "Europe/London", + "Eastern Standard Time": "America/New_York", + "Central Standard Time": "America/Chicago", + "Pacific Standard Time": "America/Los_Angeles", + "Mountain Standard Time": "America/Denver", + "Tokyo Standard Time": "Asia/Tokyo", + "China Standard Time": "Asia/Shanghai", + "India Standard Time": "Asia/Kolkata", + "AUS Eastern Standard Time": "Australia/Sydney", +} + +_DAYS_OF_WEEK_MAP: dict[str, str] = { + "Sunday": "SUN", + "Monday": "MON", + "Tuesday": "TUE", + "Wednesday": "WED", + "Thursday": "THU", + "Friday": "FRI", + "Saturday": "SAT", +} + + +def _compile_pipeline_schedule( + pipeline: AdfPipeline, + definitions: AdfDefinitions, +) -> dict[str, Any] | None: + """Compiles the first matching ADF trigger into a Pipeline.schedule dict. + + C-10 (SCHED-001): translates ScheduleTrigger recurrence into a + quartz_cron_expression + timezone_id pair the DAB writer can emit + as the job's ``schedule:`` block. BlobEventsTrigger maps to a + ``trigger.file_arrival`` spec. TumblingWindowTrigger and + CustomEventsTrigger are best-effort: they emit a SETUP-style hint + so the user can finish wiring them manually. + """ + triggers = getattr(definitions, "triggers", None) or [] + pipeline_name = pipeline.name + matching_triggers = [t for t in triggers if _trigger_references(t, pipeline_name)] + if not matching_triggers: + return None + + # First matching trigger wins -- ADF allows multiple triggers per + # pipeline but DAB schedules are 1:1. Subsequent triggers can be + # surfaced via SETUP.md by downstream tooling. + trigger = matching_triggers[0] + spec = _adf_trigger_to_schedule(trigger) + if spec is not None: + # SCHED3-003: pull per-pipeline parameter overrides off the + # matching pipelineReference so trigger-injected params (e.g. + # ``{applicationName: 'app0001', negocio: 'GLP'}``) propagate to + # the job's default parameter values. + overrides = _extract_trigger_parameter_overrides(trigger, pipeline_name) + if overrides: + spec["parameter_overrides"] = overrides + return spec + + +def _trigger_references(trigger: Any, pipeline_name: str) -> bool: + """Returns True if *trigger* references the named pipeline.""" + refs = trigger.pipelines or [] + for ref in refs: + if not isinstance(ref, dict): + continue + pipeline_ref = ref.get("pipelineReference") or {} + if isinstance(pipeline_ref, dict) and pipeline_ref.get("referenceName") == pipeline_name: + return True + return False + + +def _extract_trigger_parameter_overrides( + trigger: Any, pipeline_name: str +) -> dict[str, Any]: + """Returns the parameters block on the trigger's pipelineReference entry. + + SCHED3-003: ADF triggers attach per-pipeline parameter overrides at the + ``triggers[].pipelines[].parameters`` level so scheduled runs receive + deterministic values for pipeline parameters. Without surfacing them, + scheduled invocations would receive the pipeline parameter defaults + only. + """ + refs = trigger.pipelines or [] + for ref in refs: + if not isinstance(ref, dict): + continue + pipeline_ref = ref.get("pipelineReference") or {} + if not isinstance(pipeline_ref, dict): + continue + if pipeline_ref.get("referenceName") != pipeline_name: + continue + params = ref.get("parameters") or {} + if isinstance(params, dict) and params: + return dict(params) + return {} + + +def _adf_trigger_to_schedule(trigger: Any) -> dict[str, Any] | None: + """Compiles an :class:`AdfTrigger` into a Pipeline.schedule spec dict.""" + props = trigger.properties or {} + type_properties = props.get("typeProperties") or {} + runtime_state = props.get("runtimeState", "Started") + pause_status = "PAUSED" if runtime_state == "Stopped" else "UNPAUSED" + + trigger_type = trigger.type + if trigger_type == "ScheduleTrigger": + recurrence = type_properties.get("recurrence") or {} + # SCHED3-002: Day/Week/Month with interval > 1 cannot be represented + # in quartz cron without enumerating every Nth occurrence; use the + # trigger.periodic primitive so it ships correctly. + periodic = _recurrence_to_periodic(recurrence) + if periodic is not None: + spec: dict[str, Any] = { + "kind": "periodic", + "interval": periodic["interval"], + "unit": periodic["unit"], + "pause_status": pause_status, + } + # C-36 (SCHED4-001): forward the captured time-of-day so the + # bundler can flag it in SETUP.md. + if "time_of_day_note" in periodic: + spec["time_of_day_note"] = periodic["time_of_day_note"] + return spec + # C-45 (SCHED5-002): an interval > 1 Month recurrence has no + # monthly-cron-expressible form (cron fires every month, ignoring the + # interval) and the DAB periodic enum has no MONTHS unit, so surface a + # manual setup note instead of silently emitting a monthly cron. + if _is_multi_month_recurrence(recurrence): + return { + "kind": "manual_setup", + "trigger_type": "ScheduleTrigger", + "pause_status": pause_status, + "note": ( + "Month-frequency trigger with interval > 1 has no DAB " + "equivalent (PeriodicTriggerConfigurationTimeUnit lacks " + "MONTHS and quartz cron cannot encode every-Nth-month). " + "Configure the schedule manually." + ), + } + cron = _recurrence_to_quartz_cron(recurrence) + if cron is None: + return None + timezone_id = _normalize_timezone(recurrence.get("timeZone")) + spec = { + "kind": "schedule", + "quartz_cron_expression": cron, + "timezone_id": timezone_id, + "pause_status": pause_status, + } + return spec + if trigger_type == "TumblingWindowTrigger": + # Approximate as a periodic schedule -- the user should review. + frequency = type_properties.get("frequency", "Hour") + interval = type_properties.get("interval", 1) + spec = { + "kind": "schedule", + "tumbling": True, + "frequency": frequency, + "interval": interval, + "pause_status": pause_status, + "note": "Approximated from TumblingWindowTrigger; review window boundaries.", + } + return spec + if trigger_type == "BlobEventsTrigger": + scope = type_properties.get("scope", "") + events = type_properties.get("events") or [] + spec = { + "kind": "file_arrival", + "url": scope, + "events": list(events), + "pause_status": pause_status, + } + return spec + if trigger_type == "CustomEventsTrigger": + spec = { + "kind": "manual_setup", + "trigger_type": "CustomEventsTrigger", + "pause_status": pause_status, + "note": "CustomEventsTrigger has no direct DAB equivalent; configure in SETUP.md.", + } + return spec + return None + + +def _recurrence_to_periodic(recurrence: dict[str, Any]) -> dict[str, Any] | None: + """Returns a {interval, unit} dict when the recurrence requires periodic. + + SCHED3-002: Day/Week/Month with ``interval > 1`` cannot be modelled in + quartz cron without enumerating every Nth occurrence. The DAB + ``trigger.periodic`` primitive accepts ``{interval, unit}`` directly, + so we emit a periodic spec instead. Minute/Hour with interval > 1 are + expressible in cron (``0/N``) so we still leave them to the cron path. + + C-36 (SCHED4-001): when the recurrence carries a non-empty + ``schedule`` block (hours / minutes / weekDays / monthDays), the + ``trigger.periodic`` primitive can't encode the time-of-day so the + schedule silently fires at midnight instead. Capture the original + schedule on a ``time_of_day_note`` field so the bundler can emit a + ``manual_schedule_time_of_day`` SetupTask (SETUP.md). + """ + frequency = recurrence.get("frequency") + interval = recurrence.get("interval", 1) + if isinstance(interval, str) and interval.isdigit(): + interval = int(interval) + if not isinstance(interval, int) or interval <= 1: + return None + # C-45 (SCHED5-002): the DAB PeriodicTriggerConfigurationTimeUnit enum + # only defines DAYS / HOURS / WEEKS — emitting MONTHS makes bundle + # validate/deploy reject the trigger. Month frequencies are routed to + # the quartz cron path (monthDays) instead; an interval > 1 Month, which + # is not monthly-cron-expressible, is surfaced as a setup note by the + # caller. + unit_map = {"Day": "DAYS", "Week": "WEEKS"} + unit = unit_map.get(frequency or "") + if unit is None: + return None + spec: dict[str, Any] = {"interval": interval, "unit": unit} + schedule = recurrence.get("schedule") or {} + if isinstance(schedule, dict): + time_of_day = { + key: schedule.get(key) + for key in ("hours", "minutes", "weekDays", "monthDays") + if schedule.get(key) + } + if time_of_day: + spec["time_of_day_note"] = time_of_day + return spec + + +def _is_multi_month_recurrence(recurrence: dict[str, Any]) -> bool: + """Returns True for a Month-frequency recurrence with ``interval > 1``. + + C-45 (SCHED5-002): these triggers cannot ship as either a periodic spec + (no MONTHS unit in the DAB enum) or a quartz cron (cron has no + every-Nth-month form), so the caller emits a manual setup note. + """ + if recurrence.get("frequency") != "Month": + return False + interval = recurrence.get("interval", 1) + if isinstance(interval, str) and interval.isdigit(): + interval = int(interval) + return isinstance(interval, int) and interval > 1 + + +def _recurrence_to_quartz_cron(recurrence: dict[str, Any]) -> str | None: + """Compiles a ScheduleTrigger recurrence block into a quartz cron expression. + + ADF recurrence has ``frequency`` + ``interval`` + ``schedule``. The + quartz format expected by DAB is + ``second minute hour day-of-month month day-of-week``. + """ + frequency = recurrence.get("frequency") + interval = recurrence.get("interval", 1) + schedule = recurrence.get("schedule") or {} + minutes = schedule.get("minutes") + hours = schedule.get("hours") + week_days = schedule.get("weekDays") or [] + month_days = schedule.get("monthDays") or [] + + # C-44 (SCHED5-001): when the schedule block carries no explicit + # time-of-day, ADF defaults it to the first-execution time derived from + # ``startTime``. Reading only ``schedule.minutes/hours`` (falling back + # to '0'/'0') silently shifts a ``startTime`` of 21:00 to midnight. + # Derive the hour/minute from ``startTime`` so the cron fires at the + # ADF-intended time. + start_hour, start_minute = _start_time_hour_minute(recurrence.get("startTime")) + minute_default = str(start_minute) if start_minute is not None else "0" + hour_default = str(start_hour) if start_hour is not None else "0" + + minute_field = _list_or_default(minutes, minute_default) + hour_field = _list_or_default(hours, hour_default) + if isinstance(interval, str) and interval.isdigit(): + interval = int(interval) + + if frequency == "Minute": + if not isinstance(interval, int) or interval <= 0: + interval = 1 + return f"0 0/{interval} * * * ?" + if frequency == "Hour": + if not isinstance(interval, int) or interval <= 0: + interval = 1 + return f"0 {minute_field} 0/{interval} * * ?" + if frequency == "Day": + return f"0 {minute_field} {hour_field} * * ?" + if frequency == "Week": + days = ",".join(_DAYS_OF_WEEK_MAP.get(d, d) for d in week_days) or "MON" + return f"0 {minute_field} {hour_field} ? * {days}" + if frequency == "Month": + dom_field = _list_or_default(month_days, "1") + return f"0 {minute_field} {hour_field} {dom_field} * ?" + return None + + +def _list_or_default(value: Any, default: str) -> str: + """Renders a recurrence list/scalar as a cron-field string.""" + if value is None: + return default + if isinstance(value, list): + if not value: + return default + return ",".join(str(v) for v in value) + return str(value) + + +def _start_time_hour_minute(start_time: Any) -> tuple[int | None, int | None]: + """Parses an ISO 8601 ``startTime`` into ``(hour, minute)``. + + C-44 (SCHED5-001): ADF uses the trigger's first-execution time (from + ``startTime``) as the default time-of-day when the recurrence carries no + explicit ``schedule.hours/minutes``. Returns ``(None, None)`` when the + value is missing or unparseable so the caller keeps the midnight + fallback. + """ + if not isinstance(start_time, str) or not start_time.strip(): + return None, None + value = start_time.strip() + # ``datetime.fromisoformat`` rejects a trailing 'Z' before 3.11; map it + # to the explicit UTC offset so older interpreters parse it too. + if value.endswith("Z"): + value = value[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return None, None + return parsed.hour, parsed.minute + + +def _normalize_timezone(tz: Any) -> str: + """Maps an ADF timezone string to an IANA zone the DAB writer accepts.""" + if not tz or not isinstance(tz, str): + return "UTC" + return _ADF_TIMEZONE_TO_IANA.get(tz, tz) + + +def _build_variable_init_activities( + pipeline: AdfPipeline, + context: TranslationContext, +) -> tuple[list[Activity], TranslationContext]: + """Synthesise init SetVariable IR tasks for variables carrying defaultValue. + + C-05 (VAREX-002): without an explicit ADF SetVariable activity, a + variable's defaultValue is never materialised, so downstream + ``@variables('X')`` references fall back to a dangling + ``{{tasks.X.values.X}}`` reference. This helper emits an init task + per default-valued variable so the variable_cache carries a real + setter task_key. + """ + from flowx.parser.expression_parser import resolve_expression + + if not pipeline.variables: + return [], context + + init_tasks: list[Activity] = [] + for var_name, var_def in pipeline.variables.items(): + default = var_def.default_value + if default is None: + return_default = False + else: + return_default = True + if not return_default: + continue + task_key = f"_init_{_sanitize_task_key(var_name)}" + expr_result = resolve_expression(default, context) + if expr_result is not None: + variable_value = expr_result.value + value_kind = expr_result.kind + notebook_code = expr_result.value if expr_result.kind == "notebook_code" else None + notebook_imports = list(expr_result.imports) if expr_result.kind == "notebook_code" else [] + required_parameters = dict(expr_result.required_parameters) + else: + # VAREX3-002: Boolean defaults must render lowercase ('true'/'false') + # so downstream ``@equals(variables('continue'), true)`` evaluates + # consistently with ADF semantics. Python ``str(True)`` would + # produce title-case 'True' and silently invert the comparison. + if isinstance(default, bool): + variable_value = "true" if default else "false" + else: + variable_value = str(default) if not isinstance(default, str) else default + value_kind = "literal" + notebook_code = None + notebook_imports = [] + required_parameters = {} + + init_activity = SetVariableActivity( + name=f"_init_{var_name}", + task_key=task_key, + description=None, + timeout_seconds=None, + max_retries=None, + min_retry_interval_millis=None, + depends_on=None, + cluster=None, + variable_name=var_name, + variable_value=variable_value, + value_kind=value_kind, + notebook_code=notebook_code, + notebook_imports=notebook_imports, + required_parameters=required_parameters, + ) + init_tasks.append(init_activity) + # Register the synthesised setter so @variables('X') resolves to + # {{tasks._init_X.values.X}}. When the value is itself a DAB ref + # (e.g. from @utcNow()), inline it directly per existing semantics. + dab_ref_value = variable_value if value_kind == "dab_ref" else None + context = context.with_variable(var_name, task_key, dab_ref_value=dab_ref_value) + context = context.with_activity(init_activity.name, init_activity) + return init_tasks, context + + def _topological_visit(activities: list[AdfActivity]) -> list[AdfActivity]: """Return activities in dependency-first (topological) order. @@ -402,7 +852,12 @@ def _topological_visit(activities: list[AdfActivity]) -> list[AdfActivity]: _TIMEOUT_RE = re.compile(r"(?:(\d+)\.)?(\d{2}):(\d{2}):(\d{2})") -def _build_base_kwargs(activity: AdfActivity, definitions: AdfDefinitions) -> dict[str, Any]: +def _build_base_kwargs( + activity: AdfActivity, + definitions: AdfDefinitions, + *, + context: TranslationContext | None = None, +) -> dict[str, Any]: """Extracts common fields shared by all Activity IR subclasses. Args: @@ -431,7 +886,7 @@ def _build_base_kwargs(activity: AdfActivity, definitions: AdfDefinitions) -> di if activity.depends_on: depends_on = [] for dependency in activity.depends_on: - outcome = dependency.dependency_conditions[0] if dependency.dependency_conditions else None + outcome = _map_dependency_conditions(dependency.dependency_conditions) depends_on.append( Dependency( task_key=_sanitize_task_key(dependency.activity), @@ -445,7 +900,15 @@ def _build_base_kwargs(activity: AdfActivity, definitions: AdfDefinitions) -> di linked_service_name = activity.linked_service_name.reference_name linked_service_def = definitions.linked_services.get(linked_service_name) if linked_service_def: - cluster = _extract_cluster_config(linked_service_def.properties) + ls_param_overrides = _resolve_ls_parameters( + linked_service_def.properties, + activity.linked_service_name.parameters, + context=context, + ) + cluster = _extract_cluster_config( + linked_service_def.properties, + ls_param_overrides, + ) if cluster: existing_cluster_id = cluster.get("existing_cluster_id") @@ -477,6 +940,46 @@ def _sanitize_task_key(name: str) -> str: return key or "unnamed" +def _map_dependency_conditions(conditions: list[str] | None) -> str | None: + """Map an ADF dependsOn[].dependencyConditions list to a single outcome. + + ADF accepts multiple conditions on a single edge — e.g. + ``['Succeeded', 'Failed']`` means "run regardless of upstream + success/failure". Databricks Workflows encodes the same semantics + by combining ``run_if`` and per-edge outcomes; the encoding that + propagates correctly through this codebase is to pick a single + representative outcome that the downstream + ``run_if_from_adf_outcomes`` reducer can interpret. + + Mapping rules (single condition): + Succeeded -> "Succeeded" + Failed -> "Failed" + Completed -> "Completed" (run regardless of upstream result) + Skipped -> "Skipped" + + Mapping rules (multi): + Any list that includes ``Failed`` AND ``Succeeded`` -> "Completed" + Any list that includes ``Skipped`` -> "Skipped" + Multi-element list including ``Failed`` only -> "Failed" + Anything else -> first item + """ + if not conditions: + return None + normalized = [c for c in conditions if c] + if not normalized: + return None + if len(normalized) == 1: + return normalized[0] + cset = set(normalized) + if "Failed" in cset and "Succeeded" in cset: + return "Completed" + if "Skipped" in cset: + return "Skipped" + if "Failed" in cset: + return "Failed" + return normalized[0] + + def _parse_adf_timeout(timeout_str: str) -> int | None: """Parses an ADF timeout string to total seconds. @@ -496,16 +999,183 @@ def _parse_adf_timeout(timeout_str: str) -> int | None: return days * 86400 + hours * 3600 + minutes * 60 + seconds -def _extract_cluster_config(ls_properties: dict[str, Any]) -> dict[str, Any] | None: +def _resolve_ls_parameters( + ls_properties: dict[str, Any], + activity_supplied: dict[str, Any] | None, + context: TranslationContext | None = None, +) -> dict[str, Any]: + """Builds the effective LS parameter map for cluster-config resolution. + + Args: + ls_properties: Full properties bag from the linked service JSON. + activity_supplied: Per-activity parameter overrides from the + ``linkedServiceName.parameters`` block (may be ``None``). + context: Translation context used to resolve ``@``-prefixed + activity-supplied values against factory global parameters. + When omitted, ADF expressions are left as raw strings. + + Returns: + Mapping of parameter name -> resolved value. Activity-supplied + overrides win over LS defaultValue. Wrapped ``{"value": ..., "type": + "Expression"}`` dicts are unwrapped and (when ``context`` is set) + passed through :func:`resolve_expression` so ``@pipeline(). + globalParameters.X`` collapses to the factory value. + """ + from flowx.parser.expression_parser import resolve_expression + + resolved: dict[str, Any] = {} + declared = ls_properties.get("parameters") or {} + if isinstance(declared, dict): + for pname, pdef in declared.items(): + if isinstance(pdef, dict) and "defaultValue" in pdef: + resolved[pname] = _unwrap_expression_value(pdef["defaultValue"]) + if isinstance(activity_supplied, dict): + for pname, pval in activity_supplied.items(): + raw = _unwrap_expression_value(pval) + # C-03: route @-prefixed activity-supplied values through the + # expression parser so @pipeline().globalParameters.X collapses + # to the factory value when one is set. + if context is not None and isinstance(raw, str) and raw.startswith("@"): + result = resolve_expression(raw, context) + # C-13 (NB-ITER3-002 / LSC3-003 / VAREX3-006): accept both + # literal and dab_ref so @pipeline().parameters.X collapses + # to {{job.parameters.X}} (valid in custom_tags map values). + if result is not None and result.kind in ("literal", "dab_ref"): + raw = result.value + resolved[pname] = raw + return resolved + + +def _unwrap_expression_value(value: Any) -> Any: + """Unwrap a ``{"value": ..., "type": "Expression"}`` ADF dict-wrapper. + + C-02 (NB-ITER2-2 / LSC2-003): activity-supplied LS parameter values and + LS-derived cluster fields (``custom_tags``, ``spark_env_vars`` entries, + ...) sometimes ship as the ADF expression-dict shape. Databricks + cluster YAML rejects nested dicts in ``custom_tags``; recursive unwrap + flattens them to scalars while leaving regular dicts untouched. + """ + if isinstance(value, dict): + # Bare {"value": X, "type": "Expression"} -- collapse to inner X. + if "value" in value and value.get("type") == "Expression": + return _unwrap_expression_value(value["value"]) + # Some payloads omit the explicit type marker but follow the same + # single-key shape. Conservatively unwrap only when the dict has + # the exact two keys {"value", "type"} so we don't corrupt regular + # nested config blocks like {"workspace": {"destination": ...}}. + if set(value.keys()) == {"value", "type"}: + return _unwrap_expression_value(value["value"]) + return {k: _unwrap_expression_value(v) for k, v in value.items()} + if isinstance(value, list): + return [_unwrap_expression_value(v) for v in value] + return value + + +_LS_PARAM_REF_RE = re.compile(r"@linkedService\(\s*\)\.(\w+)", re.IGNORECASE) + + +def _substitute_ls_params(value: Any, params: dict[str, Any]) -> Any: + """Replaces ``@linkedService().X`` tokens in *value* with bound params. + + Handles scalar strings and nested dicts/lists. Returns the value + unchanged when no substitution is possible. + """ + if isinstance(value, str): + if "@linkedService()" not in value: + return value + # Full-string single-reference: drop in the resolved value with its + # original type so e.g. integer params don't get stringified. + full = _LS_PARAM_REF_RE.fullmatch(value) + if full is not None: + name = full.group(1) + if name in params: + return params[name] + return value + + def _sub(match: re.Match[str]) -> str: + name = match.group(1) + if name in params: + return str(params[name]) + return match.group(0) + + return _LS_PARAM_REF_RE.sub(_sub, value) + if isinstance(value, dict): + return {k: _substitute_ls_params(v, params) for k, v in value.items()} + if isinstance(value, list): + return [_substitute_ls_params(v, params) for v in value] + return value + + +def _coerce_int(value: Any) -> Any: + """Coerce numeric strings (``"1"``) to ``int``; leave other values alone.""" + if isinstance(value, bool): + return value + if isinstance(value, int): + return value + try: + return int(value) + except (TypeError, ValueError): + return value + + +def _coerce_parameter_default(value: Any, declared_type: str) -> Any: + """Coerce an ADF parameter default into a Python type matching its declared type. + + The declared type (``"Bool"`` / ``"Int"`` / ``"Float"`` / ``"String"`` / + ``"Array"`` / ``"Object"``) is what ADF stores in the pipeline JSON. + Defaults round-trip as strings through JSON, so a Bool parameter with + default ``false`` arrives as the literal ``"False"``. The fix re-types + each default per the declared type so the emitted YAML carries a real + bool / int / float, not a quoted string. + """ + t = (declared_type or "String").lower() + if t in ("bool", "boolean"): + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in ("true", "false"): + return lowered == "true" + return value + if t in ("int", "integer"): + return _coerce_int(value) + if t == "float": + try: + return float(value) + except (TypeError, ValueError): + return value + return value + + +def _extract_cluster_config( + ls_properties: dict[str, Any], + ls_param_overrides: dict[str, Any] | None = None, +) -> dict[str, Any] | None: """Extracts Databricks cluster configuration from a linked-service properties dict. Args: ls_properties: Full properties bag from the linked service JSON. + ls_param_overrides: Optional map of ``@linkedService().X`` -> value + overrides to substitute before extraction. When supplied, + every string value in the LS payload is rewritten through + ``_substitute_ls_params`` so cluster fields like + ``newClusterVersion: '@linkedService().clusterVersion'`` + resolve to a real Spark version string. Returns: Cluster configuration dict, or ``None`` if no Databricks cluster details are present. """ + overrides = ls_param_overrides or {} + if overrides: + ls_properties = _substitute_ls_params(ls_properties, overrides) + + # C-02 (NB-ITER2-2 / LSC2-003): unwrap any {value, type:'Expression'} + # dicts that survived the substitution pass. Map fields like + # custom_tags and spark_env_vars must be plain Map[String, String] for + # Databricks to accept the cluster YAML. + ls_properties = _unwrap_expression_value(ls_properties) + nested = ls_properties.get("typeProperties") or {} # Merge: nested values win over flat ones when both exist (matches ARM # template precedence). @@ -520,16 +1190,43 @@ def _extract_cluster_config(ls_properties: dict[str, Any]) -> dict[str, Any] | N new_cluster = fields.get("newClusterVersion") or fields.get("newClusterSparkVersion") if new_cluster: config["spark_version"] = new_cluster - num_workers_raw = fields.get("newClusterNumOfWorker", 1) - try: - config["num_workers"] = int(num_workers_raw) - except (TypeError, ValueError): - config["num_workers"] = num_workers_raw + config["num_workers"] = _coerce_int(fields.get("newClusterNumOfWorker", 1)) config["node_type_id"] = fields.get("newClusterNodeType", "Standard_DS3_v2") spark_conf = fields.get("newClusterSparkConf") if spark_conf: config["spark_conf"] = spark_conf + # Extended cluster fields (LSC-003, NB-3). + driver_node = fields.get("newClusterDriverNodeType") + if driver_node: + config["driver_node_type_id"] = driver_node + spark_env_vars = fields.get("newClusterSparkEnvVars") + if spark_env_vars: + config["spark_env_vars"] = spark_env_vars + custom_tags = fields.get("newClusterCustomTags") + if custom_tags: + config["custom_tags"] = custom_tags + init_scripts = fields.get("newClusterInitScripts") + if init_scripts: + config["init_scripts"] = init_scripts + data_security_mode = fields.get("dataSecurityMode") or fields.get("newClusterDataSecurityMode") + if data_security_mode: + config["data_security_mode"] = data_security_mode + cluster_log_conf = fields.get("clusterLogConf") or fields.get("newClusterLogDestination") + if cluster_log_conf: + config["cluster_log_conf"] = cluster_log_conf + + # C-39 (LSC4-004): capture the ADF authentication shape (e.g. "MSI" or + # any CredentialReference) so the bundler can emit a manual_credential + # SetupTask warning that ``single_user_name`` was rewritten to the + # deploying user. + authentication = fields.get("authentication") + if authentication: + config["_adf_authentication"] = authentication + credential = fields.get("credential") + if isinstance(credential, dict) and credential.get("type") == "CredentialReference": + config["_adf_credential_reference"] = credential.get("referenceName") or "" + return config if config else None @@ -634,6 +1331,12 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["notebook_path"] = activity.notebook_path if activity.base_parameters: extra["base_parameters"] = activity.base_parameters + if activity.notebook_path_unresolved: + extra["notebook_path_unresolved"] = True + if activity.notebook_path_expression is not None: + extra["notebook_path_expression"] = activity.notebook_path_expression + if activity.unresolved_libraries: + extra["unresolved_libraries"] = list(activity.unresolved_libraries) case CopyActivity(): extra["source_type"] = activity.source_type extra["sink_type"] = activity.sink_type @@ -659,12 +1362,24 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["items_expression"] = activity.items_expression extra["concurrency"] = activity.concurrency extra["inner_activities"] = [_activity_to_dict(inner) for inner in activity.inner_activities] + if activity.inputs_bridge_notebook_code: + extra["inputs_bridge_notebook_code"] = activity.inputs_bridge_notebook_code + if activity.inputs_bridge_notebook_imports: + extra["inputs_bridge_notebook_imports"] = list(activity.inputs_bridge_notebook_imports) + if activity.inputs_bridge_required_parameters: + extra["inputs_bridge_required_parameters"] = dict(activity.inputs_bridge_required_parameters) case IfConditionActivity(): extra["op"] = activity.op extra["left"] = activity.left extra["right"] = activity.right extra["if_true_activities"] = [_activity_to_dict(inner) for inner in activity.if_true_activities] extra["if_false_activities"] = [_activity_to_dict(inner) for inner in activity.if_false_activities] + if activity.bridge_notebook_code: + extra["bridge_notebook_code"] = activity.bridge_notebook_code + if activity.bridge_notebook_imports: + extra["bridge_notebook_imports"] = list(activity.bridge_notebook_imports) + if activity.bridge_required_parameters: + extra["bridge_required_parameters"] = dict(activity.bridge_required_parameters) case LookupActivity(): extra["source_type"] = activity.source_type if activity.source_properties: @@ -682,6 +1397,8 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["notebook_imports"] = activity.notebook_imports if activity.required_parameters: extra["required_parameters"] = dict(activity.required_parameters) + if activity.raw_expression: + extra["raw_expression"] = activity.raw_expression case FilterActivity(): extra["items_expression"] = activity.items_expression extra["condition_expression"] = activity.condition_expression @@ -706,6 +1423,12 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: for case_item in activity.cases ] extra["default_activities"] = [_activity_to_dict(inner) for inner in activity.default_activities] + if activity.bridge_notebook_code: + extra["bridge_notebook_code"] = activity.bridge_notebook_code + if activity.bridge_notebook_imports: + extra["bridge_notebook_imports"] = list(activity.bridge_notebook_imports) + if activity.bridge_required_parameters: + extra["bridge_required_parameters"] = dict(activity.bridge_required_parameters) case WaitActivity(): extra["wait_time_seconds"] = activity.wait_time_seconds case SparkJarActivity(): diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index bddcf21..4cfa416 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -156,7 +156,9 @@ def test_setup_notebooks_for_secrets(self, tmp_path): secrets_nb = setup_dir / "create_secrets.py" if secrets_nb.exists(): content = secrets_nb.read_text() - assert "createScope" in content + # C-46 (LSC5-002): provision via the SDK WorkspaceClient, not + # the non-existent dbutils.secrets write API. + assert "create_scope" in content def test_write_bundle_returns_created_files(self, tmp_path): """write_bundle returns a list of all created file paths.""" @@ -261,6 +263,572 @@ def test_load_report_handles_aggregated_translations_format(self, tmp_path): assert task_keys == {"pause", "run_nb"} +class TestScheduleEmission: + """C-10 (SCHED-001): schedule spec on PreparedWorkflow lands in job YAML.""" + + def test_schedule_block_emitted(self, tmp_path): + pipeline = Pipeline( + name="scheduled_job", + tasks=[ + WaitActivity(name="Pause", task_key="pause", wait_time_seconds=10), + ], + schedule={ + "kind": "schedule", + "quartz_cron_expression": "0 0 8 * * ?", + "timezone_id": "Europe/Madrid", + "pause_status": "UNPAUSED", + }, + ) + wf = prepare_workflow(pipeline) + write_bundle(wf, tmp_path) + resource_file = list((tmp_path / "resources").glob("*.yml"))[0] + content = yaml.safe_load(resource_file.read_text()) + job_key = list(content["resources"]["jobs"].keys())[0] + job = content["resources"]["jobs"][job_key] + assert job["schedule"]["quartz_cron_expression"] == "0 0 8 * * ?" + assert job["schedule"]["timezone_id"] == "Europe/Madrid" + assert job["schedule"]["pause_status"] == "UNPAUSED" + + def test_periodic_trigger_emitted(self, tmp_path): + """SCHED3-002: periodic schedule spec renders as trigger.periodic.""" + pipeline = Pipeline( + name="periodic_job", + tasks=[ + WaitActivity(name="Pause", task_key="pause", wait_time_seconds=10), + ], + schedule={ + "kind": "periodic", + "interval": 3, + "unit": "DAYS", + "pause_status": "UNPAUSED", + }, + ) + wf = prepare_workflow(pipeline) + write_bundle(wf, tmp_path) + resource_file = list((tmp_path / "resources").glob("*.yml"))[0] + content = yaml.safe_load(resource_file.read_text()) + job_key = list(content["resources"]["jobs"].keys())[0] + job = content["resources"]["jobs"][job_key] + assert job["trigger"]["periodic"]["interval"] == 3 + assert job["trigger"]["periodic"]["unit"] == "DAYS" + # The cron-style schedule block must NOT appear for periodic specs. + assert "schedule" not in job + + def test_trigger_parameter_overrides_mutate_job_parameter_defaults(self, tmp_path): + """SCHED3-003: schedule.parameter_overrides mutates matching + job.parameters entries' default values.""" + pipeline = Pipeline( + name="trg_override_job", + tasks=[ + WaitActivity(name="Pause", task_key="pause", wait_time_seconds=10), + ], + schedule={ + "kind": "schedule", + "quartz_cron_expression": "0 0 2 * * ?", + "timezone_id": "UTC", + "pause_status": "UNPAUSED", + "parameter_overrides": { + "negocio": "GLP", + "applicationName": "app0001", + }, + }, + ) + wf = prepare_workflow(pipeline) + # Pipeline parameters land on PreparedWorkflow via the report + # round-trip; emulate that here so the bundler has parameters to + # mutate. + wf.parameters = [ + {"name": "negocio", "default": "DEFAULT"}, + {"name": "applicationName", "default": "DEFAULT"}, + ] + write_bundle(wf, tmp_path) + resource_file = list((tmp_path / "resources").glob("*.yml"))[0] + content = yaml.safe_load(resource_file.read_text()) + job_key = list(content["resources"]["jobs"].keys())[0] + job = content["resources"]["jobs"][job_key] + params = {p["name"]: p["default"] for p in job["parameters"]} + assert params["negocio"] == "GLP" + assert params["applicationName"] == "app0001" + + def test_file_arrival_trigger_emitted(self, tmp_path): + pipeline = Pipeline( + name="blob_triggered_job", + tasks=[ + WaitActivity(name="Pause", task_key="pause", wait_time_seconds=10), + ], + schedule={ + "kind": "file_arrival", + "url": "/subscriptions/x/y", + "pause_status": "UNPAUSED", + }, + ) + wf = prepare_workflow(pipeline) + write_bundle(wf, tmp_path) + resource_file = list((tmp_path / "resources").glob("*.yml"))[0] + content = yaml.safe_load(resource_file.read_text()) + job_key = list(content["resources"]["jobs"].keys())[0] + job = content["resources"]["jobs"][job_key] + assert job["trigger"]["file_arrival"]["url"] == "/subscriptions/x/y" + + +class TestStripDanglingTaskValueRefs: + """C-12 (VAREX-005): safety net widens to job_parameters and condition operands.""" + + def test_strips_dangling_run_job_task_job_parameters(self): + from flowx.bundler.dab_writer import _strip_dangling_task_value_refs + + tasks = [ + { + "task_key": "outer", + "run_job_task": { + "job_id": "${resources.jobs.inner.id}", + "job_parameters": { + "valid": "{{tasks.outer.values.something}}", + "dangling": "{{tasks.gone.values.x}}", + }, + }, + }, + ] + _strip_dangling_task_value_refs(tasks, {"outer"}) + assert tasks[0]["run_job_task"]["job_parameters"]["valid"] == "{{tasks.outer.values.something}}" + assert tasks[0]["run_job_task"]["job_parameters"]["dangling"] == "" + + def test_strips_dangling_condition_task_operands(self): + from flowx.bundler.dab_writer import _strip_dangling_task_value_refs + + tasks = [ + { + "task_key": "branch", + "condition_task": { + "op": "EQUAL_TO", + "left": "{{tasks.missing.values.x}}", + "right": "1", + }, + }, + ] + neutralized = _strip_dangling_task_value_refs(tasks, {"branch"}) + # Dangling ref blanked; right operand untouched. + assert tasks[0]["condition_task"]["left"] == "" + assert tasks[0]["condition_task"]["right"] == "1" + # C-43 (CF5-001 / CF5-002): the blanked condition operand is + # recorded so SETUP.md can flag the always-true predicate. + assert neutralized == [ + { + "task_key": "branch", + "field": "left", + "original_ref": "{{tasks.missing.values.x}}", + } + ] + + def test_neutralized_condition_renders_setup_section(self): + """C-43 (CF5-001 / CF5-002): a blanked condition operand surfaces a + 'Conditions neutralized to always-true' section in SETUP.md so the + always-true predicate is never silent.""" + from flowx.bundler.prereqs_writer import build_prereqs, render_setup_md + + prereqs = build_prereqs( + notebooks=[], + tasks=[], + known_bundle_jobs=set(), + neutralized_conditions=[ + { + "task_key": "branch", + "field": "left", + "original_ref": "{{tasks._init_continue.values.continue}}", + } + ], + ) + assert not prereqs.is_empty() + md = render_setup_md(prereqs, bundle_name="b") + assert "Conditions neutralized to always-true" in md + assert "{{tasks._init_continue.values.continue}}" in md + assert "`branch`" in md + + def test_recurses_into_for_each_task_body(self): + from flowx.bundler.dab_writer import _strip_dangling_task_value_refs + + tasks = [ + { + "task_key": "loop", + "for_each_task": { + "inputs": "[1, 2, 3]", + "task": { + "task_key": "loop_body", + "run_job_task": { + "job_id": "inner", + "job_parameters": {"x": "{{tasks.absent.values.x}}"}, + }, + }, + }, + }, + ] + _strip_dangling_task_value_refs(tasks, {"loop", "loop_body"}) + assert tasks[0]["for_each_task"]["task"]["run_job_task"]["job_parameters"]["x"] == "" + + +class TestAggregatedReportPipelineParameters: + """Change pipeline-parameters-and-variables-round-trip (P0): VAR-001.""" + + def test_load_report_carries_pipeline_parameters(self, tmp_path): + import json + + from flowx.bundler.dab_writer import _load_report + + report = { + "translations": [ + { + "pipeline": "p1", + "status": "translated", + "parameters": [{"name": "env", "default": "dev"}], + "ir": { + "type": "WaitActivity", + "name": "Pause", + "task_key": "pause", + "wait_time_seconds": 5, + }, + }, + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + + workflows = _load_report(report_path) + assert len(workflows) == 1 + wf = workflows[0] + # Pipeline-level parameters must survive round-trip. + assert wf.parameters + env_param = next(p for p in wf.parameters if p["name"] == "env") + assert env_param["default"] == "dev" + + +class TestAggregatedReportSchedule: + """Change fix-aggregated-report-propagates-schedule (P0): SCHED3-001.""" + + def test_load_report_carries_pipeline_schedule(self, tmp_path): + import json + + from flowx.bundler.dab_writer import _load_report + + schedule_spec = { + "kind": "cron", + "quartz_cron_expression": "0 0 2 ? * * *", + "timezone_id": "UTC", + "pause_status": "UNPAUSED", + } + report = { + "translations": [ + { + "pipeline": "p_with_schedule", + "status": "translated", + "schedule": schedule_spec, + "ir": { + "type": "WaitActivity", + "name": "Pause", + "task_key": "pause", + "wait_time_seconds": 5, + }, + }, + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + + workflows = _load_report(report_path) + assert len(workflows) == 1 + wf = workflows[0] + assert wf.schedule is not None + assert wf.schedule["kind"] == "cron" + assert wf.schedule["quartz_cron_expression"] == "0 0 2 ? * * *" + + def test_load_report_carries_pipeline_schedule_from_ir(self, tmp_path): + """Older single-pipeline reports nest schedule under ``ir.schedule``.""" + import json + + from flowx.bundler.dab_writer import _load_report + + schedule_spec = { + "kind": "cron", + "quartz_cron_expression": "0 0 4 ? * MON,TUE,WED,THU,FRI *", + "timezone_id": "Europe/Madrid", + "pause_status": "UNPAUSED", + } + report = { + "translations": [ + { + "pipeline": "p_with_ir_schedule", + "status": "translated", + "ir": { + "type": "WaitActivity", + "name": "Pause", + "task_key": "pause", + "wait_time_seconds": 5, + "schedule": schedule_spec, + }, + }, + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + + workflows = _load_report(report_path) + assert len(workflows) == 1 + assert workflows[0].schedule is not None + assert workflows[0].schedule["quartz_cron_expression"].startswith("0 0 4") + + +class TestStubBaseParameterCleanup: + """Change base-parameters-cleanup-and-stub-widgets (P1): NB-5.""" + + def test_stub_notebook_strips_unresolvable_adf_expression(self): + from flowx.bundler.dab_writer import ( + _extract_manual_parameters_from_existing_notebook_tasks, + ) + + tasks = [ + { + "task_key": "lakeh_custom_notebook", + "notebook_task": { + "notebook_path": "../src/notebooks/x.py", + "base_parameters": { + "appName": "@string(coalesce(json(activity('X').output).mail_app_name, ''))", + "kept": "literal-value", + }, + }, + } + ] + manual = _extract_manual_parameters_from_existing_notebook_tasks(tasks) + assert len(manual) == 1 + assert manual[0].task_key == "lakeh_custom_notebook" + assert manual[0].widget_name == "appName" + assert "@string" in manual[0].raw_expression + # The ADF expression value must be dropped from base_parameters. + bp = tasks[0]["notebook_task"]["base_parameters"] + assert "appName" not in bp + assert bp["kept"] == "literal-value" + + +class TestStubLibraryBinding: + """Change library-resolution-and-stub-binding (P0): NB-2.""" + + def test_stub_notebook_with_jar_library_binds_to_default_cluster(self): + from flowx.bundler.constants import DEFAULT_JOB_CLUSTER_KEY + from flowx.bundler.dab_writer import _bind_cluster_to_notebook_tasks + + tasks = [ + { + "task_key": "lakeh_custom_notebook", + "notebook_task": {"notebook_path": "../src/notebooks/x.py"}, + "libraries": [{"jar": "/Volumes/x/my.jar"}], + } + ] + _bind_cluster_to_notebook_tasks(tasks) + # Stub path that ships libraries must bind to the default cluster. + assert tasks[0]["job_cluster_key"] == DEFAULT_JOB_CLUSTER_KEY + + def test_stub_notebook_no_libraries_stays_unbound(self): + from flowx.bundler.dab_writer import _bind_cluster_to_notebook_tasks + + tasks = [ + { + "task_key": "k", + "notebook_task": {"notebook_path": "../src/notebooks/x.py"}, + } + ] + _bind_cluster_to_notebook_tasks(tasks) + assert "job_cluster_key" not in tasks[0] + + def test_serverless_compute_mode_with_jar_library_binds_classic(self): + from flowx.bundler.constants import DEFAULT_JOB_CLUSTER_KEY + from flowx.bundler.dab_writer import _bind_cluster_to_notebook_tasks + + tasks = [ + { + "task_key": "k", + "notebook_task": {"notebook_path": "/Shared/nb"}, + "libraries": [{"whl": "/Volumes/x/wheel.whl"}], + "_compute_mode": "serverless", + } + ] + _bind_cluster_to_notebook_tasks(tasks) + # Serverless can't host whl libraries -> classic cluster bind. + assert tasks[0]["job_cluster_key"] == DEFAULT_JOB_CLUSTER_KEY + + +class TestClusterExtrasPropagation: + """Change linked-service-cluster-field-coverage (P1): NB-3, LSC-003.""" + + def test_extras_merged_into_default_cluster(self, tmp_path): + from flowx.bundler.constants import DEFAULT_JOB_CLUSTER_KEY + from flowx.bundler.dab_writer import ( + _build_default_cluster, + _build_default_job_clusters, + _infer_bundle_cluster_extras, + ) + + # Hint set with consistent extras. + wf = _simple_workflow() + wf.cluster_hints = [ + { + "spark_version": "16.4.x-scala2.12", + "node_type_id": "Standard_D4s_v3", + "driver_node_type_id": "Standard_D8s_v3", + "spark_env_vars": {"PYSPARK_PYTHON": "/databricks/python3/bin/python3"}, + "custom_tags": {"DigitalCase": "X"}, + "data_security_mode": "SINGLE_USER", + } + ] + extras = _infer_bundle_cluster_extras(wf) + assert extras["driver_node_type_id"] == "Standard_D8s_v3" + assert extras["spark_env_vars"]["PYSPARK_PYTHON"] == "/databricks/python3/bin/python3" + assert extras["custom_tags"]["DigitalCase"] == "X" + + clusters = _build_default_job_clusters({DEFAULT_JOB_CLUSTER_KEY}, extras=extras) + assert len(clusters) == 1 + new_cluster = clusters[0]["new_cluster"] + assert new_cluster["driver_node_type_id"] == "Standard_D8s_v3" + assert new_cluster["spark_env_vars"]["PYSPARK_PYTHON"] == "/databricks/python3/bin/python3" + assert new_cluster["custom_tags"]["DigitalCase"] == "X" + + # _build_default_cluster() with no extras keeps the legacy shape. + baseline = _build_default_cluster() + assert "driver_node_type_id" not in baseline["new_cluster"] + + def test_num_workers_mined_into_default_cluster(self): + """C-40 (NB-ITER5-001): cluster_hints carrying num_workers!=1 must + flow into the default job_cluster instead of the hardcoded 1.""" + from flowx.bundler.constants import DEFAULT_JOB_CLUSTER_KEY + from flowx.bundler.dab_writer import ( + _build_default_cluster, + _build_default_job_clusters, + _infer_bundle_cluster_extras, + ) + + wf = _simple_workflow() + wf.cluster_hints = [ + { + "spark_version": "16.4.x-scala2.12", + "node_type_id": "Standard_D4s_v3", + "num_workers": 2, + } + ] + extras = _infer_bundle_cluster_extras(wf) + assert extras["num_workers"] == 2 + + clusters = _build_default_job_clusters({DEFAULT_JOB_CLUSTER_KEY}, extras=extras) + assert clusters[0]["new_cluster"]["num_workers"] == 2 + + # No hint -> legacy single-worker default preserved. + baseline = _build_default_cluster() + assert baseline["new_cluster"]["num_workers"] == 1 + + +class TestManualCredentialFromMsiLinkedService: + """C-39 (LSC4-004): when an ADF linked service authenticates via MSI + (or a CredentialReference), the bundle's default_cluster silently + uses ``single_user_name: ${workspace.current_user.userName}``. The + workflow_preparer must surface a manual_credential SetupTask so + SETUP.md flags the substitution.""" + + def test_msi_authentication_emits_manual_credential_setup_task(self): + from flowx.models.ir import NotebookActivity, Pipeline + from flowx.preparer.workflow_preparer import prepare_workflow + + activity = NotebookActivity( + name="Notebook1", + task_key="notebook1", + notebook_path="/Shared/x", + cluster={ + "spark_version": "15.4.x-scala2.12", + "node_type_id": "Standard_D4s_v3", + "data_security_mode": "SINGLE_USER", + "_adf_authentication": "MSI", + }, + ) + pipeline = Pipeline(name="msi_pipe", tasks=[activity]) + wf = prepare_workflow(pipeline) + manual = [st for st in wf.setup_tasks if st.type == "manual_credential"] + assert len(manual) == 1 + config = manual[0].config + assert config["authentication"] == "MSI" + assert "service principal" in config["note"].lower() + + +class TestUnparseableClusterHintsFiltered: + """C-29 (NB-ITER4-002): unparseable spark_version / node_type_id values + are filtered before Counter so the bundle default stays deployable.""" + + def test_unparseable_spark_version_falls_back_to_default(self): + from flowx.bundler.dab_writer import ( + _DEFAULT_SPARK_VERSION, + _infer_bundle_cluster_defaults, + ) + + wf = _simple_workflow() + wf.cluster_hints = [ + { + "spark_version": "@if(equals(item()?.photon,true),'15.4.x-photon-scala2.12','15.4.x-scala2.12')", + "node_type_id": "Standard_DS3_v2", + }, + ] + spark_version, node_type_id = _infer_bundle_cluster_defaults(wf) + assert spark_version == _DEFAULT_SPARK_VERSION + assert node_type_id == "Standard_DS3_v2" + + def test_unparseable_node_type_falls_back_to_default(self): + from flowx.bundler.dab_writer import ( + _DEFAULT_NODE_TYPE_ID, + _infer_bundle_cluster_defaults, + ) + + wf = _simple_workflow() + wf.cluster_hints = [ + { + "spark_version": "15.4.x-scala2.12", + "node_type_id": "@pipeline().parameters.unresolved", + }, + ] + spark_version, node_type_id = _infer_bundle_cluster_defaults(wf) + assert spark_version == "15.4.x-scala2.12" + assert node_type_id == _DEFAULT_NODE_TYPE_ID + + def test_real_spark_version_still_wins(self): + from flowx.bundler.dab_writer import _infer_bundle_cluster_defaults + + wf = _simple_workflow() + wf.cluster_hints = [ + {"spark_version": "15.4.x-photon-scala2.12", "node_type_id": "Standard_D4s_v3"}, + {"spark_version": "15.4.x-photon-scala2.12", "node_type_id": "Standard_D4s_v3"}, + {"spark_version": "@if(equals(item()?.photon,true),X,Y)", "node_type_id": "Standard_D4s_v3"}, + ] + spark_version, _ = _infer_bundle_cluster_defaults(wf) + assert spark_version == "15.4.x-photon-scala2.12" + + +class TestSingleUserNameOnSingleUserClusters: + """Change fix-single-user-cluster-requires-single-user-name (P0): NB-ITER3-003.""" + + def test_default_cluster_includes_single_user_name(self): + from flowx.bundler.dab_writer import _build_default_cluster + + cluster = _build_default_cluster()["new_cluster"] + assert cluster["data_security_mode"] == "SINGLE_USER" + assert cluster["single_user_name"] == "${workspace.current_user.userName}" + + def test_single_node_cluster_includes_single_user_name(self): + from flowx.bundler.dab_writer import _build_single_node_cluster + + cluster = _build_single_node_cluster()["new_cluster"] + assert cluster["data_security_mode"] == "SINGLE_USER" + assert cluster["single_user_name"] == "${workspace.current_user.userName}" + + def test_multi_node_cluster_includes_single_user_name(self): + from flowx.bundler.dab_writer import _build_multi_node_cluster + + cluster = _build_multi_node_cluster()["new_cluster"] + assert cluster["data_security_mode"] == "SINGLE_USER" + assert cluster["single_user_name"] == "${workspace.current_user.userName}" + + class TestSetupMd: def test_parameter_approximations_render_to_setup_md(self, tmp_path): pipeline = Pipeline( @@ -293,6 +861,47 @@ def test_parameter_approximations_render_to_setup_md(self, tmp_path): assert "Mapped ADF `utcnow()`" in setup_md +class TestManualVariableRollupSetupMd: + """Change fix-cross-foreach-variable-read-warning (P1): VAREX3-003.""" + + def test_setup_md_surfaces_manual_variable_rollup(self, tmp_path): + from flowx.models.ir import ForEachActivity, IfConditionActivity, SetVariableActivity + + # Mirror the preparer-side detection by building the same pipeline. + inner_set = SetVariableActivity( + name="MarkStop", + task_key="mark_stop", + variable_name="continue", + variable_value="false", + ) + loop = ForEachActivity( + name="Loop", + task_key="loop", + items_expression="@output.value", + inner_activities=[inner_set], + concurrency=2, + ) + sibling = IfConditionActivity( + name="CheckCont", + task_key="check_cont", + op="EQUAL_TO", + left="@variables('continue')", + right="true", + if_true_activities=[], + if_false_activities=[], + ) + pipeline = Pipeline( + name="rollup_pipeline", + tasks=[loop, sibling], + ) + wf = prepare_workflow(pipeline) + write_bundle(wf, tmp_path) + setup_md = (tmp_path / "SETUP.md").read_text() + assert "## Manual variable roll-ups" in setup_md + assert "`continue`" in setup_md + assert "`loop`" in setup_md + + class TestSetupGenerator: def test_secrets_setup_notebook_content(self): from flowx.bundler.setup_generator import generate_setup_tasks @@ -305,7 +914,13 @@ def test_secrets_setup_notebook_content(self): assert len(notebooks) == 1 nb = notebooks[0] assert nb.relative_path == "setup/create_secrets.py" - assert "createScope" in nb.content + # C-46 (LSC5-002): generated against the SDK WorkspaceClient, since + # dbutils.secrets is read-only (no createScope / put). + assert "w.secrets.create_scope" in nb.content + assert "w.secrets.put_secret" in nb.content + assert "WorkspaceClient" in nb.content + assert "dbutils.secrets.createScope" not in nb.content + assert "dbutils.secrets.put" not in nb.content assert "my-scope" in nb.content assert "jdbc-url" in nb.content assert "jdbc-password" in nb.content @@ -342,3 +957,64 @@ def test_no_setup_when_empty(self): notebooks = generate_setup_tasks(secrets=[], setup_tasks=[], catalog="main", schema="default") assert len(notebooks) == 0 + + +class TestPipelineDictToIrBridgeFields: + """C-14 (CF3-001 / VAREX3-001): bridge fields survive JSON roundtrip.""" + + def test_if_condition_bridge_fields_preserved(self): + from flowx.bundler.dab_writer import pipeline_dict_to_ir + from flowx.models.ir import IfConditionActivity + + pipeline_dict = { + "name": "p", + "parameters": [], + "tasks": [ + { + "type": "IfConditionActivity", + "name": "Branch", + "task_key": "branch", + "op": "EQUAL_TO", + "left": "__BRIDGE__::result", + "right": "True", + "if_true_activities": [], + "if_false_activities": [], + "bridge_notebook_code": "result = not bool(some_param)", + "bridge_notebook_imports": ["import os"], + "bridge_required_parameters": {"some_param": "{{job.parameters.x}}"}, + } + ], + } + pipeline, _ = pipeline_dict_to_ir(pipeline_dict) + task = pipeline.tasks[0] + assert isinstance(task, IfConditionActivity) + assert task.bridge_notebook_code == "result = not bool(some_param)" + assert task.bridge_notebook_imports == ["import os"] + assert task.bridge_required_parameters == {"some_param": "{{job.parameters.x}}"} + + def test_switch_bridge_fields_preserved(self): + from flowx.bundler.dab_writer import pipeline_dict_to_ir + from flowx.models.ir import SwitchActivity + + pipeline_dict = { + "name": "p", + "parameters": [], + "tasks": [ + { + "type": "SwitchActivity", + "name": "Sw", + "task_key": "sw", + "on_expression": "__BRIDGE__::result", + "cases": [], + "default_activities": [], + "bridge_notebook_code": "result = item.get('type', 'default').upper()", + "bridge_notebook_imports": [], + "bridge_required_parameters": {"item": "{{tasks.upstream.values.row}}"}, + } + ], + } + pipeline, _ = pipeline_dict_to_ir(pipeline_dict) + task = pipeline.tasks[0] + assert isinstance(task, SwitchActivity) + assert task.bridge_notebook_code == "result = item.get('type', 'default').upper()" + assert task.bridge_required_parameters == {"item": "{{tasks.upstream.values.row}}"} diff --git a/tests/unit/test_code_generator.py b/tests/unit/test_code_generator.py index 84a07d0..4e4b237 100644 --- a/tests/unit/test_code_generator.py +++ b/tests/unit/test_code_generator.py @@ -225,6 +225,68 @@ def test_unparseable_dynamic_query_falls_back_to_literal(self): # ("dbutils.widgets.get(..."), not as executable Python. assert 'query = "dbutils.widgets.get(' in content + def test_file_lookup_coerces_expression_dict_path_components(self): + """C-37 (LSC4-001): folder_path / file_name that arrive as ADF + expression dicts must not crash ``_assemble_file_lookup_source_path`` + with AttributeError on ``.strip('/')``.""" + activity = LookupActivity( + **_make_base("Read_Cfg", "read_cfg"), + source_type="JsonSource", + first_row_only=False, + source_properties={ + "dataset_type": "Json", + "container": "configs", + "folder_path": {"value": "@pipeline().parameters.folder", "type": "Expression"}, + "file_name": "tables.json", + }, + ) + # Must not raise. + content = generate_lookup_notebook(activity) + _assert_valid_python(content, "read_cfg (expression-dict folder)") + + def test_file_lookup_rewrites_https_to_abfss(self): + """C-37 (LSC4-003): AzureBlobFS https URLs lower to abfss:// in + the generated lookup notebook so the read can succeed on a + Databricks cluster.""" + activity = LookupActivity( + **_make_base("Read_Cfg", "read_cfg"), + source_type="JsonSource", + first_row_only=False, + source_properties={ + "dataset_type": "Json", + "container": "configext", + "folder_path": "lookups", + "file_name": "tables.json", + "linked_service_url": "https://examplelake.dfs.core.windows.net", + }, + ) + content = generate_lookup_notebook(activity) + _assert_valid_python(content, "read_cfg (abfss rewrite)") + assert "abfss://configext@examplelake.dfs.core.windows.net" in content + assert "https://examplelake" not in content + + def test_file_source_lookup_emits_spark_read(self): + """Change lookup-file-dataset-support (P0): JsonSource + firstRowOnly=False.""" + activity = LookupActivity( + **_make_base("Read_Configuration", "read_configuration"), + source_type="JsonSource", + first_row_only=False, + source_properties={ + "dataset_type": "Json", + "container": "configs", + "folder_path": "lookup", + "file_name": "tables.json", + "multiLineJson": True, + }, + ) + content = generate_lookup_notebook(activity) + _assert_valid_python(content, "read_configuration (file source)") + # File-source branch: no spark.sql(''), uses spark.read.format().load(). + assert "spark.sql" not in content + assert "spark.read.format('json')" in content + assert ".option(\"multiline\", \"true\")" in content + assert "source_path" in content + # --------------------------------------------------------------------------- # Web activity notebook generator @@ -278,6 +340,25 @@ def test_auth_block_service_principal(self): assert "auth-credential" in content assert "Bearer" in content + def test_auth_block_msi_raises_not_implemented(self): + """LSC3-002: MSI / ManagedServiceIdentity has no static secret to + read; the generated notebook must raise NotImplementedError pointing + at SETUP.md, not emit a fake dbutils.secrets.get('auth-credential').""" + for auth_type in ("MSI", "ManagedServiceIdentity"): + activity = WebActivity( + **_make_base(f"MsiApi_{auth_type}", f"msi_api_{auth_type.lower()}"), + url="https://api.example.com", + method="GET", + authentication={"type": auth_type, "resource": "https://management.azure.com"}, + ) + content = generate_web_activity_notebook( + activity, scope=f"msi_api_{auth_type.lower()}" + ) + _assert_valid_python(content, f"msi_api ({auth_type})") + assert "auth-credential" not in content + assert "NotImplementedError" in content + assert "SETUP.md" in content + def test_auth_block_basic(self): """Basic auth generates username/password secret retrieval.""" activity = WebActivity( diff --git a/tests/unit/test_expression_parser.py b/tests/unit/test_expression_parser.py index d672c62..cd5e769 100644 --- a/tests/unit/test_expression_parser.py +++ b/tests/unit/test_expression_parser.py @@ -38,10 +38,16 @@ def test_float(self): assert result.value == "3.14" def test_boolean(self): - result = resolve_expression(True, _context()) - assert result is not None - assert result.kind == "literal" - assert result.value == "True" + # VAREX3-002: Python bool renders lowercase to match ADF semantics. + result_t = resolve_expression(True, _context()) + assert result_t is not None + assert result_t.kind == "literal" + assert result_t.value == "true" + + result_f = resolve_expression(False, _context()) + assert result_f is not None + assert result_f.kind == "literal" + assert result_f.value == "false" def test_expression_dict_wrapping(self): result = resolve_expression({"type": "Expression", "value": "hello"}, _context()) @@ -136,11 +142,12 @@ def test_variable_with_explicit_task_keys(self): assert result.kind == "dab_ref" assert result.value == "{{tasks.SetRunDate.values.runDate}}" - def test_variable_fallback_to_name(self): + def test_variable_returns_none_when_no_setter(self): + """C-05 (VAREX-002): unknown variables resolve to ``None`` instead of + a self-referential dangling ``{{tasks.X.values.X}}`` placeholder + that never gets satisfied at runtime.""" result = resolve_expression("@variables('unknown')", _context()) - assert result is not None - assert result.kind == "dab_ref" - assert result.value == "{{tasks.unknown.values.unknown}}" + assert result is None class TestItem: @@ -150,6 +157,43 @@ def test_item(self): assert result.kind == "dab_ref" assert result.value == "{{input}}" + def test_item_safe_nav_single_segment(self): + """C-16 (CF3-005 / VAREX3-005): single-segment item()?.X must lower to + notebook_code so bridge lowering can fire downstream (Switch on-expr, + SetVariable expressions wrapping the safe-nav).""" + result = resolve_expression("@item()?.foo", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "json" in result.value + assert "get('foo')" in result.value + + def test_item_safe_nav_multi_segment_unchanged(self): + """Two-segment item()?.a?.b continues to lower to notebook_code.""" + result = resolve_expression("@item()?.foo?.bar", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "get('foo')" in result.value + assert "get('bar')" in result.value + + def test_item_field_no_safe_nav_remains_dab_ref(self): + """Plain item().foo with no safe-nav operator stays a dab_ref.""" + result = resolve_expression("@item().foo", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{input.foo}}" + + def test_item_field_multi_segment_lowers_to_notebook_code(self): + """C-35 (CF4-004): ``item().condition.name`` must walk both + ``.condition`` and ``.name`` instead of truncating to + ``{{input.condition}}``. Previously ``_ITEM_FIELD_RE`` matched the + first segment without an end-anchor so the trailing ``.name`` + was silently dropped.""" + result = resolve_expression("@item().condition.name", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "get('condition')" in result.value + assert "get('name')" in result.value + class TestUtcNow: def test_utcnow_no_format(self): @@ -192,11 +236,30 @@ def test_utcnow_expression_dict(self): class TestConcat: def test_concat_literals(self): + # C-01: when every part is a literal, the whole concat collapses to + # a single literal so consumers (cluster fields, library jar paths, + # ...) receive a plain string instead of Python source. result = resolve_expression("@concat('hello', ' ', 'world')", _context()) assert result is not None - assert result.kind == "notebook_code" - # Should produce a Python concatenation - assert "+" in result.value + assert result.kind == "literal" + assert result.value == "hello world" + + def test_concat_collapses_when_all_parts_resolve_to_literals(self): + # C-01: factory globals collapse @concat parts to literal kinds, + # so the whole concat should likewise be a literal string. + ctx = TranslationContext( + global_parameters=MappingProxyType( + {"env_variable": "t", "deequLibFileName": "deequ-3.5.6.jar"} + ), + ) + result = resolve_expression( + "@concat('/Volumes/datahub01', pipeline().globalParameters.env_variable, " + "'/lib/', pipeline().globalParameters.deequLibFileName)", + ctx, + ) + assert result is not None + assert result.kind == "literal" + assert result.value == "/Volumes/datahub01t/lib/deequ-3.5.6.jar" def test_concat_with_variable(self): result = resolve_expression( @@ -311,6 +374,65 @@ def test_substring(self): # Should produce a slice expression assert "[" in result.value + def test_equals_quoted_string_emits_repr(self): + """C-34 (VAREX4-002): a quoted ``'12'`` argument must keep its + quotedness through codegen so the comparison emits ``... == '12'`` + rather than the bare token ``12`` (which silently compares against + a numeric value).""" + result = resolve_expression( + "@equals(variables('month'), '12')", + _context(month="set_month"), + ) + assert result is not None + assert "== '12'" in result.value + + def test_less_quoted_leading_zero_is_valid_python(self): + """C-34 (VAREX4-002): a leading-zero quoted argument like ``'09'`` + must round-trip as a Python string literal, not the bare token + ``09`` (which is a SyntaxError in modern Python).""" + result = resolve_expression( + "@less(variables('month'), '09')", + _context(month="set_month"), + ) + assert result is not None + # Result must parse as valid Python. + compile(result.value, "", "eval") + assert "< '09'" in result.value + + def test_equals_bool_literal_emits_lowercase_string(self): + """C-34 (VAREX4-003): an ADF Boolean ``true`` argument lowers to + the lowercase string literal ``'true'`` so the comparison matches + what C-21 SetVariable writes on the consumer side.""" + result = resolve_expression( + "@equals(variables('X'), true)", + _context(X="set_x"), + ) + assert result is not None + assert "== 'true'" in result.value + + def test_substring_two_arg_form(self): + """C-33 (VAREX4-001): ADF accepts substring(text, start) without an + explicit length argument.""" + result = resolve_expression( + "@substring(string(pipeline().parameters.params), 1)", _context() + ) + assert result is not None + assert result.kind == "notebook_code" + assert "[int(" in result.value + assert "):]" in result.value + + def test_split_with_subscript(self): + """C-33 (VAREX4-001): a trailing ``[N]`` on a function call lowers + to notebook_code Python source so SetVariable activities wrapping + ``@split(...)[0]`` actually resolve.""" + result = resolve_expression( + "@split(pipeline().parameters.referenceDate,'/')[0]", _context() + ) + assert result is not None + assert result.kind == "notebook_code" + assert ".split(str('/'))" in result.value + assert ")[0]" in result.value + def test_to_lower(self): result = resolve_expression("@toLower('HELLO')", _context()) assert result is not None @@ -766,3 +888,153 @@ def test_parse_expression_for_dab_returns_ref_for_utcnow(self): def test_parse_expression_for_dab_returns_none_for_non_expression(self): result = parse_expression_for_dab("plain_string") assert result is None + + +class TestGlobalParameters: + """Change expr-resolver-globalparams-and-wrappers (P0).""" + + def _ctx_with_globals(self, **globals_) -> TranslationContext: + return TranslationContext( + global_parameters=MappingProxyType(dict(globals_)), + ) + + def test_global_parameter_resolves_to_literal(self): + ctx = self._ctx_with_globals(env_variable="t") + result = resolve_expression("@pipeline().globalParameters.env_variable", ctx) + assert result is not None + assert result.kind == "literal" + assert result.value == "t" + + def test_global_parameter_value_dict(self): + ctx = self._ctx_with_globals( + env_variable={"type": "string", "value": "t"}, + ) + result = resolve_expression("@pipeline().globalParameters.env_variable", ctx) + assert result is not None + assert result.kind == "literal" + assert result.value == "t" + + def test_global_parameter_missing_falls_back_to_dab_ref(self): + ctx = TranslationContext() + result = resolve_expression("@pipeline().globalParameters.something", ctx) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.parameters.something}}" + + def test_concat_with_globals_resolves_fully(self): + # C-01: when every part collapses to a literal, the whole concat + # is itself a literal so downstream consumers don't have to eval. + ctx = self._ctx_with_globals(env_variable="t", libFileName="myjar.jar") + expr = ( + "@concat('/Volumes/datahub01', pipeline().globalParameters.env_variable, " + "'/x/', pipeline().globalParameters.libFileName)" + ) + result = resolve_expression(expr, ctx) + assert result is not None + assert result.kind == "literal" + assert result.value == "/Volumes/datahub01t/x/myjar.jar" + + +class TestNoopWrappers: + """Change expr-resolver-globalparams-and-wrappers (P0): @json/@string/@array.""" + + def test_json_wrapper_around_pipeline_param(self): + ctx = TranslationContext() + result = resolve_expression("@json(pipeline().parameters.items)", ctx) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.parameters.items}}" + + def test_string_wrapper_around_pipeline_param(self): + ctx = TranslationContext() + result = resolve_expression("@string(pipeline().parameters.value)", ctx) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.parameters.value}}" + + def test_array_wrapper_around_pipeline_param(self): + ctx = TranslationContext() + result = resolve_expression("@array(pipeline().parameters.lst)", ctx) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.parameters.lst}}" + + +class TestTrailingWhitespaceFunctionCall: + """Change expr-resolver-globalparams-and-wrappers (P0): VAR-003 regex anchor bug.""" + + def test_string_wrapper_with_trailing_newlines(self): + ctx = TranslationContext() + # Previously the regex anchor at end refused trailing whitespace. + result = resolve_expression( + "@string(activity('X').output.runOutput.year)\n\n", + ctx, + ) + assert result is not None + # Should hit the function-call branch (string wraps activity output). + assert result.kind == "dab_ref" + assert "tasks.X.values" in result.value + + +class TestItemSafeNav: + """Change expr-resolver-globalparams-and-wrappers (P0): VAR-005.""" + + def test_item_safe_nav_chain_resolves(self): + ctx = TranslationContext() + result = resolve_expression( + "@coalesce(item()?.condition?.name, 'fallback')", + ctx, + ) + assert result is not None + assert result.kind == "notebook_code" + # The chain walk should emit nested .get() calls + assert ".get('condition')" in result.value + assert ".get('name')" in result.value + assert "'fallback'" in result.value + + +class TestLinkedServiceParameter: + """Change linked-service-parameter-resolution (P0): NB-4, LSC-001.""" + + def test_linked_service_param_resolves_with_supplied_value(self): + ctx = TranslationContext( + linked_service_parameters=MappingProxyType({"clusterVersion": "16.4.x-scala2.12"}), + ) + result = resolve_expression("@linkedService().clusterVersion", ctx) + assert result is not None + assert result.kind == "literal" + assert result.value == "16.4.x-scala2.12" + + def test_linked_service_param_missing_returns_none(self): + ctx = TranslationContext() + result = resolve_expression("@linkedService().clusterVersion", ctx) + # Without a value, we can't deterministically resolve; caller must + # supply via with_linked_service_parameters or accept None. + assert result is None + + +class TestFunctionCallWithAttribute: + """Change fix-attribute-access-on-function-results (P1): CF3-004.""" + + def test_json_call_with_trailing_attribute_lowers_to_notebook_code(self): + ctx = TranslationContext() + result = resolve_expression( + "@toUpper(json(pipeline().parameters.items).type)", + ctx, + ) + assert result is not None + # toUpper wraps the json(...).type expression; the inner + # json(...).type chain must resolve to notebook_code so the bridge + # can pick it up. + assert result.kind == "notebook_code" + + def test_function_call_with_attribute_alone_lowers_to_notebook_code(self): + ctx = TranslationContext() + result = resolve_expression( + "@json(pipeline().parameters.items).type", + ctx, + ) + assert result is not None + assert result.kind == "notebook_code" + # The lowered code chains .get('type') onto a json-loaded widget. + assert ".get('type')" in result.value diff --git a/tests/unit/test_for_each_inner_job_params.py b/tests/unit/test_for_each_inner_job_params.py new file mode 100644 index 0000000..8c20140 --- /dev/null +++ b/tests/unit/test_for_each_inner_job_params.py @@ -0,0 +1,148 @@ +"""Unit tests for flowx.bundler.inner_job_params. + +Covers C-06 (VAREX-004): variable references in a ForEach inner-job body +must not produce undeclared inner-job parameters. When the parent job has +a known setter task for the variable, the inner job receives a +``{{tasks..values.}}`` reference; when no setter is known the +name still surfaces as an inner parameter (legacy fallback so test-only +flows that omit the mapping continue to work). +""" + +from __future__ import annotations + +from flowx.bundler.inner_job_params import collect_inner_job_params + + +def _notebook_task(base_parameters: dict[str, str]) -> dict[str, object]: + return { + "task_key": "inner_nb", + "notebook_task": { + "notebook_path": "/Shared/inner", + "base_parameters": base_parameters, + }, + } + + +class TestVariableTaskKeysRouting: + def test_variable_with_known_setter_routes_via_task_value(self): + """C-06: a variable referenced inside the inner job with a known + parent-side setter is NOT declared as an inner job parameter.""" + inner_tasks = [_notebook_task({"continue": "@variables('continue')"})] + parameters, job_parameters = collect_inner_job_params( + inner_tasks, + variable_task_keys={"continue": "_init_continue"}, + ) + # No inner parameter declared for `continue` -- the parent passes + # the task-value reference through job_parameters instead. + param_names = {p["name"] for p in parameters} + assert "continue" not in param_names + assert job_parameters["continue"] == "{{tasks._init_continue.values.continue}}" + + def test_variable_without_setter_falls_back_to_parent_job_parameter(self): + """Legacy fallback for tests that don't supply a setter map.""" + inner_tasks = [_notebook_task({"continue": "@variables('continue')"})] + parameters, job_parameters = collect_inner_job_params(inner_tasks) + param_names = {p["name"] for p in parameters} + # Without variable_task_keys we still emit the (broken) job.parameters + # reference so prior behaviour is preserved when callers don't opt in. + assert "continue" in param_names + assert job_parameters["continue"] == "{{job.parameters.continue}}" + + def test_pipeline_parameter_still_uses_job_parameters_ref(self): + """C-06 only redirects variables -- pipeline parameters still flow + via the inner job's parameter declarations as before.""" + inner_tasks = [_notebook_task({"env": "@pipeline().parameters.env"})] + parameters, job_parameters = collect_inner_job_params( + inner_tasks, + variable_task_keys={"continue": "_init_continue"}, # unrelated var + ) + param_names = {p["name"] for p in parameters} + assert "env" in param_names + assert job_parameters["env"] == "{{job.parameters.env}}" + + def test_multi_child_for_each_threads_variable_task_keys(self, monkeypatch): + """CF3-006: ForEach preparer's multi-child path must thread + variable_task_keys into collect_inner_job_params just like the + single-child escalation path does, so the same parent->setter map + is honoured regardless of how many children the ForEach has. + + Asserts the kwarg is forwarded by intercepting collect_inner_job_params. + """ + from flowx.models.ir import ForEachActivity, NotebookActivity + from flowx.preparer.activity_preparers import for_each as for_each_module + from flowx.preparer.workflow_preparer import prepare_activity + + def _base(name: str, key: str) -> dict[str, object]: + return { + "name": name, + "task_key": key, + "description": None, + "timeout_seconds": None, + "max_retries": None, + "min_retry_interval_millis": None, + "depends_on": None, + "cluster": None, + } + + captured: list[dict[str, str] | None] = [] + from flowx.bundler import inner_job_params as ijp_module + + original = ijp_module.collect_inner_job_params + + def _spy(tasks, *, raw_ir_tasks=None, variable_task_keys=None): + captured.append(variable_task_keys) + return original( + tasks, raw_ir_tasks=raw_ir_tasks, variable_task_keys=variable_task_keys + ) + + monkeypatch.setattr(for_each_module, "collect_inner_job_params", _spy) + + nb_a = NotebookActivity( + **_base("InnerA", "inner_a"), + notebook_path="/Shared/a", + base_parameters={"continue": "@variables('continue')"}, + ) + nb_b = NotebookActivity( + **_base("InnerB", "inner_b"), + notebook_path="/Shared/b", + base_parameters={"continue": "@variables('continue')"}, + ) + loop = ForEachActivity( + **_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[nb_a, nb_b], + concurrency=2, + ) + prepared = prepare_activity( + loop, + variable_task_keys={"continue": "_init_continue"}, + ) + # Multi-child escalation -> exactly one collect call from for_each preparer. + for_each_call = next( + (m for m in captured if m and "continue" in m), None + ) + assert for_each_call is not None, "variable_task_keys must be forwarded" + assert for_each_call["continue"] == "_init_continue" + # Multi-child path -> inner_workflows populated. + assert prepared.inner_workflows + + def test_mixed_variable_and_pipeline_param_payload(self): + """A base_parameters block with both a variable and a pipeline param.""" + inner_tasks = [ + _notebook_task( + { + "ctx_continue": "@variables('continue')", + "ctx_env": "@pipeline().parameters.env", + } + ) + ] + parameters, job_parameters = collect_inner_job_params( + inner_tasks, + variable_task_keys={"continue": "_init_continue"}, + ) + param_names = {p["name"] for p in parameters} + # Only the pipeline parameter is declared on the inner job. + assert "continue" not in param_names + assert "env" in param_names + assert job_parameters["continue"] == "{{tasks._init_continue.values.continue}}" + assert job_parameters["env"] == "{{job.parameters.env}}" diff --git a/tests/unit/test_preparers.py b/tests/unit/test_preparers.py index 302eadf..3b9a8fc 100644 --- a/tests/unit/test_preparers.py +++ b/tests/unit/test_preparers.py @@ -15,6 +15,7 @@ ExecutePipelineActivity, FilterActivity, ForEachActivity, + IfConditionActivity, LookupActivity, NotebookActivity, Pipeline, @@ -77,6 +78,59 @@ def test_prepare_notebook_task_structure(self): assert prepared.task["notebook_task"]["base_parameters"] == {"env": "dev"} assert prepared.notebooks == [] + def test_prepare_notebook_dispatch_stub_for_unresolved_path(self): + """C-28 (NB-ITER4-001): a NotebookActivity flagged + ``notebook_path_unresolved`` produces a dispatch-stub notebook + (not a NotImplementedError placeholder) plus a + ``dynamic_notebook_dispatch`` SetupTask for SETUP.md.""" + activity = NotebookActivity( + **_make_base("Dispatch", "dispatch"), + notebook_path="", + notebook_path_unresolved=True, + notebook_path_expression="@trim(json(activity('cfg').output.firstRow).notebook_path)", + base_parameters={"env": "dev"}, + ) + prepared = prepare_activity(activity) + assert len(prepared.notebooks) == 1 + content = prepared.notebooks[0].content + assert "dbutils.widgets.get('notebook_path')" in content + assert "dbutils.notebook.run" in content + assert "raise NotImplementedError" not in content + # SETUP.md SetupTask is emitted. + kinds = [st.type for st in prepared.setup_tasks] + assert "dynamic_notebook_dispatch" in kinds + config = next( + st.config for st in prepared.setup_tasks if st.type == "dynamic_notebook_dispatch" + ) + assert config["task_key"] == "dispatch" + assert "@trim" in config["expression"] + # The notebook_path widget is registered with an empty default. + assert prepared.task["notebook_task"]["base_parameters"]["notebook_path"] == "" + + def test_prepare_notebook_emits_unresolved_library_setup_task(self): + """C-30 (NB-ITER4-003): unresolved_libraries on the IR emerge as + ``unresolved_library`` SetupTasks the bundler renders in SETUP.md.""" + activity = NotebookActivity( + **_make_base("Run NB", "run_nb"), + notebook_path="/Shared/x", + unresolved_libraries=[ + { + "type": "jar", + "expression": "@concat('/Volumes/x/', pipeline().globalParameters.proj4jLibFileName)", + "missing": ["proj4jLibFileName"], + } + ], + ) + prepared = prepare_activity(activity) + kinds = [st.type for st in prepared.setup_tasks] + assert "unresolved_library" in kinds + config = next( + st.config for st in prepared.setup_tasks if st.type == "unresolved_library" + ) + assert config["task_key"] == "run_nb" + assert config["library_type"] == "jar" + assert "proj4jLibFileName" in config["missing"] + def test_prepare_notebook_no_params(self): activity = NotebookActivity( **_make_base("NB", "nb"), @@ -341,6 +395,58 @@ def test_prepare_web_activity_with_auth_creates_secrets(self): assert len(prepared.secrets) >= 1 assert any(s.key == "auth-credential" for s in prepared.secrets) + def test_prepare_web_activity_key_vault_secret_uses_vault_scope_and_secret_name(self): + """C-11 (LSC2-005): an AzureKeyVaultSecret payload preserves the Key + Vault scope and secret name instead of collapsing to the generic + ``auth-credential`` placeholder.""" + activity = WebActivity( + **_make_base("Auth API", "auth_api"), + url="https://api.example.com", + method="POST", + authentication={ + "type": "ServicePrincipal", + "password": { + "type": "AzureKeyVaultSecret", + "store": {"referenceName": "lakeh_ls_keyvault"}, + "secretName": "adapp-auccommonutilssp-secret", + "typeProperties": {"baseUrl": "https://kv.example.net/"}, + }, + }, + ) + prepared = prepare_activity(activity) + assert any( + s.scope == "lakeh_ls_keyvault" and s.key == "adapp-auccommonutilssp-secret" + for s in prepared.secrets + ) + # The generic auth-credential placeholder is suppressed when a real + # secret reference is available. + assert not any(s.key == "auth-credential" for s in prepared.secrets) + # C-38 (LSC4-002): the generated notebook must reference the + # resolved AKV scope and key, not the legacy + # ``(task_key, 'auth-credential')`` placeholder. + notebook_content = prepared.notebooks[0].content + assert 'scope="lakeh_ls_keyvault"' in notebook_content + assert 'key="adapp-auccommonutilssp-secret"' in notebook_content + assert 'key="auth-credential"' not in notebook_content + + def test_prepare_web_activity_credential_reference_emits_setup_note(self): + """C-11: CredentialReference (managed identity) routes to a SetupTask + instead of fabricating a static secret placeholder.""" + activity = WebActivity( + **_make_base("Auth API", "auth_api"), + url="https://api.example.com", + method="POST", + authentication={ + "type": "MSI", + "credential": {"referenceName": "msi_credential"}, + }, + ) + prepared = prepare_activity(activity) + manual = [t for t in prepared.setup_tasks if t.type == "manual_credential"] + assert manual, "credential reference must surface a manual_credential SetupTask" + # No static placeholder secret emitted for managed-identity auth. + assert not any(s.key == "auth-credential" for s in prepared.secrets) + class TestDeletePreparer: def test_prepare_delete_generates_notebook(self): @@ -469,6 +575,294 @@ def test_prepare_for_each_wraps_inner(self): assert prepared.task["for_each_task"]["concurrency"] == 10 assert prepared.task["for_each_task"]["inputs"] == "@output.value" + def test_prepare_for_each_uses_ir_bridge_for_variable_based_split(self): + """C-31 (CF4-001): when the items expression references a + ``@variables('X')`` setter, the translator captures the bridge + code on the IR while the variable_cache is populated. The + preparer must consume that IR-supplied bridge rather than + re-resolving against an empty TranslationContext (which used to + silently fail and ship the raw @split string as inputs).""" + inner = WaitActivity(**_make_base("Inner", "inner"), wait_time_seconds=1) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@split(variables('fecha'),',')", + inputs_bridge_notebook_code=( + "str(dbutils.jobs.taskValues.get(taskKey='_init_fecha', key='fecha')).split(str(','))" + ), + inputs_bridge_notebook_imports=[], + inputs_bridge_required_parameters={"fecha": "{{tasks._init_fecha.values.fecha}}"}, + inner_activities=[inner], + concurrency=10, + ) + prepared = prepare_activity(activity) + # The bridge fires off the IR fields even though resolve_expression + # against a bare context would fail to resolve @variables('fecha'). + bridge_keys = [ + t.get("task_key") for t in prepared.extra_tasks if t.get("task_key", "").endswith("_inputs_bridge") + ] + assert bridge_keys == ["loop_inputs_bridge"] + assert prepared.task["for_each_task"]["inputs"] == "{{tasks.loop_inputs_bridge.values.items}}" + bridge_task = next(t for t in prepared.extra_tasks if t["task_key"] == "loop_inputs_bridge") + assert bridge_task["notebook_task"]["base_parameters"]["fecha"] == "{{tasks._init_fecha.values.fecha}}" + + def test_prepare_for_each_bridges_split_items_via_seed_task(self): + """C-08 (CF-iter2-002): @split(, ',') as items_expression must + route through a seed bridge task so for_each_task.inputs is a real + DAB task-value reference rather than a raw ADF expression.""" + inner = WaitActivity(**_make_base("Inner", "inner"), wait_time_seconds=1) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@split(pipeline().parameters.ejecuciones, ';')", + inner_activities=[inner], + concurrency=10, + ) + prepared = prepare_activity(activity) + # Bridge task synthesised ahead of the ForEach. + bridge_keys = [ + t.get("task_key") for t in prepared.extra_tasks if t.get("task_key", "").endswith("_inputs_bridge") + ] + assert bridge_keys == ["loop_inputs_bridge"] + bridge_task = next(t for t in prepared.extra_tasks if t["task_key"] == "loop_inputs_bridge") + assert "notebook_task" in bridge_task + assert "ejecuciones" in bridge_task["notebook_task"]["base_parameters"] + # inputs now reference the bridge task value, not the @split string. + assert prepared.task["for_each_task"]["inputs"] == "{{tasks.loop_inputs_bridge.values.items}}" + # ForEach depends on the bridge so the value is materialised first. + assert any( + dep.get("task_key") == "loop_inputs_bridge" for dep in prepared.task.get("depends_on") or [] + ) + + def test_for_each_with_inner_if_condition_carries_branches(self): + """Change foreach-inner-extra-tasks (P0): CF-001. + + When the ForEach has multiple children and one of them is an + IfCondition / Switch, the branch bodies live in the child's + extra_tasks. The preparer must extend inner_tasks with those + so they land in the inner-job, not get dropped. + """ + from flowx.models.ir import IfConditionActivity + + # IfCondition with two branch tasks. + true_act = WaitActivity(**_make_base("TrueWait", "true_wait"), wait_time_seconds=1) + false_act = WaitActivity(**_make_base("FalseWait", "false_wait"), wait_time_seconds=2) + if_act = IfConditionActivity( + **_make_base("If_Condition1", "if_condition1"), + op="EQUAL_TO", + left="@item().x", + right="1", + if_true_activities=[true_act], + if_false_activities=[false_act], + ) + sibling = WaitActivity(**_make_base("Sibling", "sibling"), wait_time_seconds=3) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[if_act, sibling], + concurrency=5, + ) + prepared = prepare_activity(activity) + assert prepared.inner_workflows, "should escalate to sub-job" + inner_wf = prepared.inner_workflows[0] + task_keys = {t["task_key"] for t in inner_wf.tasks} + # Branch tasks survive alongside the condition task. + assert "true_wait" in task_keys + assert "false_wait" in task_keys + assert "sibling" in task_keys + + def test_for_each_inner_workflow_carries_cluster_hints_from_inner_activity(self): + """LSC3-001: ForEach inner-job PreparedWorkflow must carry cluster + hints lifted from nested NotebookActivity.cluster so the inner job's + default_cluster picks up LS-derived spark_env_vars / custom_tags / + driver_node_type_id. + """ + inner_nb = NotebookActivity( + **_make_base("InnerNB", "inner_nb"), + notebook_path="/Shared/ETL/inner", + base_parameters={}, + ) + inner_nb.cluster = { + "spark_version": "16.4.x-scala2.12", + "node_type_id": "Standard_D4s_v3", + "driver_node_type_id": "Standard_D8s_v3", + "spark_env_vars": {"PYSPARK_PYTHON": "/databricks/python3/bin/python3"}, + "custom_tags": {"DigitalCase": "X"}, + } + sibling = WaitActivity(**_make_base("Sibling", "sibling"), wait_time_seconds=1) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[inner_nb, sibling], + concurrency=5, + ) + prepared = prepare_activity(activity) + assert prepared.inner_workflows + inner_wf = prepared.inner_workflows[0] + # The cluster hint from inner_nb propagates through to the inner + # workflow so _infer_bundle_cluster_extras picks it up. + assert inner_wf.cluster_hints, "inner workflow must carry cluster hints" + hint = inner_wf.cluster_hints[0] + assert hint["driver_node_type_id"] == "Standard_D8s_v3" + assert hint["spark_env_vars"]["PYSPARK_PYTHON"] == "/databricks/python3/bin/python3" + assert hint["custom_tags"]["DigitalCase"] == "X" + + def test_for_each_single_child_inner_workflow_carries_cluster_hints(self): + """LSC3-001 single-child escalation path equally must propagate + cluster hints from the single nested IfCondition-wrapped activity. + """ + inner_nb = NotebookActivity( + **_make_base("InnerNB", "inner_nb"), + notebook_path="/Shared/ETL/inner", + base_parameters={}, + ) + inner_nb.cluster = {"custom_tags": {"DigitalCase": "Y"}} + if_act = IfConditionActivity( + **_make_base("If1", "if1"), + op="EQUAL_TO", + left="@item().x", + right="1", + if_true_activities=[inner_nb], + if_false_activities=[], + ) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[if_act], + concurrency=2, + ) + prepared = prepare_activity(activity) + assert prepared.inner_workflows + inner_wf = prepared.inner_workflows[0] + assert inner_wf.cluster_hints, "single-child escalation must propagate cluster hints" + assert inner_wf.cluster_hints[0]["custom_tags"]["DigitalCase"] == "Y" + + def test_for_each_with_single_child_if_condition_escalates_to_subjob(self): + """Single-child IfCondition forces the sub-job path so branches survive.""" + from flowx.models.ir import IfConditionActivity + + true_act = WaitActivity(**_make_base("Hot", "hot"), wait_time_seconds=1) + false_act = WaitActivity(**_make_base("Cold", "cold"), wait_time_seconds=2) + if_act = IfConditionActivity( + **_make_base("Maybe", "maybe"), + op="EQUAL_TO", + left="@item().x", + right="1", + if_true_activities=[true_act], + if_false_activities=[false_act], + ) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[if_act], + concurrency=2, + ) + prepared = prepare_activity(activity) + # Branch bodies require a sub-job (for_each_task.task is a single task). + assert prepared.inner_workflows + inner_wf = prepared.inner_workflows[0] + task_keys = {t["task_key"] for t in inner_wf.tasks} + assert "hot" in task_keys + assert "cold" in task_keys + + +class TestCrossForEachVariableReadDetection: + """Change fix-cross-foreach-variable-read-warning (P1): VAREX3-003.""" + + def test_set_var_in_foreach_read_by_sibling_emits_setup_task(self): + """When a SetVariable for `continue` lives only inside a ForEach and + a sibling IfCondition reads @variables('continue'), prepare_workflow + must emit a manual_variable_rollup SetupTask naming the variable + and the parent ForEach.""" + # Inside the ForEach: a SetVariable that mutates `continue`. + setter = SetVariableActivity( + **_make_base("Mark Continue", "mark_continue"), + variable_name="continue", + variable_value="false", + ) + loop = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[setter], + concurrency=2, + ) + sibling = IfConditionActivity( + **_make_base("CheckCont", "check_cont"), + op="EQUAL_TO", + left="@variables('continue')", + right="true", + if_true_activities=[], + if_false_activities=[], + ) + pipeline = Pipeline(name="cross_foreach_pipe", tasks=[loop, sibling]) + wf = prepare_workflow(pipeline) + rollups = [ + st for st in wf.setup_tasks if st.type == "manual_variable_rollup" + ] + assert len(rollups) == 1 + config = rollups[0].config + assert config["variable_name"] == "continue" + assert config["parent_foreach"] == "loop" + + def test_set_var_with_parent_scope_setter_emits_no_warning(self): + """When the variable is ALSO set at the parent scope, no warning.""" + parent_setter = SetVariableActivity( + **_make_base("Init", "init_cont"), + variable_name="continue", + variable_value="true", + ) + inner_setter = SetVariableActivity( + **_make_base("ResetInside", "reset_inside"), + variable_name="continue", + variable_value="false", + ) + loop = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[inner_setter], + concurrency=2, + ) + sibling = IfConditionActivity( + **_make_base("CheckCont", "check_cont"), + op="EQUAL_TO", + left="@variables('continue')", + right="true", + if_true_activities=[], + if_false_activities=[], + ) + pipeline = Pipeline( + name="parent_scope_pipe", + tasks=[parent_setter, loop, sibling], + ) + wf = prepare_workflow(pipeline) + rollups = [ + st for st in wf.setup_tasks if st.type == "manual_variable_rollup" + ] + assert rollups == [] + + +class TestManualScheduleTimeOfDaySetupTask: + """C-36 (SCHED4-001): a pipeline schedule whose recurrence carries + hours/minutes/weekDays that periodic can't encode emits a + manual_schedule_time_of_day SetupTask so SETUP.md can flag it.""" + + def test_periodic_schedule_with_time_of_day_emits_setup_task(self): + pipeline = Pipeline( + name="every_three_days", + tasks=[], + schedule={ + "kind": "periodic", + "interval": 3, + "unit": "DAYS", + "pause_status": "UNPAUSED", + "time_of_day_note": {"hours": [2]}, + }, + ) + wf = prepare_workflow(pipeline) + tasks = [st for st in wf.setup_tasks if st.type == "manual_schedule_time_of_day"] + assert len(tasks) == 1 + config = tasks[0].config + assert config["pipeline"] == "every_three_days" + assert config["time_of_day_note"] == {"hours": [2]} + class TestExecutePipelinePreparer: def test_prepare_execute_pipeline_task(self): @@ -633,8 +1027,28 @@ def test_prepare_switch_multi_case_chains_conditions(self): # Default hangs off the last case's outcome=false. assert extra_by_key["default_wait"]["depends_on"] == [{"task_key": "route_case_prod", "outcome": "false"}] - def test_prepare_switch_resolves_variables_expression(self): - """Switch on @variables('x') resolves to a DAB task value ref.""" + def test_resolve_switch_on_expression_is_idempotent_for_dab_refs(self): + """C-13 (CF-iter2-004): an already-resolved {{tasks.X.values.Y}} ref + passes through resolve_switch_on_expression unchanged rather than + being re-resolved with an empty context (which would strip globals + and variable_cache).""" + from flowx.preparer.activity_preparers.switch import ( + resolve_switch_on_expression, + ) + + assert resolve_switch_on_expression("{{tasks.x.values.x}}") == "{{tasks.x.values.x}}" + assert resolve_switch_on_expression("{{job.parameters.env}}") == "{{job.parameters.env}}" + # A bare literal passes through unchanged. + assert resolve_switch_on_expression("hello") == "hello" + # Translator-side bridge placeholder is preserved. + assert resolve_switch_on_expression("__BRIDGE__::result") == "__BRIDGE__::result" + + def test_prepare_switch_unresolved_variable_left_as_raw(self): + """C-05 (VAREX-002): when no setter for the variable is known the + Switch on-expression is left as the raw ``@variables(...)`` string + rather than producing a self-referential dangling task ref. C-07 + will eventually bridge this through a hidden task; until then the + raw string is preserved so a SETUP.md note can flag it.""" inner = WaitActivity(**_make_base("CaseWait", "case_wait"), wait_time_seconds=1) activity = SwitchActivity( **_make_base("Route", "route"), @@ -644,9 +1058,9 @@ def test_prepare_switch_resolves_variables_expression(self): ) prepared = prepare_activity(activity) cond = prepared.task["condition_task"] - # Should be resolved to a DAB ref (fallback: variable name used as task key) - assert "tasks." in cond["left"] - assert "sourceType" in cond["left"] + # Without a setter the raw ADF expression is preserved (no + # dangling {{tasks.sourceType.values.sourceType}} placeholder). + assert cond["left"] == "@variables('sourceType')" def test_prepare_switch_resolves_pipeline_param(self): """Switch on @pipeline().parameters.X resolves to a DAB job parameter ref.""" @@ -888,6 +1302,55 @@ def test_prepare_workflow_with_dependencies(self): assert "depends_on" in second_task assert second_task["depends_on"][0]["task_key"] == "first" + def test_prepare_workflow_collects_cluster_hints_from_nested_activities(self): + """C-04 (NB-ITER2-4 / LSC2-001): cluster hints from activities + nested inside IfCondition / Switch / ForEach must surface in the + workflow-level cluster_hints aggregation so the default-cluster + inference picks the LS-intended node type.""" + nested_nb = NotebookActivity( + **_make_base("Inner", "inner"), + notebook_path="/Shared/inner", + ) + # Override the cluster after construction since _make_base sets it None. + nested_nb.cluster = { + "spark_version": "16.4.x-scala2.12", + "num_workers": 0, + "node_type_id": "Standard_D8s_v3", + } + if_act = IfConditionActivity( + **_make_base("IfCond", "ifcond"), + op="EQUAL", + left="x", + right="y", + if_true_activities=[nested_nb], + ) + pipeline = Pipeline(name="nested_cluster", tasks=[if_act]) + wf = prepare_workflow(pipeline) + node_types = [hint.get("node_type_id") for hint in wf.cluster_hints] + assert "Standard_D8s_v3" in node_types + + def test_prepare_workflow_collects_cluster_hints_from_switch_default_branch(self): + """C-04: Switch default_activities cluster hints surface too.""" + nested_nb = NotebookActivity( + **_make_base("DefaultBranch", "default_branch"), + notebook_path="/Shared/default", + ) + nested_nb.cluster = { + "spark_version": "16.4.x-scala2.12", + "num_workers": 0, + "node_type_id": "Standard_D16s_v3", + } + switch_act = SwitchActivity( + **_make_base("Switch", "switch"), + on_expression="x", + cases=[], + default_activities=[nested_nb], + ) + pipeline = Pipeline(name="switch_default", tasks=[switch_act]) + wf = prepare_workflow(pipeline) + node_types = [hint.get("node_type_id") for hint in wf.cluster_hints] + assert "Standard_D16s_v3" in node_types + def test_prepare_workflow_with_retries(self): """Retry settings are carried through.""" pipeline = Pipeline( diff --git a/tests/unit/test_prereqs_writer.py b/tests/unit/test_prereqs_writer.py new file mode 100644 index 0000000..a04fe0a --- /dev/null +++ b/tests/unit/test_prereqs_writer.py @@ -0,0 +1,79 @@ +"""Unit tests for flowx.bundler.prereqs_writer.""" + +from __future__ import annotations + +from flowx.bundler.prereqs_writer import build_prereqs, render_setup_md +from flowx.models.dab import DabNotebook, SecretInstruction + + +class TestSecretsUnion: + """Change fix-setup-md-secrets-union-with-secret-instructions (P1): LSC3-006.""" + + def test_workflow_secrets_union_with_notebook_scanned_scopes(self): + """SETUP.md secrets section must list both notebook-scanned and + workflow.secrets sources without duplicating any (scope, key) pair.""" + notebooks = [ + DabNotebook( + relative_path="notebooks/x.py", + content=( + "# Databricks notebook source\n" + "auth_token = dbutils.secrets.get(" + "scope=\"lakeh_a_pl_operational_sendMail\", key=\"auth-credential\")\n" + ), + ) + ] + secret_instructions = [ + SecretInstruction( + scope="lakeh_ls_keyvault", + key="adapp-clientSecret", + value_source="Azure Key Vault: lakeh-kv/adapp-clientSecret", + ), + # Same pair as the notebook scan -- must not be duplicated. + SecretInstruction( + scope="lakeh_a_pl_operational_sendMail", + key="auth-credential", + value_source="duplicate of notebook scan", + ), + ] + prereqs = build_prereqs( + notebooks=notebooks, + tasks=[], + known_bundle_jobs=set(), + secret_instructions=secret_instructions, + ) + # Both scopes present in the union, no duplicate keys. + assert "lakeh_ls_keyvault" in prereqs.secrets + assert "adapp-clientSecret" in prereqs.secrets["lakeh_ls_keyvault"] + assert "lakeh_a_pl_operational_sendMail" in prereqs.secrets + assert prereqs.secrets["lakeh_a_pl_operational_sendMail"] == {"auth-credential"} + + def test_setup_md_lists_unioned_secrets(self): + """SETUP.md Option A renders every (scope, key) from the union.""" + notebooks = [ + DabNotebook( + relative_path="notebooks/x.py", + content=( + "auth_token = dbutils.secrets.get(" + "scope=\"scope_from_notebook\", key=\"key_from_notebook\")\n" + ), + ) + ] + secret_instructions = [ + SecretInstruction( + scope="scope_from_workflow", + key="key_from_workflow", + value_source="Azure Key Vault", + ), + ] + prereqs = build_prereqs( + notebooks=notebooks, + tasks=[], + known_bundle_jobs=set(), + secret_instructions=secret_instructions, + ) + md = render_setup_md(prereqs, bundle_name="test_bundle") + # Both (scope, key) pairs must appear in the rendered SETUP.md. + assert "scope_from_notebook" in md + assert "key_from_notebook" in md + assert "scope_from_workflow" in md + assert "key_from_workflow" in md diff --git a/tests/unit/test_resolve_field.py b/tests/unit/test_resolve_field.py index a025c81..c965066 100644 --- a/tests/unit/test_resolve_field.py +++ b/tests/unit/test_resolve_field.py @@ -68,8 +68,9 @@ def test_activity_output_ref(self): assert "tasks.Lookup.values.cnt" in result def test_boolean_value(self): - result = resolve_field(True, _ctx()) - assert result == "True" + # VAREX3-002: Python bool renders lowercase 'true'/'false' to match ADF. + assert resolve_field(True, _ctx()) == "true" + assert resolve_field(False, _ctx()) == "false" def test_variables_with_context(self): result = resolve_field("@variables('runDate')", _ctx(runDate="SetRunDate")) diff --git a/tests/unit/test_translators.py b/tests/unit/test_translators.py index bab80ab..a4ffa7d 100644 --- a/tests/unit/test_translators.py +++ b/tests/unit/test_translators.py @@ -50,6 +50,7 @@ def _base_kwargs(name: str = "test_activity") -> dict[str, Any]: "min_retry_interval_millis": None, "depends_on": None, "cluster": None, + "existing_cluster_id": None, } @@ -176,6 +177,58 @@ def test_translate_notebook_no_params(self): assert isinstance(result, NotebookActivity) assert result.base_parameters == {} + def test_translate_notebook_resolves_library_with_globals(self): + """C-01 (NB-ITER2-1, LSC2-004): @concat of literals collapses to a + literal jar path so the library install succeeds.""" + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Run NB", + "DatabricksNotebook", + { + "notebookPath": "/Shared/x", + "libraries": [ + { + "jar": ( + "@concat('/Volumes/x/', " + "pipeline().globalParameters.libFileName)" + ) + } + ], + }, + ) + # Context with global parameter so the @concat resolves. + ctx = TranslationContext( + global_parameters=MappingProxyType({"libFileName": "my-job.jar"}), + ) + result = translate(activity, _base_kwargs(), ctx, _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + # With C-01 the concat collapses to a literal string -- the bundle + # YAML carries the resolved jar path directly instead of a Python + # source string. + assert result.libraries == [{"jar": "/Volumes/x/my-job.jar"}] + + def test_translate_notebook_resolves_pipeline_param_in_library(self): + """Library entry referencing a single pipeline parameter resolves to a literal.""" + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Run NB", + "DatabricksNotebook", + { + "notebookPath": "/Shared/x", + "libraries": [ + {"jar": "@pipeline().globalParameters.libPath"}, + ], + }, + ) + ctx = TranslationContext( + global_parameters=MappingProxyType({"libPath": "/Volumes/my.jar"}), + ) + result = translate(activity, _base_kwargs(), ctx, _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.libraries == [{"jar": "/Volumes/my.jar"}] + def test_translate_notebook_passes_libraries_through(self): from flowx.translator.activity_translators.notebook import translate @@ -195,6 +248,59 @@ def test_translate_notebook_passes_libraries_through(self): assert isinstance(result, NotebookActivity) assert result.libraries == libraries + def test_translate_notebook_dynamic_path_marks_unresolved(self): + """C-28 (NB-ITER4-001): an expression notebookPath is captured as + ``notebook_path_unresolved`` so the preparer emits a dispatch stub.""" + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Dispatch", + "DatabricksNotebook", + { + "notebookPath": { + "value": "@trim(json(activity('cfg').output.firstRow).notebook_path)", + "type": "Expression", + }, + "baseParameters": {"env": "dev"}, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.notebook_path_unresolved is True + assert result.notebook_path == "" + assert "@trim" in (result.notebook_path_expression or "") + + def test_translate_notebook_unresolved_library_captured(self): + """C-30 (NB-ITER4-003): library jar/whl entries whose @concat + references a missing globalParameter surface as + ``unresolved_libraries`` so SETUP.md can flag them.""" + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Run NB", + "DatabricksNotebook", + { + "notebookPath": "/Shared/x", + "libraries": [ + { + "jar": ( + "@concat('/Volumes/x/', " + "pipeline().globalParameters.proj4jLibFileName)" + ) + } + ], + }, + ) + ctx = TranslationContext() + result = translate(activity, _base_kwargs(), ctx, _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + # The unresolved entry is captured with the missing identifier. + assert len(result.unresolved_libraries) == 1 + entry = result.unresolved_libraries[0] + assert entry["type"] == "jar" + assert "proj4jLibFileName" in entry["expression"] + assert "proj4jLibFileName" in entry["missing"] + def test_translate_notebook_captures_utcnow_approximation(self): from flowx.translator.activity_translators.notebook import translate @@ -280,6 +386,300 @@ def test_existing_cluster_id_none_when_linked_service_uses_new_cluster(self): kwargs = _build_base_kwargs(activity, definitions) assert kwargs["existing_cluster_id"] is None + def test_linked_service_parameter_overrides_cluster_version(self): + """Change linked-service-parameter-resolution (P0): NB-4, LSC-001.""" + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="APP0001_ls_databricks", + type="AzureDatabricks", + properties={ + "parameters": { + "clusterVersion": { + "type": "string", + "defaultValue": "16.4.x-scala2.12", + }, + }, + "typeProperties": { + "newClusterVersion": "@linkedService().clusterVersion", + "newClusterNumOfWorker": "1", + "newClusterNodeType": "Standard_D4s_v3", + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"APP0001_ls_databricks": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference( + reference_name="APP0001_ls_databricks", + parameters={"clusterVersion": "16.4.x-scala2.12"}, + ), + ) + kwargs = _build_base_kwargs(activity, definitions) + assert kwargs["cluster"] is not None + # No literal ADF expression should leak into the cluster spec. + assert kwargs["cluster"]["spark_version"] == "16.4.x-scala2.12" + # num_workers='1' must coerce to int. + assert kwargs["cluster"]["num_workers"] == 1 + assert isinstance(kwargs["cluster"]["num_workers"], int) + + def test_parameter_default_coerces_bool_string_to_real_bool(self): + """Change expression-resolver-bool-and-numeric-coercion (P1): VAR-006.""" + from flowx.translator.engine import _coerce_parameter_default + + assert _coerce_parameter_default("false", "Bool") is False + assert _coerce_parameter_default("True", "Bool") is True + assert _coerce_parameter_default("FALSE", "boolean") is False + + def test_parameter_default_coerces_int_string_to_int(self): + from flowx.translator.engine import _coerce_parameter_default + + assert _coerce_parameter_default("42", "Int") == 42 + assert _coerce_parameter_default(42, "Int") == 42 + + def test_parameter_default_string_left_alone(self): + from flowx.translator.engine import _coerce_parameter_default + + assert _coerce_parameter_default("hello", "String") == "hello" + + def test_dependency_multi_condition_succeeded_and_failed_maps_to_completed(self): + """Change dependency-multi-condition-mapping (P1): CF-004.""" + from flowx.translator.engine import _map_dependency_conditions + + assert _map_dependency_conditions(["Succeeded"]) == "Succeeded" + assert _map_dependency_conditions(["Failed"]) == "Failed" + assert _map_dependency_conditions(["Completed"]) == "Completed" + assert _map_dependency_conditions(["Skipped"]) == "Skipped" + # [Succeeded, Failed] semantics = "run regardless" -> Completed + assert _map_dependency_conditions(["Succeeded", "Failed"]) == "Completed" + # [Succeeded, Skipped] -> Skipped wins (ALL_DONE downstream) + assert _map_dependency_conditions(["Succeeded", "Skipped"]) == "Skipped" + # [Failed] (multi-element with same) handled in single-item branch. + assert _map_dependency_conditions([]) is None + assert _map_dependency_conditions(None) is None + + def test_ls_param_expression_wrapper_unwrapped_in_custom_tags(self): + """C-02 (NB-ITER2-2 / LSC2-003): Expression-dict-wrapped LS params + must collapse to plain scalars in cluster fields like custom_tags.""" + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="LS", + type="AzureDatabricks", + properties={ + "parameters": { + "digitalCase": {"type": "string", "defaultValue": "APP0001"}, + }, + "typeProperties": { + "newClusterVersion": "16.4.x-scala2.12", + "newClusterNumOfWorker": 0, + "newClusterNodeType": "Standard_D4s_v3", + "newClusterCustomTags": { + "DigitalCase": "@linkedService().digitalCase", + }, + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"LS": linked_service}, + triggers=[], + ) + # Activity supplies the LS param as the {value, type:'Expression'} + # wrapper shape -- the same shape the ADF JSON corpus ships. + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference( + reference_name="LS", + parameters={"digitalCase": {"value": "APP0001", "type": "Expression"}}, + ), + ) + cluster = _build_base_kwargs(activity, definitions)["cluster"] + assert cluster is not None + # custom_tags must be Map[String, String] -- no dict wrapper survives. + assert cluster["custom_tags"] == {"DigitalCase": "APP0001"} + # spark_env_vars likewise stays scalar-valued. + assert "DigitalCase" in cluster["custom_tags"] + assert not isinstance(cluster["custom_tags"]["DigitalCase"], dict) + + def test_ls_param_resolved_against_factory_global_parameters(self): + """C-03 (NB-ITER2-3 / LSC2-002): activity-supplied LS param values + that reference @pipeline().globalParameters.X must collapse to the + factory-provided literal so cluster.spark_version is a real DBR.""" + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="LS", + type="AzureDatabricks", + properties={ + "parameters": { + "clusterVersion": {"type": "string", "defaultValue": "15.4.x-scala2.12"}, + }, + "typeProperties": { + "newClusterVersion": "@linkedService().clusterVersion", + "newClusterNumOfWorker": 0, + "newClusterNodeType": "Standard_D4s_v3", + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"LS": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference( + reference_name="LS", + parameters={ + "clusterVersion": { + "value": "@pipeline().globalParameters.clusterVersion", + "type": "Expression", + }, + }, + ), + ) + context = TranslationContext( + global_parameters=MappingProxyType({"clusterVersion": "16.4.x-scala2.12"}), + ) + cluster = _build_base_kwargs(activity, definitions, context=context)["cluster"] + assert cluster is not None + assert cluster["spark_version"] == "16.4.x-scala2.12" + # The raw @pipeline() expression must not leak into the cluster spec. + assert not cluster["spark_version"].startswith("@") + + def test_ls_param_resolved_against_pipeline_parameters_as_dab_ref(self): + """C-13 (NB-ITER3-002 / LSC3-003 / VAREX3-006): activity-supplied LS + param values that reference @pipeline().parameters.X must collapse to + {{job.parameters.X}} (a dab_ref), valid in custom_tags map values.""" + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="LS", + type="AzureDatabricks", + properties={ + "parameters": { + "digitalCase": {"type": "string", "defaultValue": "APP0001"}, + }, + "typeProperties": { + "newClusterVersion": "16.4.x-scala2.12", + "newClusterNumOfWorker": 0, + "newClusterNodeType": "Standard_D4s_v3", + "newClusterCustomTags": { + "DigitalCase": "@linkedService().digitalCase", + }, + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"LS": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference( + reference_name="LS", + parameters={ + "digitalCase": { + "value": "@pipeline().parameters.digitalCaseCode", + "type": "Expression", + }, + }, + ), + ) + # Pipeline parameters resolver routes via dab_ref kind, not literal. + context = TranslationContext() + cluster = _build_base_kwargs(activity, definitions, context=context)["cluster"] + assert cluster is not None + # The resolver should now substitute the dab_ref so the raw @pipeline + # expression does not leak. + assert cluster["custom_tags"]["DigitalCase"] == "{{job.parameters.digitalCaseCode}}" + assert not cluster["custom_tags"]["DigitalCase"].startswith("@") + + def test_notebook_library_resolves_pipeline_param_dab_ref(self): + """C-13 (NB-ITER3-004): a jar path referencing @pipeline().parameters.X + collapses to {{job.parameters.X}} in the emitted library entry.""" + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Run NB", + "DatabricksNotebook", + { + "notebookPath": "/Shared/x", + "libraries": [ + {"jar": "@pipeline().parameters.libName"}, + ], + }, + ) + ctx = TranslationContext() + result = translate(activity, _base_kwargs(), ctx, _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.libraries == [{"jar": "{{job.parameters.libName}}"}] + + def test_extended_cluster_fields_propagated(self): + """Change linked-service-cluster-field-coverage (P1): NB-3, LSC-003.""" + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="LS", + type="AzureDatabricks", + properties={ + "typeProperties": { + "newClusterVersion": "16.4.x-scala2.12", + "newClusterNumOfWorker": 0, + "newClusterNodeType": "Standard_D4s_v3", + "newClusterDriverNodeType": "Standard_D8s_v3", + "newClusterSparkEnvVars": {"PYSPARK_PYTHON": "/databricks/python3/bin/python3"}, + "newClusterCustomTags": {"DigitalCase": "MyCase"}, + "newClusterInitScripts": [{"workspace": {"destination": "/init.sh"}}], + "dataSecurityMode": "SINGLE_USER", + "clusterLogConf": {"dbfs": {"destination": "dbfs:/cluster-logs"}}, + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"LS": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference(reference_name="LS"), + ) + cluster = _build_base_kwargs(activity, definitions)["cluster"] + assert cluster is not None + assert cluster["driver_node_type_id"] == "Standard_D8s_v3" + assert cluster["spark_env_vars"]["PYSPARK_PYTHON"] == "/databricks/python3/bin/python3" + assert cluster["custom_tags"]["DigitalCase"] == "MyCase" + assert cluster["init_scripts"] == [{"workspace": {"destination": "/init.sh"}}] + assert cluster["data_security_mode"] == "SINGLE_USER" + assert cluster["cluster_log_conf"] == {"dbfs": {"destination": "dbfs:/cluster-logs"}} + class TestSparkJarTranslator: def test_translate_spark_jar(self): @@ -365,6 +765,202 @@ def test_translate_lookup_all_rows(self): assert isinstance(result, LookupActivity) assert result.first_row_only is False + def test_translate_lookup_resolves_json_file_dataset(self): + """Change lookup-file-dataset-support (P0).""" + from flowx.models.adf_ast import AdfDataset + from flowx.translator.activity_translators.lookup import translate + + json_dataset = AdfDataset( + name="ConfigDataset", + type="Json", + properties={ + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileSystem": "configs", + "folderPath": "lookup", + "fileName": "tables.json", + }, + "formatSettings": {"multiLineJson": True}, + }, + }, + linked_service_name="LS", + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={"ConfigDataset": json_dataset}, + linked_services={}, + triggers=[], + ) + activity = _make_activity( + "Read_Configuration", + "Lookup", + { + "source": {"type": "JsonSource"}, + "dataset": { + "referenceName": "ConfigDataset", + "type": "DatasetReference", + }, + "firstRowOnly": False, + }, + ) + result = translate(activity, _base_kwargs(), _context(), definitions) + assert isinstance(result, LookupActivity) + assert result.source_type == "JsonSource" + assert result.first_row_only is False + # Source properties carry the dataset type plus location bits. + assert result.source_properties is not None + assert result.source_properties["dataset_type"] == "Json" + assert result.source_properties["container"] == "configs" + assert result.source_properties["file_name"] == "tables.json" + assert result.source_properties.get("multiLineJson") is True + + + def test_translate_lookup_substitutes_dataset_parameter_refs(self): + """C-47 (LSC5-001): a file Lookup whose dataset folderPath references + ``dataset().X`` substitutes the dataset reference's parameter bindings + so the baked path carries no literal ``dataset(`` expression.""" + from flowx.models.adf_ast import AdfDataset + from flowx.translator.activity_translators.lookup import translate + + ds = AdfDataset( + name="arq_ds", + type="Json", + properties={ + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileSystem": "configs", + "folderPath": { + "value": "@toLower(dataset().digitalCase)", + "type": "Expression", + }, + "fileName": {"value": "@dataset().fileName", "type": "Expression"}, + }, + }, + }, + linked_service_name="LS", + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={"arq_ds": ds}, + linked_services={}, + triggers=[], + ) + activity = _make_activity( + "Read_Arq", + "Lookup", + { + "source": {"type": "JsonSource"}, + "dataset": { + "referenceName": "arq_ds", + "type": "DatasetReference", + "parameters": { + "digitalCase": "@pipeline().parameters.digitalCaseCode", + "fileName": "@pipeline().parameters.fileName", + }, + }, + "firstRowOnly": True, + }, + ) + result = translate(activity, _base_kwargs(), _context(), definitions) + assert isinstance(result, LookupActivity) + assert result.source_properties is not None + folder = result.source_properties.get("folder_path", "") + filename = result.source_properties.get("file_name", "") + # The dataset() reference is gone; the pipeline-param binding takes over. + assert "dataset(" not in folder + assert "dataset(" not in filename + assert "{{job.parameters.digitalCaseCode}}" in folder + + +class TestLookupCaseInsensitiveAndLinkedService: + """Change fix-dataset-and-linked-service-case-insensitive-lookup (P1): LSC3-005.""" + + def test_lookup_resolves_dataset_case_insensitively(self): + """ADF identifiers are case-insensitive; a pipeline referencing + 'app0001_a_ds_conf_json' must resolve dataset 'APP0001_a_ds_conf_json'.""" + from flowx.models.adf_ast import AdfDataset, AdfLinkedService + from flowx.translator.activity_translators.lookup import translate + + ds = AdfDataset( + name="APP0001_a_ds_conf_json", + type="Json", + properties={ + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileSystem": "configext", + "folderPath": "settings", + "fileName": "tables.json", + }, + }, + }, + linked_service_name="LS_ABFSS", + ) + ls = AdfLinkedService( + name="LS_ABFSS", + type="AzureBlobFS", + properties={ + "typeProperties": { + "url": "abfss://configext@myacct.dfs.core.windows.net", + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={"APP0001_a_ds_conf_json": ds}, + linked_services={"LS_ABFSS": ls}, + triggers=[], + ) + # NOTE: lowercase reference name in the activity. + activity = _make_activity( + "Read_Conf", + "Lookup", + { + "source": {"type": "JsonSource"}, + "dataset": { + "referenceName": "app0001_a_ds_conf_json", + "type": "DatasetReference", + }, + "firstRowOnly": True, + }, + ) + result = translate(activity, _base_kwargs(), _context(), definitions) + assert isinstance(result, LookupActivity) + assert result.source_properties is not None + assert result.source_properties["dataset_type"] == "Json" + assert result.source_properties["container"] == "configext" + # Linked-service URL surfaces so the code generator can build abfss://... + assert result.source_properties["linked_service_url"] == ( + "abfss://configext@myacct.dfs.core.windows.net" + ) + + def test_generated_file_lookup_notebook_uses_abfss_path(self): + """LSC3-005 end-to-end: generated file-lookup notebook ships a real + abfss:// default path instead of an empty widget fallback.""" + from flowx.preparer.code_generator import generate_lookup_notebook + + base = _base_kwargs("Read_Conf") + base.pop("existing_cluster_id", None) + activity = LookupActivity( + **base, + source_type="JsonSource", + source_properties={ + "dataset_type": "Json", + "container": "configext", + "folder_path": "settings", + "file_name": "tables.json", + "linked_service_url": "abfss://configext@myacct.dfs.core.windows.net", + }, + first_row_only=True, + ) + content = generate_lookup_notebook(activity) + assert "abfss://configext@myacct.dfs.core.windows.net" in content + assert "tables.json" in content + # spark.sql('') sentinel must not appear for file-source lookups. + assert "spark.sql('')" not in content + class TestWebActivityTranslator: def test_translate_web_activity_get(self): @@ -437,6 +1033,37 @@ def test_translate_execute_pipeline(self): assert result.parameters == {"date": "2024-01-01"} assert result.wait_on_completion is True + def test_translate_execute_pipeline_drops_notebook_code_parameters(self): + """C-09 (VAREX-001): an ExecutePipeline parameter value that resolves + to notebook_code (e.g. @concat('x', pipeline().parameters.Y)) must NOT + ride through as a literal Python source string -- it's dropped from + the parameters dict and surfaced via parameter_approximations.""" + from flowx.translator.activity_translators.execute_pipeline import translate + + activity = _make_activity( + "Run Child", + "ExecutePipeline", + { + "pipeline": {"referenceName": "child_pipeline", "type": "PipelineReference"}, + "parameters": { + "ok_value": "@pipeline().parameters.env", # dab_ref -- kept + "bad_value": { + "value": "@concat('json: ', pipeline().parameters.configFile)", + "type": "Expression", + }, + }, + "waitOnCompletion": True, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, ExecutePipelineActivity) + assert result.parameters == {"ok_value": "{{job.parameters.env}}"} + # bad_value surfaced as a parameter_approximation for SETUP.md. + approximations = result.parameter_approximations + assert any(a.get("widget_name") == "bad_value" for a in approximations) + # Literal Python source must NOT leak into the parameters dict. + assert "dbutils.widgets.get" not in str(result.parameters) + class TestDatabricksJobTranslator: def test_translate_databricks_job(self): @@ -562,6 +1189,56 @@ def test_translate_foreach_sequential(self): assert isinstance(result, ForEachActivity) assert result.concurrency == 1 + def test_translate_foreach_propagates_globals_to_child_context(self): + """C-13 (NB-ITER3-001 / CF3-002 / LSC3-004): ForEach child context + must carry global_parameters and linked_service_parameters so inner + notebooks resolve @pipeline().globalParameters.X to literals.""" + from flowx.translator.activity_translators.for_each import translate + + # Inner notebook whose library jar references a global parameter. + inner_activity = _make_activity( + "InnerNB", + "DatabricksNotebook", + { + "notebookPath": "/Shared/nb", + "libraries": [{"jar": "@pipeline().globalParameters.libPath"}], + }, + ) + activity = _make_activity( + "Loop", + "ForEach", + {"items": "@activity('GetList').output.value"}, + activities=[inner_activity], + ) + + # The parent context carries the global parameter the inner notebook + # needs. We use the real notebook translator inside our mock callback + # so the inner activity is processed exactly as the engine would. + from flowx.translator.activity_translators.notebook import translate as translate_nb + + def _mock_translate(activities, ctx, defs): + results: list[Any] = [] + for child in activities: + results.append(translate_nb(child, _base_kwargs(child.name), ctx, defs)) + return results, ctx + + ctx = TranslationContext( + global_parameters=MappingProxyType({"libPath": "/Volumes/my.jar"}), + ) + result, _ = translate( + activity, + _base_kwargs("Loop"), + ctx, + _EMPTY_DEFS, + translate_activities_fn=_mock_translate, + ) + assert isinstance(result, ForEachActivity) + inner = result.inner_activities[0] + assert isinstance(inner, NotebookActivity) + # The jar should resolve to the literal from global_parameters, not the + # raw @pipeline() expression. + assert inner.libraries == [{"jar": "/Volumes/my.jar"}] + class TestIfConditionTranslator: def test_translate_if_condition_equals(self): @@ -610,6 +1287,176 @@ def test_translate_if_condition_greater(self): assert "tasks.Copy.values.rowsCopied" in result.left assert result.right == "0" + def test_translate_if_condition_empty_bridges_via_notebook(self): + """C-07 (CF-iter2-001 / VAREX-003): @empty(...) operand routes through + a bridge SetVariable task rather than shipping as a raw ADF expression.""" + from flowx.translator.activity_translators.if_condition import translate + + activity = _make_activity( + "Branch", + "IfCondition", + { + "expression": { + "type": "Expression", + "value": "@empty(pipeline().parameters.X)", + } + }, + ) + result, _ = translate(activity, _base_kwargs("Branch"), _context(), _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + # Bridge populated and right operand is False (no longer the legacy '0'). + assert result.bridge_notebook_code is not None + assert "len(" in result.bridge_notebook_code # @empty -> (len(X) == 0) + assert result.op == "NOT_EQUAL" + assert result.right == "False" + # Left operand is a translator placeholder the preparer rewrites. + assert result.left.startswith("__BRIDGE__::") + + def test_translate_if_condition_boolean_variable_uses_lowercase_false(self): + """C-32 (CF4-002): the truthy fallback path emits ``right='false'`` (not + ``'0'``) when the operand is a known-Boolean variable, since C-21 + SetVariable now writes lowercase ``'true'/'false'`` strings.""" + from flowx.translator.activity_translators.if_condition import translate + + # Seed the context with a Boolean default-valued variable so the + # truthy fallback knows the operand renders as 'true'/'false'. + ctx = _context().with_variable("continue", "_init_continue", dab_ref_value="true") + activity = _make_activity( + "Branch", + "IfCondition", + {"expression": {"type": "Expression", "value": "@variables('continue')"}}, + ) + result, _ = translate(activity, _base_kwargs("Branch"), ctx, _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + assert result.op == "NOT_EQUAL" + # C-32: lowercase 'false' (compatible with C-21 SetVariable output); + # was '0' before this change. + assert result.right == "false" + + def test_translate_if_condition_boolean_variable_by_declared_type(self): + """C-41 (CF5-001): a Boolean variable seeded only by a literal default + init task never populates variable_value_cache as a dab_ref, so the + IfCondition fallback must fall back to the declared type and still emit + ``right='false'`` (not the always-true ``'0'``).""" + from flowx.translator.activity_translators.if_condition import translate + + # No dab_ref value cached — only the declared Boolean type is known. + ctx = _context().with_variable_types({"continue": "Boolean"}) + activity = _make_activity( + "Branch", + "IfCondition", + {"expression": {"type": "Expression", "value": "@variables('continue')"}}, + ) + result, _ = translate(activity, _base_kwargs("Branch"), ctx, _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + assert result.op == "NOT_EQUAL" + assert result.right == "false" + + def test_translate_if_condition_boolean_variable_bridges_when_default_literal_known(self): + """C-43 (CF5-001): when a Boolean variable carries a seeded literal + default, the IfCondition operand is recomputed locally via a + BridgeRequest (``left='__BRIDGE__::...'`` + ``bridge_notebook_code``) + rather than left as a parent-job task-value ref the bundler would + blank. This keeps the operand local so an inner-ForEach condition + survives the dangling-ref safety net.""" + from flowx.translator.activity_translators.if_condition import translate + + # Declared Boolean type AND a seeded literal default -> bridge. + ctx = _context().with_variable_types( + {"continue": "Boolean"}, default_literals={"continue": "true"} + ) + activity = _make_activity( + "Branch", + "IfCondition", + {"expression": {"type": "Expression", "value": "@variables('continue')"}}, + ) + result, _ = translate(activity, _base_kwargs("Branch"), ctx, _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + assert result.op == "NOT_EQUAL" + assert result.left.startswith("__BRIDGE__::") + assert result.right == "False" + assert result.bridge_notebook_code == "True" + + def test_translate_if_condition_not_of_function_uses_false_right(self): + """C-15 (CF3-003 / VAREX3-004): @not() produces a bridge + task value compared against 'False', not '' or '0', so the condition + can actually evaluate to FALSE against the Python bool the bridge writes.""" + from flowx.translator.activity_translators.if_condition import translate + + activity = _make_activity( + "Branch", + "IfCondition", + { + "expression": { + "type": "Expression", + # @not(empty(...)) — the bridge writes a Python bool for + # the comparison; right operand must be 'False'. + "value": "@not(empty(pipeline().parameters.X))", + } + }, + ) + result, _ = translate(activity, _base_kwargs("Branch"), _context(), _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + assert result.op == "NOT_EQUAL" + # When bridged, right must be 'False' (was '' under the legacy code path). + assert result.right == "False" + assert result.left.startswith("__BRIDGE__::") or result.bridge_notebook_code is not None + + def test_translate_if_condition_truthy_fallback_bridges_with_false_right(self): + """C-15 (CF3-003 / VAREX3-004): the legacy truthy fallback path emits + right='False' when the resolved operand is a bridge placeholder. + Previously emitted right='0', which the bridge's Python bool output + can never satisfy.""" + from flowx.translator.activity_translators.if_condition import translate + + # An expression with a function call that bridges (e.g. @toUpper). + activity = _make_activity( + "Branch", + "IfCondition", + { + "expression": { + "type": "Expression", + "value": "@toUpper(pipeline().parameters.X)", + } + }, + ) + result, _ = translate(activity, _base_kwargs("Branch"), _context(), _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + assert result.op == "NOT_EQUAL" + # Either bridge=task value with right='False', or the truthy path - + # both legitimate; ensure right is not '0'. + assert result.right != "0" + + +class TestIfConditionPreparer: + """C-07: preparer rewrites bridge placeholder to the real task value.""" + + def test_prepare_if_condition_emits_bridge_task(self): + from flowx.preparer.activity_preparers.if_condition import prepare + + if_act = IfConditionActivity( + name="Branch", + task_key="branch", + op="NOT_EQUAL", + left="__BRIDGE__::result", + right="False", + bridge_notebook_code="(len(dbutils.widgets.get('X')) == 0)", + bridge_required_parameters={"X": "{{job.parameters.X}}"}, + ) + prepared = prepare(if_act) + # A bridge task is prepended ahead of the condition. + bridge_tasks = [t for t in prepared.extra_tasks if t.get("task_key", "").endswith("_bridge")] + assert len(bridge_tasks) == 1 + bridge_task = bridge_tasks[0] + assert "notebook_task" in bridge_task + assert bridge_task["notebook_task"]["base_parameters"] == {"X": "{{job.parameters.X}}"} + # Condition operand now references the bridge task value. + cond = prepared.task["condition_task"] + assert cond["left"] == "{{tasks.branch_bridge.values.result}}" + assert cond["right"] == "False" + # Condition depends on the bridge task. + assert any(dep.get("task_key") == "branch_bridge" for dep in prepared.task.get("depends_on") or []) + class TestSetVariableTranslator: def test_translate_set_variable_literal(self): @@ -629,6 +1476,36 @@ def test_translate_set_variable_literal(self): # Context should have the variable mapped assert context.get_variable_task_key("status") == "Set_Status" + def test_translate_set_variable_return_value_pairs_resolves_inner(self): + """C-42 (VAREX5-001): a Set Pipeline Return Value list-of-pairs value + whose inner expression references a resolvable variable lowers to a + dab_ref task-value reference instead of being stringified and blanked.""" + from flowx.translator.activity_translators.set_variable import translate + + # Seed the referenced variable so @variables('executionOutputs') + # resolves to its setter task value. + ctx = _context().with_variable("executionOutputs", "set_outputs") + activity = _make_activity( + "Set Return", + "SetVariable", + { + "variableName": "result", + "value": [ + { + "key": "result", + "value": { + "type": "Expression", + "content": "@variables('executionOutputs')", + }, + } + ], + }, + ) + result, _ = translate(activity, _base_kwargs("Set_Return"), ctx, _EMPTY_DEFS) + assert isinstance(result, SetVariableActivity) + assert result.value_kind == "dab_ref" + assert "{{tasks." in result.variable_value + def test_translate_set_variable_utcnow(self): from flowx.translator.activity_translators.set_variable import translate @@ -644,6 +1521,54 @@ def test_translate_set_variable_utcnow(self): assert result.value_kind == "dab_ref" assert result.variable_value == "{{job.start_time.iso_date}}" + def test_translate_set_variable_split_subscript_lowers_to_notebook_code(self): + """C-33 (VAREX4-001): ``split(...)[N]`` previously left value_kind + stamped as 'literal' with the raw @concat text; now it lowers to + notebook_code so the SetVariable notebook computes the value.""" + from flowx.translator.activity_translators.set_variable import translate + + activity = _make_activity( + "SetPart", + "SetVariable", + { + "variableName": "year", + "value": { + "type": "Expression", + "value": "@split(pipeline().parameters.referenceDate,'/')[0]", + }, + }, + ) + result, _ = translate(activity, _base_kwargs("SetPart"), _context(), _EMPTY_DEFS) + assert isinstance(result, SetVariableActivity) + assert result.value_kind == "notebook_code" + assert result.notebook_code is not None + assert ".split(str('/'))" in result.notebook_code + + def test_translate_set_variable_unresolved_expression_blanks_value(self): + """C-33 (VAREX4-001 / CF4-003): an ADF expression the resolver + cannot lower no longer ships as value_kind='literal' with the raw + @-expression. The value is blanked, value_kind='unresolved', and + raw_expression captures the original text for SETUP.md.""" + from flowx.translator.activity_translators.set_variable import translate + + activity = _make_activity( + "SetX", + "SetVariable", + { + "variableName": "x", + "value": { + "type": "Expression", + # No handler exists for foo(...) so the resolver returns None. + "value": "@foo(pipeline().parameters.bar)", + }, + }, + ) + result, _ = translate(activity, _base_kwargs("SetX"), _context(), _EMPTY_DEFS) + assert isinstance(result, SetVariableActivity) + assert result.value_kind == "unresolved" + assert result.variable_value == "" + assert result.raw_expression == "@foo(pipeline().parameters.bar)" + def test_translate_set_variable_utcnow_unknown_format(self): from flowx.translator.activity_translators.set_variable import translate @@ -732,6 +1657,427 @@ def _mock_translate(activities, context, definitions): assert len(result.default_activities) == 1 + def test_translate_switch_function_call_routes_through_bridge(self): + """C-07 (CF-iter2-001 / CF-iter2-003): @toUpper(coalesce(...)) on the + Switch on-expression lowers to a bridge SetVariable task rather than + shipping as a raw ADF expression.""" + from flowx.translator.activity_translators.switch import translate + + activity = _make_activity( + "Route", + "Switch", + { + "on": { + "type": "Expression", + "value": "@toUpper(coalesce(item().type, 'default'))", + }, + "cases": [], + "defaultActivities": [], + }, + ) + result, _ = translate(activity, _base_kwargs("Route"), _context(), _EMPTY_DEFS) + assert isinstance(result, SwitchActivity) + assert result.bridge_notebook_code is not None + assert ".upper()" in result.bridge_notebook_code + # The on-expression carries the translator placeholder so the + # preparer can rewrite it to the bridge task value. + assert result.on_expression.startswith("__BRIDGE__::") + + +class TestVariableInitTasks: + """C-05 (VAREX-002): init SetVariable tasks for default-valued variables.""" + + def test_default_valued_variable_yields_init_task(self): + from flowx.models.adf_ast import AdfPipeline, AdfVariable + + pipeline = AdfPipeline( + name="pl_with_var_default", + activities=[ + _make_activity( + "Echo", + "DatabricksNotebook", + { + "notebookPath": "/Shared/nb", + "baseParameters": { + "uuid": {"value": "@variables('uuid')", "type": "Expression"}, + }, + }, + ), + ], + variables={"uuid": AdfVariable(type="String", default_value="seed-value")}, + ) + definitions = AdfDefinitions( + pipelines=[pipeline], datasets={}, linked_services={}, triggers=[] + ) + report = translate_pipeline(pipeline, definitions) + # An init task is prepended before the regular activities. + task_keys = [t.task_key for t in report.pipeline.tasks] + assert "_init_uuid" in task_keys + init_task = next(t for t in report.pipeline.tasks if t.task_key == "_init_uuid") + assert isinstance(init_task, SetVariableActivity) + assert init_task.variable_name == "uuid" + # Downstream @variables('uuid') routes through the init task value. + notebook_task = next(t for t in report.pipeline.tasks if t.name == "Echo") + assert isinstance(notebook_task, NotebookActivity) + assert notebook_task.base_parameters["uuid"] == "{{tasks._init_uuid.values.uuid}}" + + def test_default_valued_boolean_variable_renders_lowercase(self): + """VAREX3-002: Boolean variable default ``True`` must serialise as + the lowercase string 'true' so downstream ADF + ``@equals(variables('continue'), true)`` evaluates consistently. + Python's title-case ``'True'`` silently inverted comparisons.""" + from flowx.models.adf_ast import AdfPipeline, AdfVariable + + pipeline = AdfPipeline( + name="pl_bool_var", + activities=[], + variables={ + "continue_t": AdfVariable(type="Boolean", default_value=True), + "continue_f": AdfVariable(type="Boolean", default_value=False), + }, + ) + definitions = AdfDefinitions( + pipelines=[pipeline], datasets={}, linked_services={}, triggers=[] + ) + report = translate_pipeline(pipeline, definitions) + init_true = next(t for t in report.pipeline.tasks if t.task_key == "_init_continue_t") + init_false = next(t for t in report.pipeline.tasks if t.task_key == "_init_continue_f") + assert isinstance(init_true, SetVariableActivity) + assert isinstance(init_false, SetVariableActivity) + assert init_true.variable_value == "true" + assert init_false.variable_value == "false" + + def test_set_variable_with_raw_bool_value_renders_lowercase(self): + """VAREX3-002: a SetVariable activity carrying a raw Python ``False`` + as its typeProperties.value must serialise as 'false' (lowercase), + not 'False' (title-case).""" + from flowx.models.adf_ast import AdfPipeline + + pipeline = AdfPipeline( + name="pl_set_var_bool", + activities=[ + _make_activity( + "Reset", + "SetVariable", + {"variableName": "flag", "value": False}, + ), + ], + ) + definitions = AdfDefinitions( + pipelines=[pipeline], datasets={}, linked_services={}, triggers=[] + ) + report = translate_pipeline(pipeline, definitions) + set_var = next(t for t in report.pipeline.tasks if t.name == "Reset") + assert isinstance(set_var, SetVariableActivity) + assert set_var.variable_value == "false" + + def test_default_valued_variable_with_concat_expression(self): + """An @concat defaultValue resolves like a SetVariable value would.""" + from flowx.models.adf_ast import AdfPipeline, AdfVariable + + pipeline = AdfPipeline( + name="pl_var_default_concat", + activities=[], + variables={ + "fullPath": AdfVariable( + type="String", + default_value="@concat('/Volumes/', pipeline().globalParameters.env)", + ) + }, + ) + definitions = AdfDefinitions( + pipelines=[pipeline], + datasets={}, + linked_services={}, + triggers=[], + global_parameters={"env": "prod"}, + ) + report = translate_pipeline(pipeline, definitions) + init_task = next(t for t in report.pipeline.tasks if t.task_key == "_init_fullPath") + assert isinstance(init_task, SetVariableActivity) + # All-literal concat collapses to a literal (C-01 interplay). + assert init_task.value_kind == "literal" + assert init_task.variable_value == "/Volumes/prod" + + +class TestScheduleCompilation: + """C-10 (SCHED-001): map AdfTrigger objects onto Pipeline.schedule.""" + + def _build_definitions(self, trigger_props, *, runtime_state="Started", trigger_type="ScheduleTrigger"): + from flowx.models.adf_ast import AdfPipeline, AdfTrigger + + pipeline = AdfPipeline(name="pl_with_trigger", activities=[]) + props = dict(trigger_props) + props["runtimeState"] = runtime_state + trigger = AdfTrigger( + name="trg", + type=trigger_type, + properties=props, + pipelines=[{"pipelineReference": {"referenceName": "pl_with_trigger"}}], + ) + definitions = AdfDefinitions( + pipelines=[pipeline], datasets={}, linked_services={}, triggers=[trigger] + ) + return pipeline, definitions + + def test_schedule_trigger_daily_at_specific_time(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "schedule": {"hours": [4], "minutes": [30]}, + "timeZone": "UTC", + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule["kind"] == "schedule" + assert report.pipeline.schedule["quartz_cron_expression"] == "0 30 4 * * ?" + assert report.pipeline.schedule["timezone_id"] == "UTC" + assert report.pipeline.schedule["pause_status"] == "UNPAUSED" + + def test_schedule_trigger_derives_time_of_day_from_start_time(self): + """C-44 (SCHED5-001): a Day recurrence with no schedule block derives + the cron hour/minute from ``startTime`` instead of silently defaulting + to midnight.""" + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "startTime": "2023-03-15T21:00:00Z", + "timeZone": "UTC", + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule["quartz_cron_expression"] == "0 0 21 * * ?" + + def test_schedule_trigger_interval_3_days_emits_periodic(self): + """SCHED3-002 + C-36 (SCHED4-001): Day/Week/Month with interval > 1 + emits periodic, AND the time-of-day from the schedule block is + captured as ``time_of_day_note`` so SETUP.md can surface it.""" + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 3, + "schedule": {"hours": [2]}, + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule["kind"] == "periodic" + assert report.pipeline.schedule["interval"] == 3 + assert report.pipeline.schedule["unit"] == "DAYS" + # C-36: the schedule block (``hours: [2]``) is captured as + # ``time_of_day_note`` rather than silently dropped. + assert report.pipeline.schedule["time_of_day_note"] == {"hours": [2]} + + def test_schedule_trigger_interval_2_months_does_not_emit_months_unit(self): + """C-45 (SCHED5-002): a Month recurrence with interval > 1 must never + emit a periodic spec with the invalid DAB unit 'MONTHS' (the + PeriodicTriggerConfigurationTimeUnit enum only has DAYS/HOURS/WEEKS). + Instead it surfaces a manual setup note.""" + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Month", + "interval": 2, + "schedule": {"monthDays": [1]}, + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule.get("unit") != "MONTHS" + # Either a manual setup note or a cron expr, never a MONTHS periodic. + assert report.pipeline.schedule["kind"] in ("manual_setup", "schedule") + + def test_schedule_trigger_interval_2_weeks_emits_periodic(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Week", + "interval": 2, + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule["kind"] == "periodic" + assert report.pipeline.schedule["unit"] == "WEEKS" + assert report.pipeline.schedule["interval"] == 2 + + def test_trigger_carries_per_pipeline_parameter_overrides(self): + """SCHED3-003: parameters on the trigger's pipelineReference entry + must surface on the schedule spec so the bundler can mutate the + matching job.parameter defaults for scheduled runs.""" + from flowx.models.adf_ast import AdfParameter, AdfPipeline, AdfTrigger + + pipeline = AdfPipeline( + name="pl_with_overrides", + activities=[], + parameters={ + "negocio": AdfParameter(type="String", default_value="DEFAULT"), + "applicationName": AdfParameter(type="String", default_value="DEFAULT"), + }, + ) + trigger = AdfTrigger( + name="nightly", + type="ScheduleTrigger", + properties={ + "runtimeState": "Started", + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "schedule": {"hours": [2]}, + } + }, + }, + pipelines=[ + { + "pipelineReference": {"referenceName": "pl_with_overrides"}, + "parameters": { + "negocio": "GLP", + "applicationName": "app0001", + }, + } + ], + ) + definitions = AdfDefinitions( + pipelines=[pipeline], datasets={}, linked_services={}, triggers=[trigger] + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + overrides = report.pipeline.schedule.get("parameter_overrides") or {} + assert overrides["negocio"] == "GLP" + assert overrides["applicationName"] == "app0001" + + def test_schedule_trigger_interval_1_day_still_cron(self): + """Interval == 1 stays on the cron path so we keep timezone/hour spec.""" + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "schedule": {"hours": [4], "minutes": [30]}, + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule["kind"] == "schedule" + assert "quartz_cron_expression" in report.pipeline.schedule + + def test_schedule_trigger_runtime_state_stopped_pauses(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "schedule": {"hours": [0], "minutes": [0]}, + "timeZone": "UTC", + } + } + }, + runtime_state="Stopped", + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule["pause_status"] == "PAUSED" + + def test_schedule_trigger_normalises_romance_standard_time(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "schedule": {"hours": [8], "minutes": [0]}, + "timeZone": "Romance Standard Time", + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + # Romance Standard Time -> Europe/Madrid per the IANA map. + assert report.pipeline.schedule["timezone_id"] == "Europe/Madrid" + + def test_schedule_trigger_weekly_with_week_days(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Week", + "interval": 1, + "schedule": { + "hours": [9], + "minutes": [0], + "weekDays": ["Monday", "Wednesday", "Friday"], + }, + "timeZone": "UTC", + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule["quartz_cron_expression"] == "0 0 9 ? * MON,WED,FRI" + + def test_tumbling_window_trigger_surfaces_setup_note(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "frequency": "Hour", + "interval": 1, + } + }, + trigger_type="TumblingWindowTrigger", + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule["kind"] == "schedule" + assert report.pipeline.schedule["tumbling"] is True + + def test_blob_events_trigger_maps_to_file_arrival(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "scope": "/subscriptions/x/y", + "events": ["Microsoft.Storage.BlobCreated"], + } + }, + trigger_type="BlobEventsTrigger", + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule["kind"] == "file_arrival" + assert report.pipeline.schedule["url"] == "/subscriptions/x/y" + + def test_custom_events_trigger_routed_to_manual_setup(self): + pipeline, definitions = self._build_definitions( + {"typeProperties": {}}, + trigger_type="CustomEventsTrigger", + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule["kind"] == "manual_setup" + + class TestTranslateEngine: def test_translate_pipeline_produces_report(self, adf_definitions): """translate_pipeline returns a TranslationReport for every pipeline.""" From 32e7f6cd0099fc75a13552ee29a915b498d0aaae Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:17:32 -0400 Subject: [PATCH 12/77] Improve expression resolution, control flow parsing, and setup (#11) * Refactor expression parsing (#1) * Improve control flow conversion (#2) * Update repo structure * Format modules --- .build-constraints.txt | 6 +- .claude-plugin/plugin.json | 1 + .github/workflows/push.yml | 5 + .gitignore | 2 + AGENTS.md | 11 +- Makefile | 10 +- docs/content/docs/installation.mdx | 50 +- docs/content/docs/options.mdx | 26 +- requirements.txt | 32 + scripts/bootstrap.sh | 111 ++ skills/ingest/SKILL.md | 22 + skills/migrate/SKILL.md | 34 +- skills/prepare/SKILL.md | 22 + skills/setup/SKILL.md | 94 ++ skills/translate/SKILL.md | 57 +- src/orchestra/adapter/__init__.py | 4 +- src/orchestra/adapter/__main__.py | 11 +- src/orchestra/adapter/constants.py | 7 +- src/orchestra/adapter/models.py | 44 +- src/orchestra/adapter/operations.py | 169 +- src/orchestra/adapter/session.py | 27 +- src/orchestra/bundler/dab_writer.py | 439 ++++- src/orchestra/bundler/inner_job_params.py | 55 +- src/orchestra/bundler/prereqs_writer.py | 238 ++- src/orchestra/bundler/setup_generator.py | 16 +- src/orchestra/models/adf_ast.py | 38 + src/orchestra/models/dab.py | 21 + src/orchestra/models/ir.py | 212 ++- src/orchestra/parser/adf_loader.py | 46 + src/orchestra/parser/expression_parser.py | 438 ++++- src/orchestra/parser/ir_rewriter.py | 252 +++ .../preparer/activity_preparers/for_each.py | 221 ++- .../activity_preparers/if_condition.py | 109 +- .../preparer/activity_preparers/notebook.py | 183 ++- .../activity_preparers/set_variable.py | 21 +- .../activity_preparers/spark_python.py | 2 + .../preparer/activity_preparers/switch.py | 127 +- .../activity_preparers/web_activity.py | 124 +- src/orchestra/preparer/code_generator.py | 213 ++- src/orchestra/preparer/workflow_preparer.py | 207 ++- .../activity_translators/execute_pipeline.py | 60 +- .../activity_translators/for_each.py | 24 + .../activity_translators/if_condition.py | 213 ++- .../translator/activity_translators/lookup.py | 165 ++ .../activity_translators/notebook.py | 168 +- .../activity_translators/resolve.py | 82 +- .../activity_translators/set_variable.py | 74 + .../activity_translators/spark_python.py | 2 + .../translator/activity_translators/switch.py | 47 +- src/orchestra/translator/engine.py | 849 +++++++++- tests/unit/test_adapter.py | 102 +- tests/unit/test_bundler.py | 712 +++++++- tests/unit/test_code_generator.py | 79 + tests/unit/test_expression_parser.py | 334 +++- tests/unit/test_for_each_inner_job_params.py | 144 ++ tests/unit/test_helpers.py | 9 +- tests/unit/test_ir_rewriter.py | 234 +++ tests/unit/test_preparers.py | 535 +++++- tests/unit/test_prereqs_writer.py | 76 + tests/unit/test_resolve_field.py | 5 +- tests/unit/test_translators.py | 1452 +++++++++++++++++ 61 files changed, 8623 insertions(+), 450 deletions(-) create mode 100644 requirements.txt create mode 100755 scripts/bootstrap.sh create mode 100644 skills/setup/SKILL.md create mode 100644 src/orchestra/parser/ir_rewriter.py create mode 100644 tests/unit/test_for_each_inner_job_params.py create mode 100644 tests/unit/test_ir_rewriter.py create mode 100644 tests/unit/test_prereqs_writer.py diff --git a/.build-constraints.txt b/.build-constraints.txt index 48078f6..c6be8d7 100644 --- a/.build-constraints.txt +++ b/.build-constraints.txt @@ -13,7 +13,7 @@ pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 # via hatchling -trove-classifiers==2026.5.20.19 \ - --hash=sha256:6e611993987ca9326968ad70452733dadd31471599d39896045b28970a9bb81e \ - --hash=sha256:7a173916960d0635fcbf610550d2c27bcc9125164d6f397adf46fc1ef6455c7c +trove-classifiers==2026.5.22.10 \ + --hash=sha256:01fe864225726e03efb843827ecabfe319fc4dee8dd66d65b8996cb09be46e2c \ + --hash=sha256:5477e9974e91904fb2cfa4a7581ab6e2f30c2c38d847fd00ed866080748101d5 # via hatchling diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 28e8695..cf9403a 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -9,6 +9,7 @@ "license": "MIT", "keywords": ["adf", "databricks", "migration", "dabs", "lakeflow", "orchestration", "azure-data-factory"], "skills": [ + "./skills/setup", "./skills/ingest", "./skills/translate", "./skills/prepare", diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 97d7338..71081c5 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -18,6 +18,11 @@ jobs: run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock - run: uv sync --frozen - run: make test + - name: Verify requirements.txt is in sync with the lockfile + run: | + make requirements + git diff --exit-code -- requirements.txt \ + || { echo "requirements.txt is stale. Run 'make requirements' (or 'make precommit') and commit it."; exit 1; } fmt: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 20db1d0..5bbf192 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ ENV/ htmlcov/ .mypy_cache/ .ruff_cache/ +fixlog/ # OS .DS_Store @@ -45,3 +46,4 @@ tmp_adf_ingest_*/ *.pem *.key credentials.json +/fixlog/ diff --git a/AGENTS.md b/AGENTS.md index dc79f38..6d5d79e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,13 +3,22 @@ ## Quick Command Reference ```bash -make dev # Install dependencies +make dev # Install dependencies (development; uses uv) make test # Unit tests make integration # Integration tests (requires ADF fixtures) make fmt # Format + lint (ruff + mypy) make clean # Remove build artifacts ``` +To run the **plugin skills** (ingest/translate/prepare/migrate) without a uv-based dev setup, +bootstrap a self-contained virtual environment with pip via the `setup` skill or directly: + +```bash +bash scripts/bootstrap.sh # creates .venv and pip-installs requirements.txt +# then run plugin code with src/ on PYTHONPATH: +PYTHONPATH=src .venv/bin/python -m flowx.adapter inputs ingest +``` + ## Project Overview Flowx is an agent plugin that translates Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs via Declarative Automation Bundles (DABs). diff --git a/Makefile b/Makefile index 451dcde..419faa1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean dev ci test integration fmt help docs-install docs-clean docs-build docs-serve lock-dependencies +.PHONY: clean dev ci test integration fmt help docs-install docs-clean docs-build docs-serve lock-dependencies requirements precommit clean: rm -rf .venv .pytest_cache .ruff_cache .mypy_cache __pycache__ @@ -42,6 +42,12 @@ lock-dependencies: uv pip compile --generate-hashes --universal --no-header - > build-constraints-new.txt mv build-constraints-new.txt .build-constraints.txt perl -pi -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple"|g' uv.lock + $(MAKE) requirements + +requirements: + uv export --frozen --no-dev --no-emit-project --no-hashes --format requirements-txt -o requirements.txt + +precommit: fmt requirements help: @echo "Available targets:" @@ -50,9 +56,11 @@ help: @echo " test Run unit tests" @echo " integration Run integration tests" @echo " fmt Format and lint code" + @echo " precommit Format, lint, and refresh requirements.txt (run before committing)" @echo " clean Remove build artifacts" @echo " docs-install Install docs dependencies (bun)" @echo " docs-clean Remove docs build artifacts" @echo " docs-build Build the static docs site to docs/site" @echo " docs-serve Run the docs dev server (next dev)" @echo " lock-dependencies Write the uv.lock file" + @echo " requirements Generate requirements.txt from the lockfile" diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index 21b44ee..99a4ba5 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -6,8 +6,8 @@ description: Install flowx in Databricks Genie Code, Claude Code, or other agent import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; import { Callout } from 'fumadocs-ui/components/callout'; -Flowx contains four [agent skills](https://github.com/ghanse/flowx/tree/main/skills) (`ingest`, `translate`, `prepare`, and `migrate`) that teach agentic tools how to use the flowx Python modules. -Installing flowx also installs all required dependencies to run the Python modules. +Flowx is a set of [agent skills](https://github.com/ghanse/flowx/tree/main/skills) that can be installed and used with AI coding assistants. +To use these skills, install flowx as a plugin using your AI assistant's preferred installation method. @@ -41,7 +41,7 @@ flowx, run the following command from a Claude Code session: You can also copy the skill folders into your local `/.claude/skills` folder: ```bash -cp -R skills/{ingest,translate,prepare,migrate} ~/.claude/skills/ +cp -R skills/{setup,ingest,translate,prepare,migrate} ~/.claude/skills/ ``` Once installed, the skills can be invoked using `/flowx:migrate`, `/flowx:ingest`, etc. Claude Code can also invoke the skills when prompted. See the [Claude Code Plugins documentation](https://docs.claude.com/en/docs/claude-code/plugins) for more details. @@ -50,7 +50,7 @@ Once installed, the skills can be invoked using `/flowx:migrate`, `/flowx:ingest Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install the flowx skills. The general pattern: -1. Copy each skill folder (`skills/ingest`, `skills/translate`, `skills/prepare`, `skills/migrate`) into the tool's configured skills directory. +1. Copy each skill folder (`skills/setup`, `skills/ingest`, `skills/translate`, `skills/prepare`, `skills/migrate`) into the tool's configured skills directory. 2. Make sure the path contains `SKILL.md` directly, 3. Restart the tool if it caches skill metadata at startup. @@ -64,10 +64,50 @@ cat skills/*/SKILL.md > flowx-skills.md +## Setting up the Python environment + +Flowx's skills invoke Python modules that may depend on third-party packages. The `setup` skill provisions an isolated virtual environment with the required dependencies. + +Run it **once** after installing the skills, before `ingest`, `translate`, `prepare`, or `migrate`. Just ask your agent: + +> Set up the flowx environment + +The setup script can also be run directly from the plugin root: + +```bash +bash /scripts/bootstrap.sh +``` + +Running the setup process will: + +1. Check that `python3`, `pip`, and `venv` are available. +2. Create `/.venv` if it doesn't already exist. +3. Install the `requirements.txt` dependencies into your virtual environment using `pip`. + +The environment is created once and reused. Re-running the script simply confirms the venv exists and its dependencies are satisfied. + + +If `python3`, `pip`, or the `venv` module are missing, the script will print a warning and exit **without** creating anything. +To install Python in your environment, run one of the following commands: + +* **macOS:** `brew install python` +* **Debian/Ubuntu:** `sudo apt-get install python3 python3-venv python3-pip` +* **Windows:** [python.org/downloads](https://www.python.org/downloads/) (enable "Add python.exe to PATH") + + +After the venv exists, every Python command the skills run uses the venv interpreter with `src/` on `PYTHONPATH`: + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" -m flowx.adapter inputs ingest +``` + +On Windows, the interpreter is `\.venv\Scripts\python.exe`. The agent normally runs these commands for you; they are handy for troubleshooting a `ModuleNotFoundError`. + ## Verifying the installation Open your agent and ask: > What flowx skills do you have available? -You should see all four skills listed with their descriptions. If only some appear, double-check the install path your tool watches for skills. +You should see all five skills listed with their descriptions. If only some appear, double-check the install path your tool watches for skills. diff --git a/docs/content/docs/options.mdx b/docs/content/docs/options.mdx index aaa5a9a..741d581 100644 --- a/docs/content/docs/options.mdx +++ b/docs/content/docs/options.mdx @@ -49,14 +49,24 @@ Copy activities that carry an explicit SQL query (`sqlReaderQuery`, `query`, or must use a [query-based connector](https://docs.databricks.com/aws/en/ingestion/lakeflow-connect/query-based-overview). -## databricks_task_compute - -Controls the compute used to run Databricks Notebook and Spark Python tasks in the translated job. - -| Value | Default | Behavior | -|--------------|---------|-----------------------------------------------------------------------------------------------------------------------------------| -| `existing` | True | Uses the source pipeline's compute definition (e.g. a job cluster). Preserves init scripts, DBR-version, and other configuration. | -| `serverless` | False | Drops the source pipeline's cluster definition; The translated tasks run on serverless compute. | +## consolidate_motif:<motif_id> + +For every multi-activity motif the detector matches in a pipeline (`incremental_load_watermark`, +`rest_api_pagination`, `metadata_driven_bulk_copy`, ...), flowx raises a per-motif +question with id `consolidate_motif:` so each detected pattern can be approved +or rejected independently. + +| Value | Default | Behavior | +|---------------|---------|------------------------------------------------------------------------------------------------------------------------------------------| +| `keep` | True | Preserves the activity-by-activity translation; the motif detection result is informational only. | +| `consolidate` | False | Collapses the matched activities into a single `MotifActivity` whose target is the motif's `databricks_replacement` (e.g. `auto_loader`). | + + +Motif detection is heuristic. Defaulting to `keep` means a false-positive match (e.g. classifying +a submit-and-poll Web/Until/SetVariable chain as REST pagination) cannot silently rewrite the +pipeline; the activities continue to translate one-for-one until the user explicitly confirms the +pattern. + ## metadata_driven_consolidate diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..060dab7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,32 @@ +# This file was autogenerated by uv via the following command: +# uv export --frozen --no-dev --no-emit-project --no-hashes --format requirements-txt -o requirements.txt +certifi==2026.5.20 + # via requests +cffi==2.0.0 ; platform_python_implementation != 'PyPy' + # via cryptography +charset-normalizer==3.4.7 + # via requests +cryptography==48.0.0 + # via google-auth +databricks-sdk==0.110.0 + # via flowx +google-auth==2.53.0 + # via databricks-sdk +idna==3.15 + # via requests +protobuf==6.33.6 + # via databricks-sdk +pyasn1==0.6.3 + # via pyasn1-modules +pyasn1-modules==0.4.2 + # via google-auth +pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' + # via cffi +pyyaml==6.0.3 + # via flowx +requests==2.34.2 + # via databricks-sdk +sqlglot==30.8.0 + # via flowx +urllib3==2.7.0 + # via requests diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 0000000..badb2ca --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Bootstraps a Python environment for the flowx plugin. +# +# Creates a virtual environment at /.venv and installs the Python +# dependencies listed in requirements.txt using pip. +# +# If python3, pip, or the venv module are unavailable, the script prints a clear +# warning telling the user what to install and exits non-zero without making changes. +# +# After bootstrapping, run the plugin's Python code with the venv interpreter and +# src/ on PYTHONPATH, e.g.: +# +# PYTHONPATH="/src" "/.venv/bin/python" -m flowx.adapter inputs ingest +# +set -euo pipefail + +# Resolve the plugin root +SOURCE="${BASH_SOURCE[0]}" +while [ -L "$SOURCE" ]; do + DIR="$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)" + SOURCE="$(readlink "$SOURCE")" + [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" +done +SCRIPT_DIR="$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)" +PLUGIN_ROOT="$(cd -P "$SCRIPT_DIR/.." >/dev/null 2>&1 && pwd)" + +VENV_DIR="$PLUGIN_ROOT/.venv" +REQUIREMENTS="$PLUGIN_ROOT/requirements.txt" + +# Verify python3 is available +if ! command -v python3 >/dev/null 2>&1; then + cat >&2 <<'EOF' +WARNING: python3 was not found on your PATH. + +Flowx requires Python 3.12+ to run its translation code. +Please install Python (it bundles pip) before continuing: + + - macOS: brew install python (or https://www.python.org/downloads/) + - Debian/Ubuntu: sudo apt-get install python3 python3-venv python3-pip + - Windows: https://www.python.org/downloads/ (enable "Add python.exe to PATH") + +Re-run this setup step once Python is installed. +EOF + exit 1 +fi + +PYTHON_BIN="$(command -v python3)" + +# Verify pip is available +if ! "$PYTHON_BIN" -m pip --version >/dev/null 2>&1; then + cat >&2 <<'EOF' +WARNING: pip is not available for your python3 installation. + +pip is required to install the flowx plugin's dependencies. Install it with: + + - macOS/Linux: python3 -m ensurepip --upgrade + - Debian/Ubuntu: sudo apt-get install python3-pip + - or follow https://pip.pypa.io/en/stable/installation/ + +Re-run this setup step once pip is installed. +EOF + exit 1 +fi + +# Verify the venv module is available +if ! "$PYTHON_BIN" -m venv --help >/dev/null 2>&1; then + cat >&2 <<'EOF' +WARNING: the Python `venv` module is not available. + +It is required to create the virtual environment. Install it with: + + - Debian/Ubuntu: sudo apt-get install python3-venv + - or reinstall Python from https://www.python.org/downloads/ + +Re-run this setup step once `venv` is available. +EOF + exit 1 +fi + +# Create the virtual environment +if [ ! -x "$VENV_DIR/bin/python" ]; then + echo "Creating virtual environment at $VENV_DIR ..." + "$PYTHON_BIN" -m venv "$VENV_DIR" +else + echo "Using existing virtual environment at $VENV_DIR ..." +fi + +VENV_PYTHON="$VENV_DIR/bin/python" + +# Install dependencies from requirements.txt +if [ ! -f "$REQUIREMENTS" ]; then + echo "ERROR: requirements.txt not found at $REQUIREMENTS" >&2 + exit 1 +fi + +echo "Upgrading pip ..." +"$VENV_PYTHON" -m pip install --quiet --upgrade pip + +echo "Installing dependencies from requirements.txt ..." +"$VENV_PYTHON" -m pip install -r "$REQUIREMENTS" + +cat </scripts/bootstrap.sh +``` + +This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or +pip is missing, the script prints a warning telling the user what to install — relay it and stop +until they have installed Python 3.12+ and pip. + +Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` +(use it anywhere a command below shows `python3`): + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... +``` + ## Workflow Follow these steps in order: diff --git a/skills/migrate/SKILL.md b/skills/migrate/SKILL.md index 5c3d910..c0f2b9b 100644 --- a/skills/migrate/SKILL.md +++ b/skills/migrate/SKILL.md @@ -27,6 +27,28 @@ This is the top-level orchestration skill. It runs the full migration pipeline: Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. +## Prerequisite — Python environment + +This skill runs the plugin's Python code, which depends on third-party packages. Before running +any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the +**`setup`** skill, or directly: + +```bash +bash /scripts/bootstrap.sh +``` + +This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or +pip is missing, the script prints a warning telling the user what to install — relay it and stop +until they have installed Python 3.12+ and pip. + +Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` +(use it anywhere a command below shows `python3`): + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... +``` + ## Workflow Follow these steps in order: @@ -170,14 +192,22 @@ Use the stamped report (when produced) as the input to the prepare phase. When inspect emits no questions for any pipeline, skip modify and use the original report. -The four questions the adapter raises: +The questions the adapter raises: | `question_id` | Allowed values | Default | |---|---|---| | `copy_activity_paradigm` | `notebook`, `sdp` | `notebook` | | `non_databricks_task_compute` | `serverless`, `classic` | `serverless` | | `use_lakeflow_connectors` | `existing`, `lakeflow_connect` | `existing` | -| `databricks_task_compute` | `existing`, `serverless` | `existing` | +| `consolidate_motif:` | `keep`, `consolidate` | `keep` | + +DatabricksNotebook and DatabricksSparkPython tasks always inherit the cluster binding derived from +their source linked service. + +For each multi-activity motif the detector matches (rest_api_pagination, +incremental_load_watermark, metadata_driven_bulk_copy, ...) the adapter emits one +`consolidate_motif:` question. The user must explicitly opt in to `consolidate` +for each detected pattern. ### Step 6 — Checkpoint: confirm proceed to bundle generation diff --git a/skills/prepare/SKILL.md b/skills/prepare/SKILL.md index 1e6ba9e..b6eed61 100644 --- a/skills/prepare/SKILL.md +++ b/skills/prepare/SKILL.md @@ -26,6 +26,28 @@ The output is a standard DABs project with: - `src/notebooks/` — generated and helper notebooks - `setup/` — infrastructure setup scripts (volumes, secrets, connections) +## Prerequisite — Python environment + +This skill runs the plugin's Python code, which depends on third-party packages. Before running +any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the +**`setup`** skill, or directly: + +```bash +bash /scripts/bootstrap.sh +``` + +This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or +pip is missing, the script prints a warning telling the user what to install — relay it and stop +until they have installed Python 3.12+ and pip. + +Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` +(use it anywhere a command below shows `python3`): + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... +``` + ## Workflow Follow these steps in order: diff --git a/skills/setup/SKILL.md b/skills/setup/SKILL.md new file mode 100644 index 0000000..3d262c8 --- /dev/null +++ b/skills/setup/SKILL.md @@ -0,0 +1,94 @@ +--- +name: setup +description: > + Setup the Python environment for the flowx plugin. Creates a .venv virtual environment + and installs the Python dependencies (from requirements.txt via pip) needed for each phase. + Run this once before any other flowx skill, or whenever dependencies are missing. +triggers: + - "setup flowx" + - "bootstrap flowx" + - "install flowx dependencies" + - "flowx environment" + - "create flowx venv" + - "ModuleNotFoundError flowx" +--- + +# Create the Python Environment + +Create a virtual environment (`.venv`) for the plugin and install its Python dependencies. This +is the prerequisite for the `ingest`, `translate`, `prepare`, and `migrate` skills which run Python +from this environment. + +## Context + +The flowx plugin ships Python code (in `src/flowx/`) that the skills invoke (e.g. +`python -m flowx.adapter ...`, `adf_loader.py`, `engine.py`, `dab_writer.py`). Some code depends +on third-party packages (`pyyaml`, `databricks-sdk`, `sqlglot`). Running it against a bare system +Python fails with `ModuleNotFoundError`. This step provisions an isolated `.venv` with the required +dependencies installed via `pip` from `requirements.txt`. + +The environment is created once and reused. Re-running the bootstrapscript confirms the venv exists +and ensures that dependencies are satisfied. + +## Workflow + +### Step 1 — Run the bootstrap script + +From the plugin root, run: + +```bash +bash /scripts/bootstrap.sh +``` + +Where `` is the root of the flowx plugin (the directory containing `src/`, +`skills/`, and `requirements.txt`). + +The script will: +1. Check that `python3`, `pip`, and the `venv` module are available. +2. Create `/.venv` if it does not already exist. +3. Install dependencies listed in `requirements.txt` into that venv using `pip`. + +### Step 2 — Handle a missing Python or pip + +If Python, pip, or the `venv` module are **not** available, the script prints a `WARNING:` block +explaining what to install and exits non-zero **without** creating anything. + +When this happens, **do not attempt to work around it**. Relay the warning to the user, ask them +to install, and stop: + +> ⚠️ Python must be installed before I can set up the flowx environment. +> +> * On macOS: `brew install python`. +> * On Debian/Ubuntu: `sudo apt-get install python3 python3-venv python3-pip`. +> +> Let me know once it's installed and I'll re-run setup. + +Re-run this setup skill after the user confirms Python and pip are installed. + +### Step 3 — Confirm success and how to run Python code + +On success, the script prints the interpreter path and a usage example. After this, every +Python command in the flowx skills **must** be run with the venv interpreter and `src/` +on `PYTHONPATH`: + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" -m flowx.adapter inputs ingest +``` + +(On Windows the interpreter is `\.venv\Scripts\python.exe`.) + +Use `/.venv/bin/python` anywhere the other skills show `python3`. + +## Output + +| Artifact | Description | +|---|---| +| `/.venv/` | Virtual environment containing the installed dependencies | +| `requirements.txt` | The dependency list installed into the venv | + +## Examples + +- "Set up the flowx environment" +- "Bootstrap flowx so I can run a migration" +- "I got a ModuleNotFoundError running ingest — fix the environment" diff --git a/skills/translate/SKILL.md b/skills/translate/SKILL.md index 5d719e8..0ce95b9 100644 --- a/skills/translate/SKILL.md +++ b/skills/translate/SKILL.md @@ -14,17 +14,37 @@ triggers: # Translate ADF to Databricks IR -Convert the parsed ADF inventory into Databricks intermediate representation (IR) using deterministic translators for known types and agentic fallback for complex/unknown types. +Convert the parsed ADF inventory into Databricks intermediate representation (IR) using deterministic translators for known types and agentic fallback for unknown types. ## Context This is phase 2 of the flowx migration workflow. It consumes the `inventory.json` produced by the `ingest` skill and produces a `translation_report.json` that the `prepare` skill uses to generate Databricks Declarative Automation Bundles. The translation follows a **deterministic-first** strategy: -1. Activities with known, well-defined mappings are translated by built-in Python translators (fast, reliable, no LLM needed) -2. Activities that require interpretation, complex expression conversion, or lack a direct mapping are handled by agentic skills from the `adf-to-databricks-plugin` (LLM-assisted) +1. Activities with known, well-defined mappings are translated by built-in Python translators +2. Activities that require interpretation, expression conversion, or lack a Python translator are handled by agent skills from the `adf-to-databricks-plugin` -This approach maximizes reliability while covering the long tail of ADF activity types. +## Prerequisite — Python environment + +This skill runs the plugin's Python code, which depends on third-party packages. Before running +any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the +**`setup`** skill, or directly: + +```bash +bash /scripts/bootstrap.sh +``` + +This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or +pip is missing, the script prints a warning telling the user what to install — relay it and stop +until they have installed Python 3.12+ and pip. + +Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` +(use it anywhere a command below shows `python3`): + +```bash +export PYTHONPATH="/src" +"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... +``` ## Workflow @@ -68,12 +88,12 @@ Where: - `` is the root of the flowx plugin - `` is the path to `inventory.json` - `` is the original ADF JSON directory (from the ingest phase) -- `` is where to write translation output (default: `./orchestra_output/translate/`) +- `` is the translation output path (default: `./orchestra_output/translate/`) This produces: - `translation_report.json` — results for deterministic activities + placeholders for agentic gaps -- `ir/` directory — the Databricks IR for each translated activity -- `notebooks/` directory — any generated helper notebooks +- `ir/` directory — Databricks IR for each translated activity +- `notebooks/` directory — generated helper notebooks ### Step 3 — Read the translation report @@ -125,7 +145,7 @@ Invoke `adf-to-databricks:adf-dataflow-converter` with the raw activity JSON and - The raw `typeProperties` from the ADF activity - The data flow JSON definition (if available in the source directory under `dataflow/`) - The linked service configurations for source/sink connections -- Target catalog and schema for the DLT pipeline or PySpark notebook output +- Target catalog and schema for the SDP pipeline or PySpark notebook output **Control flow activities (Switch, Until, Wait, Filter, AppendVariable):** Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: @@ -174,14 +194,14 @@ This updates `translation_report.json` with the agentic results merged in, chang ### Step 6.1 — Gather just-in-time translation preferences The adapter raises several preference questions plus a chained set for -metadata-driven motifs. Drive the loop multi-pass: every time the user -answers a question whose value gates further prompts, re-run `inspect ---answers ` to surface the next batch. +metadata-driven motifs. Every time the user answers a question whose value +gates further prompts, re-run `inspect --answers ` to surface +the next batch. When the metadata-driven flow ends with `metadata_driven_lookup_tool=have` and the agent has a database tool (Genie, MCP SQL, or a workspace SDK), run the lookup query directly and write the rows to -`/lookup_values.json`. When the answer is `none`, prompt +`/lookup_values.json`. When the answer is `none`, prompt the user for a CSV file or comma-separated string and call: ```bash @@ -199,14 +219,13 @@ python3 -m flowx.adapter modify \ --out /translation_report.stamped.json ``` -When no metadata-driven motif is consolidated, `--lookup-values` is -omitted. +When no metadata-driven motif is consolidated, `--lookup-values` is omitted. #### Legacy flow details Before writing the final report, surface any pipeline-modifier questions the IR raises (Copy Data paradigm, non-Databricks task compute, Lakeflow Connect -opt-in, Databricks task compute). Use the adapter CLI bridge: +opt-in, Databricks task compute). Use the adapter CLI bridge: ```bash python3 -m flowx.adapter inspect @@ -236,20 +255,18 @@ The command emits JSON: ``` For each question, prompt the user with the rationale, options, and the -task keys it affects. Use the default when the user defers. Collect the +task keys it affects. Use the default when the user defers. Collect the answers into a JSON file (`/answers.json`) shaped like: ```json { "copy_activity_paradigm": "sdp", "non_databricks_task_compute": "serverless", - "use_lakeflow_connectors": "lakeflow_connect", - "databricks_task_compute": "existing" + "use_lakeflow_connectors": "lakeflow_connect" } ``` -Then apply the answers to produce a stamped report the prepare phase -consumes: +Then apply the answers to produce a stamped report the prepare phase consumes: ```bash python3 -m flowx.adapter modify \ diff --git a/src/orchestra/adapter/__init__.py b/src/orchestra/adapter/__init__.py index f713c38..6add5d5 100644 --- a/src/orchestra/adapter/__init__.py +++ b/src/orchestra/adapter/__init__.py @@ -34,13 +34,13 @@ from flowx.adapter.models import ( DEFAULT_PREFERENCES, CopyActivityParadigm, - DatabricksTaskCompute, LakeflowConnectorType, MetadataDrivenAccess, MetadataDrivenConsolidate, MetadataDrivenLookupTool, MetadataDrivenSize, MigrationInputQuestion, + MotifConsolidate, NonDatabricksTaskCompute, PendingMigrationInputs, PendingQuestions, @@ -68,7 +68,6 @@ __all__ = [ "DEFAULT_PREFERENCES", "CopyActivityParadigm", - "DatabricksTaskCompute", "LakeflowConnectorType", "MetadataDrivenAccess", "MetadataDrivenConsolidate", @@ -76,6 +75,7 @@ "MetadataDrivenSize", "MigrationInputQuestion", "MigrationInputSession", + "MotifConsolidate", "NonDatabricksTaskCompute", "PendingMigrationInputs", "PendingQuestions", diff --git a/src/orchestra/adapter/__main__.py b/src/orchestra/adapter/__main__.py index be5823c..490de59 100644 --- a/src/orchestra/adapter/__main__.py +++ b/src/orchestra/adapter/__main__.py @@ -19,15 +19,16 @@ from pathlib import Path from typing import Any +from flowx.adapter.constants import MOTIF_CONSOLIDATE_QUESTION_PREFIX from flowx.adapter.models import ( DEFAULT_PREFERENCES, CopyActivityParadigm, - DatabricksTaskCompute, LakeflowConnectorType, MetadataDrivenAccess, MetadataDrivenConsolidate, MetadataDrivenLookupTool, MetadataDrivenSize, + MotifConsolidate, NonDatabricksTaskCompute, PendingQuestions, TranslationPreferences, @@ -484,6 +485,10 @@ def _preferences_from_answers(answers: dict[str, str]) -> TranslationPreferences question. """ validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} + motif_consolidations: dict[str, MotifConsolidate] = {} + for qid, value in validated.items(): + if qid.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): + motif_consolidations[qid[len(MOTIF_CONSOLIDATE_QUESTION_PREFIX) :]] = MotifConsolidate(value) return TranslationPreferences( copy_activity_paradigm=CopyActivityParadigm( validated.get("copy_activity_paradigm", DEFAULT_PREFERENCES.copy_activity_paradigm) @@ -494,9 +499,6 @@ def _preferences_from_answers(answers: dict[str, str]) -> TranslationPreferences use_lakeflow_connectors=UseLakeflowConnectors( validated.get("use_lakeflow_connectors", DEFAULT_PREFERENCES.use_lakeflow_connectors) ), - databricks_task_compute=DatabricksTaskCompute( - validated.get("databricks_task_compute", DEFAULT_PREFERENCES.databricks_task_compute) - ), lakeflow_connector_type=LakeflowConnectorType( validated.get("lakeflow_connector_type", DEFAULT_PREFERENCES.lakeflow_connector_type) ), @@ -512,6 +514,7 @@ def _preferences_from_answers(answers: dict[str, str]) -> TranslationPreferences metadata_driven_lookup_tool=MetadataDrivenLookupTool( validated.get("metadata_driven_lookup_tool", DEFAULT_PREFERENCES.metadata_driven_lookup_tool) ), + motif_consolidations=motif_consolidations, ) diff --git a/src/orchestra/adapter/constants.py b/src/orchestra/adapter/constants.py index c74a434..40c3e3c 100644 --- a/src/orchestra/adapter/constants.py +++ b/src/orchestra/adapter/constants.py @@ -14,13 +14,18 @@ QUESTION_COPY_ACTIVITY_PARADIGM: Final[str] = "copy_activity_paradigm" QUESTION_NON_DATABRICKS_TASK_COMPUTE: Final[str] = "non_databricks_task_compute" QUESTION_USE_LAKEFLOW_CONNECTORS: Final[str] = "use_lakeflow_connectors" -QUESTION_DATABRICKS_TASK_COMPUTE: Final[str] = "databricks_task_compute" QUESTION_LAKEFLOW_CONNECTOR_TYPE: Final[str] = "lakeflow_connector_type" QUESTION_METADATA_DRIVEN_CONSOLIDATE: Final[str] = "metadata_driven_consolidate" QUESTION_METADATA_DRIVEN_ACCESS: Final[str] = "metadata_driven_access" QUESTION_METADATA_DRIVEN_SIZE: Final[str] = "metadata_driven_size" QUESTION_METADATA_DRIVEN_LOOKUP_TOOL: Final[str] = "metadata_driven_lookup_tool" +# Per-detected-motif consolidation question_ids carry the motif_id as a suffix +# (e.g. ``consolidate_motif:rest_api_pagination``) so each detected motif gets +# its own question. Validation strips the prefix and validates the answer +# against the :class:`MotifConsolidate` enum. +MOTIF_CONSOLIDATE_QUESTION_PREFIX: Final[str] = "consolidate_motif:" + METADATA_DRIVEN_MOTIF_ID: Final[str] = "metadata_driven_bulk_copy" PHASE_INGEST: Final[str] = "ingest" diff --git a/src/orchestra/adapter/models.py b/src/orchestra/adapter/models.py index 3aff855..70f5abc 100644 --- a/src/orchestra/adapter/models.py +++ b/src/orchestra/adapter/models.py @@ -29,13 +29,6 @@ class UseLakeflowConnectors(StrEnum): EXISTING = "existing" -class DatabricksTaskCompute(StrEnum): - """Compute mode used for ADF DatabricksNotebook and DatabricksSparkPython tasks.""" - - SERVERLESS = "serverless" - EXISTING = "existing" - - class LakeflowConnectorType(StrEnum): """Lakeflow Connect connector flavour for an eligible Copy ingestion. @@ -85,12 +78,24 @@ class MetadataDrivenLookupTool(StrEnum): NONE = "none" +class MotifConsolidate(StrEnum): + """Whether to collapse a detected motif into a single :class:`MotifActivity`. + + Default for every detected motif is :data:`KEEP` -- preserving the + underlying activity-by-activity translation -- so motif detection + can never silently rewrite a pipeline without an explicit user + opt-in. + """ + + KEEP = "keep" + CONSOLIDATE = "consolidate" + + FIELD_TO_ENUM: Final[MappingProxyType[str, type[StrEnum]]] = MappingProxyType( { "copy_activity_paradigm": CopyActivityParadigm, "non_databricks_task_compute": NonDatabricksTaskCompute, "use_lakeflow_connectors": UseLakeflowConnectors, - "databricks_task_compute": DatabricksTaskCompute, "lakeflow_connector_type": LakeflowConnectorType, "metadata_driven_consolidate": MetadataDrivenConsolidate, "metadata_driven_access": MetadataDrivenAccess, @@ -113,22 +118,26 @@ class TranslationPreferences: non_databricks_task_compute: Compute mode for non-Databricks tasks. use_lakeflow_connectors: Whether eligible database-source Copy patterns are migrated to managed Lakeflow Connect pipelines. - databricks_task_compute: Compute mode for ADF DatabricksNotebook and - DatabricksSparkPython tasks. per_task: Optional per-activity overrides keyed by task_key. Each - value is a partial mapping of the four fields above; only the + value is a partial mapping of the fields above; only the keys present win over the pipeline-wide defaults. + + ADF DatabricksNotebook and DatabricksSparkPython tasks always keep + the cluster binding derived from the source linked service -- the + serverless replacement option was removed because it silently + discarded init scripts and DBR-version constraints that the source + pipeline relied on. """ copy_activity_paradigm: CopyActivityParadigm = CopyActivityParadigm.NOTEBOOK non_databricks_task_compute: NonDatabricksTaskCompute = NonDatabricksTaskCompute.SERVERLESS use_lakeflow_connectors: UseLakeflowConnectors = UseLakeflowConnectors.EXISTING - databricks_task_compute: DatabricksTaskCompute = DatabricksTaskCompute.EXISTING lakeflow_connector_type: LakeflowConnectorType = LakeflowConnectorType.CDC metadata_driven_consolidate: MetadataDrivenConsolidate = MetadataDrivenConsolidate.KEEP metadata_driven_access: MetadataDrivenAccess = MetadataDrivenAccess.NO metadata_driven_size: MetadataDrivenSize = MetadataDrivenSize.LARGE metadata_driven_lookup_tool: MetadataDrivenLookupTool = MetadataDrivenLookupTool.NONE + motif_consolidations: dict[str, MotifConsolidate] = field(default_factory=dict) per_task: dict[str, dict[str, str]] = field(default_factory=dict) def __post_init__(self) -> None: @@ -142,6 +151,13 @@ def __post_init__(self) -> None: value = getattr(self, field_name) if not isinstance(value, enum_cls): object.__setattr__(self, field_name, enum_cls(value)) + # motif_consolidations is keyed by dynamic motif_id rather than a + # fixed field name, so it is not in FIELD_TO_ENUM. Coerce its + # values to MotifConsolidate members here. + coerced: dict[str, MotifConsolidate] = {} + for motif_id, choice in self.motif_consolidations.items(): + coerced[motif_id] = choice if isinstance(choice, MotifConsolidate) else MotifConsolidate(choice) + object.__setattr__(self, "motif_consolidations", coerced) def effective_for(self, task_key: str) -> TranslationPreferences: """Returns a preferences view where per-task overrides for *task_key* win. @@ -167,9 +183,6 @@ def effective_for(self, task_key: str) -> TranslationPreferences: use_lakeflow_connectors=UseLakeflowConnectors( override.get("use_lakeflow_connectors", self.use_lakeflow_connectors) ), - databricks_task_compute=DatabricksTaskCompute( - override.get("databricks_task_compute", self.databricks_task_compute) - ), lakeflow_connector_type=LakeflowConnectorType( override.get("lakeflow_connector_type", self.lakeflow_connector_type) ), @@ -183,6 +196,7 @@ def effective_for(self, task_key: str) -> TranslationPreferences: metadata_driven_lookup_tool=MetadataDrivenLookupTool( override.get("metadata_driven_lookup_tool", self.metadata_driven_lookup_tool) ), + motif_consolidations=dict(self.motif_consolidations), per_task=self.per_task, ) diff --git a/src/orchestra/adapter/operations.py b/src/orchestra/adapter/operations.py index cfc526c..fe21cbc 100644 --- a/src/orchestra/adapter/operations.py +++ b/src/orchestra/adapter/operations.py @@ -23,8 +23,8 @@ LAKEFLOW_CONNECT_REPLACEMENT, LAKEFLOW_CONNECTOR_TYPE_QUERY_BASED, METADATA_DRIVEN_MOTIF_ID, + MOTIF_CONSOLIDATE_QUESTION_PREFIX, QUESTION_COPY_ACTIVITY_PARADIGM, - QUESTION_DATABRICKS_TASK_COMPUTE, QUESTION_METADATA_DRIVEN_ACCESS, QUESTION_METADATA_DRIVEN_CONSOLIDATE, QUESTION_METADATA_DRIVEN_LOOKUP_TOOL, @@ -35,12 +35,12 @@ from flowx.adapter.models import ( FIELD_TO_ENUM, CopyActivityParadigm, - DatabricksTaskCompute, LakeflowConnectorType, MetadataDrivenAccess, MetadataDrivenConsolidate, MetadataDrivenLookupTool, MetadataDrivenSize, + MotifConsolidate, NonDatabricksTaskCompute, PendingQuestions, QuestionOption, @@ -62,9 +62,7 @@ ForEachActivity, IfConditionActivity, MotifActivity, - NotebookActivity, Pipeline, - SparkPythonActivity, SwitchActivity, SwitchCase, ) @@ -75,12 +73,15 @@ def enum_for(question_id: str) -> type[StrEnum] | None: """Returns the enum class backing a preference field. Args: - question_id: Field name (e.g. ``"copy_activity_paradigm"``). + question_id: Field name (e.g. ``"copy_activity_paradigm"``) or + per-motif id (e.g. ``"consolidate_motif:rest_api_pagination"``). Returns: The :class:`StrEnum` subclass that defines the allowed values, or - ``None`` when the field is unknown. + ``None`` when the question_id is unknown. """ + if question_id.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): + return MotifConsolidate return FIELD_TO_ENUM.get(question_id) @@ -241,7 +242,6 @@ def gather_questions( _build_lakeflow_connector_type_question, _build_copy_activity_paradigm_question, _build_non_databricks_task_compute_question, - _build_databricks_task_compute_question, _build_metadata_driven_consolidate_question, _build_metadata_driven_access_question, _build_metadata_driven_size_question, @@ -255,6 +255,14 @@ def gather_questions( and question.question_id not in answer_map and _conditions_met(question.conditions, answer_map) ] + # Per-motif "consolidate?" questions: one per detected motif. Each + # gets its own question_id ``consolidate_motif:`` so the + # adapter can solicit and validate them independently. Default is + # ``keep`` -- nothing is collapsed without an explicit yes. + for motif_question in _build_motif_consolidation_questions(motif_list): + if motif_question.question_id in answer_map: + continue + pending.append(motif_question) return PendingQuestions(pipeline_name=pipeline.name, questions=pending) @@ -468,52 +476,6 @@ def _build_lakeflow_connector_type_question( return None -def _build_databricks_task_compute_question( - pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None -) -> TranslationQuestion | None: - """Builds the serverless-vs-existing question for ADF Databricks-* tasks. - - Args: - pipeline: Translated pipeline IR. - motifs: Detected motifs (unused; accepted for builder uniformity). - - Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when - no Databricks notebook or Python task is present. - """ - affected = tuple( - activity.task_key - for activity in walk_activities(pipeline.tasks) - if isinstance(activity, (NotebookActivity, SparkPythonActivity)) - ) - if not affected: - return None - return TranslationQuestion( - question_id=QUESTION_DATABRICKS_TASK_COMPUTE, - prompt="Migrate existing Databricks notebook and Python tasks to serverless?", - rationale=( - "ADF DatabricksNotebook and DatabricksSparkPython tasks bind to a " - "classic cluster derived from the source linked service. Serverless " - "drops that binding; keeping the existing compute preserves init " - "scripts or DBR-specific features." - ), - options=( - QuestionOption( - value=DatabricksTaskCompute.EXISTING.value, - label="Keep linked-service compute", - description="Binds the task to the cluster derived from the ADF linked service.", - ), - QuestionOption( - value=DatabricksTaskCompute.SERVERLESS.value, - label="Serverless", - description="Removes the cluster binding so the task runs on serverless compute.", - ), - ), - affected_task_keys=affected, - default=DatabricksTaskCompute.EXISTING.value, - ) - - def _build_metadata_driven_consolidate_question( pipeline: Pipeline, motifs: list, @@ -704,6 +666,80 @@ def _build_metadata_driven_lookup_tool_question( ) +def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuestion]: + """Builds one ``consolidate_motif:`` question per detected motif. + + Args: + motifs: Detected :class:`~flowx.models.motifs.DetectedMotif` + instances from :func:`flowx.motifs.detector.detect_motifs`. + + Returns: + A list of :class:`TranslationQuestion` instances, one per + detected motif. Each question uses a unique question_id of the + form ``consolidate_motif:`` so multiple distinct motif + types in the same pipeline (e.g. ``rest_api_pagination`` *and* + ``metadata_driven_bulk_copy``) each get their own prompt. + Returns an empty list when no motifs were detected. + + Notes: + - The default for every motif is ``keep``. Motif detection is + a heuristic match and over-collapsing silently rewrites + pipelines; requiring an explicit ``consolidate`` answer is + the safer default. + - When the same motif type is detected more than once in the + same pipeline (rare in practice but possible) the builder + emits a single question covering all instances of that type. + Per-instance overrides can still be expressed by adding more + fine-grained gating in :class:`MotifActivity`. + - The ``affected_task_keys`` field lists the *underlying* ADF + activity names so the agent can quote them when asking the + user, e.g. ``"Consolidate REST API Pagination motif spanning + GetToken, InitCursor, PollLoop into a single notebook?"``. + """ + if not motifs: + return [] + seen: set[str] = set() + questions: list[TranslationQuestion] = [] + for motif in motifs: + definition = motif.definition + motif_id = definition.motif_id + if motif_id in seen: + continue + seen.add(motif_id) + affected = tuple(motif.matched_activities) + question_id = f"{MOTIF_CONSOLIDATE_QUESTION_PREFIX}{motif_id}" + confidence_suffix = "" + if motif.confidence_notes: + confidence_suffix = " Detector notes: " + " | ".join(motif.confidence_notes) + questions.append( + TranslationQuestion( + question_id=question_id, + prompt=f"Consolidate the {definition.display_name!r} motif into a single task?", + rationale=( + f"{definition.description} " + f"Affected activities: {', '.join(affected) if affected else '(none)'}.{confidence_suffix} " + "Keep preserves the activity-by-activity translation; consolidate replaces " + f"them with a single {definition.databricks_replacement!r} task." + ), + options=( + QuestionOption( + value=MotifConsolidate.KEEP.value, + label="Keep individual activities", + description="Preserves the per-activity translation; no motif collapse.", + ), + QuestionOption( + value=MotifConsolidate.CONSOLIDATE.value, + label="Consolidate into one task", + description=f"Replaces matched activities with a {definition.databricks_replacement!r} task.", + ), + ), + affected_task_keys=affected, + default=MotifConsolidate.KEEP.value, + ) + ) + return questions + + def _metadata_driven_motif_task_keys( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None ) -> tuple[str, ...]: @@ -851,11 +887,6 @@ def _stamp_activity(activity: Activity, pipeline_preferences: TranslationPrefere return _stamp_copy_activity(activity, activity_preferences) if isinstance(activity, MotifActivity): return _stamp_motif_activity(activity, activity_preferences) - if isinstance(activity, (NotebookActivity, SparkPythonActivity)): - return dataclasses.replace( - activity, - compute_mode=_resolve_databricks_task_compute_mode(activity_preferences), - ) return dataclasses.replace(activity, compute_mode=_resolve_compute_mode(activity, activity_preferences)) @@ -1097,9 +1128,11 @@ def _resolve_compute_mode(activity: Activity, activity_preferences: TranslationP :data:`COMPUTE_MODE_CLASSIC_SINGLE_NODE`, :data:`COMPUTE_MODE_CLASSIC_MULTI_NODE`, or :data:`COMPUTE_MODE_INHERIT`. + + DatabricksNotebook and DatabricksSparkPython activities always + inherit the linked-service-derived cluster binding; serverless is + no longer offered as a replacement for source-defined clusters. """ - if isinstance(activity, (NotebookActivity, SparkPythonActivity)): - return _resolve_databricks_task_compute_mode(activity_preferences) if not is_non_databricks_task(activity): return COMPUTE_MODE_INHERIT if activity_preferences.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS: @@ -1107,19 +1140,3 @@ def _resolve_compute_mode(activity: Activity, activity_preferences: TranslationP if isinstance(activity, CopyActivity): return COMPUTE_MODE_CLASSIC_MULTI_NODE return COMPUTE_MODE_CLASSIC_SINGLE_NODE - - -def _resolve_databricks_task_compute_mode(activity_preferences: TranslationPreferences) -> str: - """Resolves the compute mode for an ADF Databricks-* task. - - Args: - activity_preferences: Effective preferences for the task. - - Returns: - :data:`COMPUTE_MODE_SERVERLESS` when the caller opted into - serverless; otherwise :data:`COMPUTE_MODE_INHERIT`, which leaves - the linked-service-derived binding in place. - """ - if activity_preferences.databricks_task_compute is DatabricksTaskCompute.SERVERLESS: - return COMPUTE_MODE_SERVERLESS - return COMPUTE_MODE_INHERIT diff --git a/src/orchestra/adapter/session.py b/src/orchestra/adapter/session.py index a312438..318622d 100644 --- a/src/orchestra/adapter/session.py +++ b/src/orchestra/adapter/session.py @@ -31,13 +31,13 @@ from flowx.adapter.models import ( DEFAULT_PREFERENCES, CopyActivityParadigm, - DatabricksTaskCompute, LakeflowConnectorType, MetadataDrivenAccess, MetadataDrivenConsolidate, MetadataDrivenLookupTool, MetadataDrivenSize, MigrationInputQuestion, + MotifConsolidate, NonDatabricksTaskCompute, PendingMigrationInputs, PendingQuestions, @@ -174,9 +174,6 @@ def build_preferences(self) -> TranslationPreferences: use_lakeflow_connectors=UseLakeflowConnectors( self._answers.get("use_lakeflow_connectors", self.defaults.use_lakeflow_connectors) ), - databricks_task_compute=DatabricksTaskCompute( - self._answers.get("databricks_task_compute", self.defaults.databricks_task_compute) - ), lakeflow_connector_type=LakeflowConnectorType( self._answers.get("lakeflow_connector_type", self.defaults.lakeflow_connector_type) ), @@ -192,9 +189,31 @@ def build_preferences(self) -> TranslationPreferences: metadata_driven_lookup_tool=MetadataDrivenLookupTool( self._answers.get("metadata_driven_lookup_tool", self.defaults.metadata_driven_lookup_tool) ), + motif_consolidations=self._collect_motif_consolidations(), per_task=self.defaults.per_task, ) + def _collect_motif_consolidations(self) -> dict[str, MotifConsolidate]: + """Returns the per-motif consolidation answers gathered so far. + + Returns: + Dict mapping ``motif_id`` to the user's :class:`MotifConsolidate` + answer. Motifs the user did not answer fall back to the + value carried on ``self.defaults`` (default + :data:`MotifConsolidate.KEEP`). The dict is the union of + the defaults and any answers whose ``question_id`` starts + with ``consolidate_motif:``. + """ + from flowx.adapter.constants import MOTIF_CONSOLIDATE_QUESTION_PREFIX + + consolidations: dict[str, MotifConsolidate] = dict(self.defaults.motif_consolidations) + for question_id, answer in self._answers.items(): + if not question_id.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): + continue + motif_id = question_id[len(MOTIF_CONSOLIDATE_QUESTION_PREFIX) :] + consolidations[motif_id] = MotifConsolidate(answer) + return consolidations + def resume(self) -> Pipeline: """Returns the preference-stamped pipeline IR. diff --git a/src/orchestra/bundler/dab_writer.py b/src/orchestra/bundler/dab_writer.py index fb4ddfb..660d020 100644 --- a/src/orchestra/bundler/dab_writer.py +++ b/src/orchestra/bundler/dab_writer.py @@ -71,6 +71,12 @@ class _BundleYamlDumper(yaml.SafeDumper): # bundle's ``variables`` block + SETUP.md. _cross_bundle_variables: dict[str, str] = {} +# C-43 (CF5-001 / CF5-002): condition_task operands the dangling-ref safety +# net had to blank. Each entry is {task_key, field, original_ref}. Reset +# per write_bundle call and surfaced as a SETUP.md section so a neutralised +# branch predicate (always-true) is never silent. +_neutralized_conditions: list[dict[str, str]] = [] + _WIDGET_REFERENCE = re.compile(r"""dbutils\.widgets\.get\(\s*["']([^"']+)["']\s*\)""") @@ -98,6 +104,7 @@ def write_bundle( # cross-bundle variables from one bundle into the next. _bundle_warnings.clear() _cross_bundle_variables.clear() + _neutralized_conditions.clear() output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) @@ -244,15 +251,54 @@ def write_bundle( all_tasks = list(workflow.tasks) for inner in workflow.inner_workflows: all_tasks.extend(inner.tasks) + parameter_approximations = list(workflow.parameter_approximations) + for inner in workflow.inner_workflows: + parameter_approximations.extend(inner.parameter_approximations) known_bundle_jobs = {resource_key} | {normalize_task_key(inner.name) for inner in workflow.inner_workflows} # ``manual_parameters`` was collected above (before YAML emission) so # the broken values are also stripped from the on-disk YAML. + # VAREX3-003: manual_variable_rollup SetupTasks emitted by + # workflow_preparer surface in SETUP.md so the user knows where to add + # a roll-up notebook. + rollup_configs = [st.config for st in workflow.setup_tasks if st.type == "manual_variable_rollup"] + for inner in workflow.inner_workflows: + rollup_configs.extend(st.config for st in inner.setup_tasks if st.type == "manual_variable_rollup") + dynamic_dispatch_configs = [st.config for st in workflow.setup_tasks if st.type == "dynamic_notebook_dispatch"] + unresolved_library_configs = [st.config for st in workflow.setup_tasks if st.type == "unresolved_library"] + manual_variable_init_configs = [st.config for st in workflow.setup_tasks if st.type == "manual_variable_init"] + manual_schedule_time_of_day_configs = [ + st.config for st in workflow.setup_tasks if st.type == "manual_schedule_time_of_day" + ] + manual_credential_configs = [st.config for st in workflow.setup_tasks if st.type == "manual_credential"] + for inner in workflow.inner_workflows: + dynamic_dispatch_configs.extend(st.config for st in inner.setup_tasks if st.type == "dynamic_notebook_dispatch") + unresolved_library_configs.extend(st.config for st in inner.setup_tasks if st.type == "unresolved_library") + manual_variable_init_configs.extend(st.config for st in inner.setup_tasks if st.type == "manual_variable_init") + manual_schedule_time_of_day_configs.extend( + st.config for st in inner.setup_tasks if st.type == "manual_schedule_time_of_day" + ) + manual_credential_configs.extend(st.config for st in inner.setup_tasks if st.type == "manual_credential") + # LSC3-006: union typed SecretInstructions from the workflow (and + # inner workflows) with the notebook-scanned scopes so SETUP.md and + # create_secrets.py reference the same set of (scope, key) pairs. + all_secret_instructions = list(workflow.secrets) + for inner in workflow.inner_workflows: + all_secret_instructions.extend(inner.secrets) prereqs = build_prereqs( notebooks=all_notebooks, tasks=all_tasks, known_bundle_jobs=known_bundle_jobs, cross_bundle_variables=dict(_cross_bundle_variables), manual_parameters=manual_parameters, + parameter_approximations=parameter_approximations, + manual_variable_rollups=rollup_configs, + secret_instructions=all_secret_instructions, + dynamic_notebook_dispatches=dynamic_dispatch_configs, + unresolved_libraries=unresolved_library_configs, + manual_variable_inits=manual_variable_init_configs, + manual_schedule_time_of_day=manual_schedule_time_of_day_configs, + manual_credentials=manual_credential_configs, + neutralized_conditions=list(_neutralized_conditions), ) setup_path = output_dir / "SETUP.md" setup_path.write_text(render_setup_md(prereqs, bundle_name=effective_name), encoding="utf-8") @@ -385,6 +431,38 @@ def _warn(task_key: str, message: str) -> None: _DEFAULT_SPARK_VERSION = "15.4.x-scala2.12" _DEFAULT_NODE_TYPE_ID = "Standard_DS3_v2" +# C-29 (NB-ITER4-002): a real DBR version string matches e.g. +# "15.4.x-scala2.12" / "15.4.x-photon-scala2.12". ADF expressions like +# ``@if(equals(item()?.photon,true),...)`` slip through unfiltered today +# and land in ``databricks.yml`` as the spark_version variable default, +# which bundle deploy rejects. The regex anchors on the canonical +# Databricks Runtime shape so unrecognised strings fall through to the +# safe default. +_DBR_VERSION_RE = re.compile(r"^\d+\.\d+\.x(-[a-z0-9.]+)*$") + + +def _is_valid_spark_version(value: Any) -> bool: + """Return True when *value* parses as a real DBR runtime version string.""" + if not isinstance(value, str) or not value: + return False + return _DBR_VERSION_RE.match(value) is not None + + +def _is_valid_node_type_id(value: Any) -> bool: + """Return True when *value* looks like a real cloud instance type. + + Conservatively rejects anything that starts with ``@`` (an unresolved + ADF expression) or contains spaces; otherwise accepts the value + verbatim so we don't gate out cloud-specific instance families. + """ + if not isinstance(value, str) or not value: + return False + if value.startswith("@"): + return False + if any(ch.isspace() for ch in value): + return False + return True + def _infer_bundle_cluster_defaults(workflow: PreparedWorkflow) -> tuple[str, str]: """Derive ``spark_version`` and ``node_type_id`` defaults from task clusters. @@ -397,14 +475,68 @@ def _infer_bundle_cluster_defaults(workflow: PreparedWorkflow) -> tuple[str, str """ from collections import Counter - spark_versions = [hint["spark_version"] for hint in workflow.cluster_hints if hint.get("spark_version")] - node_types = [hint["node_type_id"] for hint in workflow.cluster_hints if hint.get("node_type_id")] + # C-29 (NB-ITER4-002): filter out unparseable spark_version / + # node_type_id hints before Counter so unresolved ADF expressions + # (e.g. ``@if(equals(item()?.photon,true),...)``) don't land as the + # bundle's default and break ``databricks bundle deploy``. + spark_versions = [ + hint["spark_version"] for hint in workflow.cluster_hints if _is_valid_spark_version(hint.get("spark_version")) + ] + node_types = [ + hint["node_type_id"] for hint in workflow.cluster_hints if _is_valid_node_type_id(hint.get("node_type_id")) + ] spark_version = Counter(spark_versions).most_common(1)[0][0] if spark_versions else _DEFAULT_SPARK_VERSION node_type_id = Counter(node_types).most_common(1)[0][0] if node_types else _DEFAULT_NODE_TYPE_ID return spark_version, node_type_id +def _infer_bundle_cluster_extras(workflow: PreparedWorkflow) -> dict[str, Any]: + """Surface non-default cluster fields shared across the workflow's tasks. + + Mines :attr:`PreparedWorkflow.cluster_hints` for cluster fields beyond + spark_version / node_type_id (including num_workers) and returns the + consensus values so the default job_cluster reflects ADF settings + end-to-end. + + Args: + workflow: The prepared workflow being written. + + Returns: + Dict of cluster fields ready to merge under ``new_cluster``. Only + the most common value across hints is propagated for each field; + ties are broken by first occurrence. + """ + from collections import Counter + + extras: dict[str, Any] = {} + extra_keys = ( + "num_workers", + "driver_node_type_id", + "data_security_mode", + "spark_env_vars", + "custom_tags", + "init_scripts", + "cluster_log_conf", + "spark_conf", + ) + for key in extra_keys: + values = [hint[key] for hint in workflow.cluster_hints if hint.get(key)] + if not values: + continue + # Use string repr to dedupe non-hashable dict entries while still + # picking the most common. + rep_counter: Counter[str] = Counter() + rep_to_value: dict[str, Any] = {} + for value in values: + rep = repr(value) + rep_counter[rep] += 1 + rep_to_value.setdefault(rep, value) + top_rep, _count = rep_counter.most_common(1)[0] + extras[key] = rep_to_value[top_rep] + return extras + + def _build_databricks_yml( bundle_name: str, catalog: str, @@ -487,40 +619,63 @@ def _build_databricks_yml( } -def _build_default_job_clusters(needed_keys: set[str]) -> list[dict[str, Any]]: +def _build_default_job_clusters( + needed_keys: set[str], + *, + extras: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: """Builds the job_clusters stanza, emitting only the clusters in use. Args: needed_keys: Set of job_cluster_key strings referenced by any task in the workflow. + extras: Optional cluster fields surfaced from per-task hints + (driver_node_type_id, spark_env_vars, custom_tags, ...). When + present these override the corresponding fields of the + multi-purpose default cluster so ADF-derived settings flow + into the emitted YAML. Returns: Ordered list of cluster definitions for inclusion under the job's ``job_clusters`` block. """ - builders = ( - (DEFAULT_JOB_CLUSTER_KEY, _build_default_cluster), + builders: tuple[tuple[str, Any], ...] = ( + (DEFAULT_JOB_CLUSTER_KEY, lambda: _build_default_cluster(extras)), (SINGLE_NODE_JOB_CLUSTER_KEY, _build_single_node_cluster), (MULTI_NODE_JOB_CLUSTER_KEY, _build_multi_node_cluster), ) return [builder() for key, builder in builders if key in needed_keys] -def _build_default_cluster() -> dict[str, Any]: +def _build_default_cluster(extras: dict[str, Any] | None = None) -> dict[str, Any]: """Builds the multi-purpose default job_cluster used for legacy bindings. + Args: + extras: Optional cluster fields lifted from per-task hints to + merge into ``new_cluster`` (num_workers, driver_node_type_id, + spark_env_vars, custom_tags, init_scripts, cluster_log_conf, + spark_conf, data_security_mode). ``num_workers`` overrides the + default single-worker value and ``data_security_mode`` overrides + the default ``SINGLE_USER`` value when supplied. + Returns: - Cluster definition with one worker and bundle-variable knobs for - spark_version and node_type_id. + Cluster definition with the mined (or default single) worker count + and bundle-variable knobs for spark_version and node_type_id, plus + any merged extras. """ + new_cluster: dict[str, Any] = { + "spark_version": "${var.spark_version}", + "node_type_id": "${var.node_type_id}", + "num_workers": 1, + "data_security_mode": "SINGLE_USER", + "single_user_name": "${workspace.current_user.userName}", + } + if extras: + for key, value in extras.items(): + new_cluster[key] = value return { "job_cluster_key": DEFAULT_JOB_CLUSTER_KEY, - "new_cluster": { - "spark_version": "${var.spark_version}", - "node_type_id": "${var.node_type_id}", - "num_workers": 1, - "data_security_mode": "SINGLE_USER", - }, + "new_cluster": new_cluster, } @@ -539,6 +694,7 @@ def _build_single_node_cluster() -> dict[str, Any]: "node_type_id": "${var.node_type_id}", "is_single_node": True, "data_security_mode": "SINGLE_USER", + "single_user_name": "${workspace.current_user.userName}", }, } @@ -557,6 +713,7 @@ def _build_multi_node_cluster() -> dict[str, Any]: "node_type_id": MULTI_NODE_CLUSTER_NODE_TYPE_ID, "num_workers": 2, "data_security_mode": "SINGLE_USER", + "single_user_name": "${workspace.current_user.userName}", }, } @@ -634,16 +791,22 @@ def _value_needs_manual_handling(value: Any) -> bool: def _extract_manual_parameters_from_existing_notebook_tasks( tasks: list[dict[str, Any]], ) -> list[ManualParameter]: - """Finds base_parameters flowx couldn't evaluate for existing-notebook tasks.""" + """Finds base_parameters flowx couldn't evaluate for notebook tasks. + + Previously this scan skipped stub notebooks under ``../src/`` because + their bodies could (in principle) be patched to inline the runtime + computation. In practice stub tasks for activities that have no + deterministic translation are emitted with raw ADF expression values + that ``dbutils.widgets.get`` returns verbatim, which fails at runtime. + Walking both the absolute-path and bundle-relative cases drops the + broken values and surfaces them as a SETUP.md row instead. + """ manual_parameters: list[ManualParameter] = [] for task in _iter_tasks_recursively(tasks): notebook_task = task.get("notebook_task") or {} notebook_path = notebook_task.get("notebook_path", "") base_params = notebook_task.get("base_parameters") - # Bundle-relative paths (``../src/...``) can have their notebook - # bodies patched to inline the runtime computation; absolute paths - # belong to the user's existing notebooks and must be surfaced. - if not notebook_path.startswith("/") or not isinstance(base_params, dict): + if not isinstance(base_params, dict): continue keys_to_drop: list[str] = [] for key, value in base_params.items(): @@ -704,17 +867,45 @@ def _bind_cluster_to_notebook_tasks(tasks: list[dict[str, Any]]) -> None: continue compute_mode = task.get("_compute_mode") if compute_mode == "serverless": + # Serverless cannot host jar/whl libraries. When the task + # ships libraries we must still bind a classic cluster so the + # Jobs API accepts the libraries block. + if _task_has_jar_or_whl_libraries(task): + task["job_cluster_key"] = DEFAULT_JOB_CLUSTER_KEY continue cluster_key = COMPUTE_MODE_TO_CLUSTER_KEY.get(compute_mode or "") if cluster_key is not None: task["job_cluster_key"] = cluster_key continue notebook_path = notebook_task.get("notebook_path", "") + # Stub notebooks (../src/...) are normally left unbound for + # serverless compute. But when libraries are attached we must + # bind to a real cluster (NB-2) -- serverless cannot install + # jar / whl libraries. if notebook_path.startswith("../src/"): + if _task_has_jar_or_whl_libraries(task): + task["job_cluster_key"] = DEFAULT_JOB_CLUSTER_KEY continue task["job_cluster_key"] = DEFAULT_JOB_CLUSTER_KEY +def _task_has_jar_or_whl_libraries(task: dict[str, Any]) -> bool: + """Return True when *task* references a library shape that needs a cluster. + + JAR / EGG / whl / PyPI / Maven / CRAN entries all require a classic + cluster — they cannot be installed on serverless. Requirements files + are treated the same way to be safe. + """ + libs = task.get("libraries") + if not isinstance(libs, list): + return False + cluster_required = {"jar", "egg", "whl", "maven", "pypi", "cran", "requirements"} + for entry in libs: + if isinstance(entry, dict) and any(key in entry for key in cluster_required): + return True + return False + + def _rewrite_post_branch_dependencies(tasks: list[dict[str, Any]]) -> None: """Rewrites ``depends_on`` edges that target a condition_task to target its branches. @@ -779,24 +970,128 @@ def expand_terminals(condition_key: str, seen: set[str]) -> list[str]: _TASK_VALUE_REF = re.compile(r"\{\{tasks\.([^.]+)\.values\.[^}]+\}\}") -def _strip_dangling_task_value_refs(tasks: list[dict[str, Any]], all_task_keys: set[str]) -> None: +def _apply_schedule_to_job(job_def: dict[str, Any], spec: dict[str, Any]) -> None: + """Renders a workflow schedule spec onto a DAB job definition. + + C-10 (SCHED-001): translates the structured schedule dict produced by + ``engine._adf_trigger_to_schedule`` into either ``schedule:`` or + ``trigger:`` keys on the job YAML. Best-effort schedule shapes + (Tumbling / CustomEvents) fall back to a comment-style placeholder + so SETUP.md can capture them. + """ + kind = spec.get("kind") + if kind == "schedule": + if "quartz_cron_expression" in spec: + schedule_block: dict[str, Any] = { + "quartz_cron_expression": spec["quartz_cron_expression"], + "timezone_id": spec.get("timezone_id", "UTC"), + } + if spec.get("pause_status"): + schedule_block["pause_status"] = spec["pause_status"] + job_def["schedule"] = schedule_block + return + # Tumbling fallback -- attach a hint rather than emitting a + # malformed schedule. SETUP.md picks it up downstream. + job_def["schedule_setup_note"] = spec + return + if kind == "periodic": + # SCHED3-002: Day/Week/Month with interval > 1 maps to trigger.periodic. + trigger_block: dict[str, Any] = { + "periodic": { + "interval": spec.get("interval", 1), + "unit": spec.get("unit", "DAYS"), + } + } + if spec.get("pause_status"): + trigger_block["pause_status"] = spec["pause_status"] + job_def["trigger"] = trigger_block + return + if kind == "file_arrival": + trigger_block = { + "file_arrival": {"url": spec.get("url", "")}, + } + if spec.get("pause_status"): + trigger_block["pause_status"] = spec["pause_status"] + job_def["trigger"] = trigger_block + return + if kind == "manual_setup": + # No DAB primitive -- surface the raw spec so SETUP.md can flag it. + job_def["schedule_setup_note"] = spec + return + + +def _strip_dangling_task_value_refs( + tasks: list[dict[str, Any]], + all_task_keys: set[str], +) -> list[dict[str, str]]: """Replaces ``{{tasks.X.values.Y}}`` refs whose ``X`` is not in the bundle. + C-12 (VAREX-005): in addition to ``notebook_task.base_parameters``, + walk ``run_job_task.job_parameters``, ``condition_task.left`` and + ``condition_task.right``, plus the nested for_each body so cross-job + parameter passing surfaces are caught. C-05 fixes most variable + defaults; this safety net catches the residual cases (renames, + scoped-out variables) by emitting an empty string in place of the + dangling ref so SETUP.md §4 can flag it. + + C-43 (CF5-001 / CF5-002): blanking a *condition_task* operand silently + turns ``NOT_EQUAL('', '0')`` into an always-true predicate, so the + branch runs unconditionally with no signal. This function now records + each condition operand it neutralises and returns them so the caller + can surface a 'conditions neutralized — manual re-wiring required' + section in SETUP.md instead of failing silently. + Args: tasks: Top-level tasks for one job (mutated in place). all_task_keys: Task keys that do exist in this job (including those inside ``for_each_task.task`` bodies). + + Returns: + List of ``{task_key, field, original_ref}`` dicts for every + condition operand that was blanked. """ + neutralized: list[dict[str, str]] = [] + + def _is_dangling(value: Any) -> bool: + if not isinstance(value, str): + return False + match = _TASK_VALUE_REF.search(value) + return bool(match and match.group(1) not in all_task_keys) def visit(task: dict[str, Any]) -> None: notebook_task = task.get("notebook_task") or {} base_parameters = notebook_task.get("base_parameters") or {} for widget_name, value in list(base_parameters.items()): - if not isinstance(value, str): - continue - match = _TASK_VALUE_REF.search(value) - if match and match.group(1) not in all_task_keys: + if _is_dangling(value): base_parameters[widget_name] = "" + + # C-12: run_job_task.job_parameters references the parent's task + # values when crossing into an inner job. Strip dangling refs. + run_job_task = task.get("run_job_task") or {} + job_parameters = run_job_task.get("job_parameters") or {} + if isinstance(job_parameters, dict): + for param_name, value in list(job_parameters.items()): + if _is_dangling(value): + job_parameters[param_name] = "" + + # C-12: condition_task operands can also carry dangling refs + # when an upstream renamed task disappeared between rewrite + # passes. C-43: record each neutralised operand for SETUP.md. + condition_task = task.get("condition_task") or {} + if condition_task: + task_key = task.get("task_key", "") + for field_name in ("left", "right"): + operand = condition_task.get(field_name) + if _is_dangling(operand): + neutralized.append( + { + "task_key": str(task_key), + "field": field_name, + "original_ref": str(operand), + } + ) + condition_task[field_name] = "" + for_each = task.get("for_each_task") if for_each and isinstance(for_each.get("task"), dict): visit(for_each["task"]) @@ -804,6 +1099,8 @@ def visit(task: dict[str, Any]) -> None: for task in tasks: visit(task) + return neutralized + def _collect_all_task_keys(tasks: list[dict[str, Any]]) -> set[str]: """Collects every task_key reachable from the job's top-level task list.""" @@ -873,8 +1170,12 @@ def _build_job_resource( _augment_base_parameters(workflow.tasks, augment_scope) # Task values don't cross ``run_job_task`` boundaries; any such # reference in this job resolves to an empty string at runtime. Emit - # the empty string now so SETUP.md §4 flags it. - _strip_dangling_task_value_refs(workflow.tasks, _collect_all_task_keys(workflow.tasks)) + # the empty string now so SETUP.md §4 flags it. C-43: a blanked + # condition operand silently makes the predicate always-true, so record + # each neutralised condition for the SETUP.md re-wiring section. + _neutralized_conditions.extend( + _strip_dangling_task_value_refs(workflow.tasks, _collect_all_task_keys(workflow.tasks)) + ) job_def: dict[str, Any] = { "name": workflow.name, @@ -885,13 +1186,32 @@ def _build_job_resource( _bind_cluster_to_notebook_tasks(workflow.tasks) needed_keys = _collect_required_cluster_keys(workflow.tasks) if needed_keys: - job_def["job_clusters"] = _build_default_job_clusters(needed_keys) + cluster_extras = _infer_bundle_cluster_extras(workflow) + job_def["job_clusters"] = _build_default_job_clusters( + needed_keys, + extras=cluster_extras or None, + ) _strip_compute_mode_markers(workflow.tasks) if workflow.parameters: job_def["parameters"] = workflow.parameters + # C-10 (SCHED-001): render the workflow schedule / trigger spec. + schedule_spec = getattr(workflow, "schedule", None) + if schedule_spec: + _apply_schedule_to_job(job_def, schedule_spec) + # SCHED3-003: trigger-supplied per-pipeline parameter overrides + # update the matching job.parameter defaults so scheduled runs + # receive the trigger's pinned values instead of the bare pipeline + # default. Overrides only mutate existing declared parameters; + # unknown names are silently ignored to keep job_def well-formed. + overrides = schedule_spec.get("parameter_overrides") or {} + if overrides and job_def.get("parameters"): + for entry in job_def["parameters"]: + if entry.get("name") in overrides: + entry["default"] = overrides[entry["name"]] + return { "resources": { "jobs": { @@ -957,6 +1277,8 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]: # format, so secret discovery / setup tasks / control-flow handling # all match. pipelines: dict[str, list[dict]] = {} + pipeline_params: dict[str, list[dict[str, Any]]] = {} + pipeline_schedules: dict[str, dict[str, Any]] = {} for translation in report.get("translations", []): pipeline_name = translation.get("pipeline", "unknown") if translation.get("status") != "translated": @@ -965,9 +1287,27 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]: if not ir: continue pipelines.setdefault(pipeline_name, []).append(ir) + # Round-trip pipeline-level parameters either supplied per- + # translation (newer report shape) or alongside the ir under + # an ``ir.parameters`` key (older single-pipeline serialisations + # roundtripped through this aggregator). + params = translation.get("parameters") or ir.get("parameters") + if params and pipeline_name not in pipeline_params: + pipeline_params[pipeline_name] = list(params) + # Likewise carry pipeline-level ``schedule`` through to the + # rehydrated pipeline_dict so trigger-derived schedule / trigger + # blocks survive the aggregated report shape. + schedule = translation.get("schedule") or ir.get("schedule") + if schedule and pipeline_name not in pipeline_schedules: + pipeline_schedules[pipeline_name] = dict(schedule) for pipeline_name, task_irs in pipelines.items(): - workflow = _pipeline_dict_to_workflow({"name": pipeline_name, "tasks": task_irs}) + pipeline_dict: dict[str, Any] = {"name": pipeline_name, "tasks": task_irs} + if pipeline_params.get(pipeline_name): + pipeline_dict["parameters"] = pipeline_params[pipeline_name] + if pipeline_schedules.get(pipeline_name): + pipeline_dict["schedule"] = pipeline_schedules[pipeline_name] + workflow = _pipeline_dict_to_workflow(pipeline_dict) workflows.append(workflow) return workflows @@ -1010,13 +1350,24 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d for param in pipeline_dict.get("parameters") or []: entry: dict[str, Any] = {"name": param["name"]} if "default" in param and param["default"] is not None: - entry["default"] = normalize_value(str(param["default"])) + default_value = param["default"] + # Bool / int / float defaults must survive the JSON round-trip + # as their declared type so the emitted YAML carries a real + # boolean / number, not a quoted string. String defaults go + # through normalize_value to resolve embedded ADF refs. + if isinstance(default_value, bool): + entry["default"] = default_value + elif isinstance(default_value, (int, float)): + entry["default"] = default_value + else: + entry["default"] = normalize_value(str(default_value)) parameters.append(entry) pipeline = Pipeline( name=pipeline_dict.get("name", "unknown"), tasks=activities, parameters=parameters or None, translation_preferences=_reconstruct_preferences(pipeline_dict.get("translation_preferences")), + schedule=pipeline_dict.get("schedule"), ) return pipeline, parameters @@ -1036,12 +1387,15 @@ def _reconstruct_preferences(raw: dict[str, Any] | None) -> Any: return None from flowx.adapter.models import TranslationPreferences + # Reports authored before the databricks_task_compute option was + # removed may still carry that key; drop it silently so old reports + # remain rehydratable. return TranslationPreferences( copy_activity_paradigm=raw.get("copy_activity_paradigm", "notebook"), non_databricks_task_compute=raw.get("non_databricks_task_compute", "serverless"), use_lakeflow_connectors=raw.get("use_lakeflow_connectors", "existing"), - databricks_task_compute=raw.get("databricks_task_compute", "existing"), lakeflow_connector_type=raw.get("lakeflow_connector_type", "cdc"), + motif_consolidations=dict(raw.get("motif_consolidations") or {}), per_task=dict(raw.get("per_task") or {}), ) @@ -1096,6 +1450,7 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: value_kind=task_ir.get("value_kind", "literal"), notebook_code=task_ir.get("notebook_code"), notebook_imports=task_ir.get("notebook_imports", []), + raw_expression=task_ir.get("raw_expression"), ) if task_type == "WaitActivity": return WaitActivity( @@ -1131,13 +1486,15 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: **base, notebook_path=task_ir.get("notebook_path", ""), base_parameters=task_ir.get("base_parameters"), + notebook_path_unresolved=bool(task_ir.get("notebook_path_unresolved", False)), + notebook_path_expression=task_ir.get("notebook_path_expression"), + unresolved_libraries=list(task_ir.get("unresolved_libraries") or []), ) if task_type == "SparkJarActivity": return SparkJarActivity( **base, main_class_name=task_ir.get("main_class_name", ""), parameters=task_ir.get("parameters"), - libraries=task_ir.get("libraries"), ) if task_type == "SparkPythonActivity": return SparkPythonActivity( @@ -1165,6 +1522,11 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: items_expression=task_ir.get("items_expression", ""), inner_activities=[_reconstruct_ir(child) for child in task_ir.get("inner_activities") or []], concurrency=task_ir.get("concurrency"), + # C-31 (CF4-001): preserve bridge fields so the preparer can + # synthesise the inputs bridge after a JSON roundtrip. + inputs_bridge_notebook_code=task_ir.get("inputs_bridge_notebook_code"), + inputs_bridge_notebook_imports=list(task_ir.get("inputs_bridge_notebook_imports") or []), + inputs_bridge_required_parameters=dict(task_ir.get("inputs_bridge_required_parameters") or {}), ) if task_type == "IfConditionActivity": return IfConditionActivity( @@ -1174,6 +1536,12 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: right=task_ir.get("right", ""), if_true_activities=[_reconstruct_ir(child) for child in task_ir.get("if_true_activities") or []], if_false_activities=[_reconstruct_ir(child) for child in task_ir.get("if_false_activities") or []], + # C-14 (CF3-001 / VAREX3-001): preserve bridge fields so the + # preparer can re-synthesise the hidden _bridge SetVariable task + # after a JSON roundtrip. + bridge_notebook_code=task_ir.get("bridge_notebook_code"), + bridge_notebook_imports=list(task_ir.get("bridge_notebook_imports") or []), + bridge_required_parameters=dict(task_ir.get("bridge_required_parameters") or {}), ) if task_type == "SwitchActivity": return SwitchActivity( @@ -1187,6 +1555,12 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: for case in task_ir.get("cases") or [] ], default_activities=[_reconstruct_ir(child) for child in task_ir.get("default_activities") or []], + # C-14 (CF3-001 / VAREX3-001): preserve bridge fields for Switch + # so the preparer can re-synthesise the bridge task after a + # JSON roundtrip. + bridge_notebook_code=task_ir.get("bridge_notebook_code"), + bridge_notebook_imports=list(task_ir.get("bridge_notebook_imports") or []), + bridge_required_parameters=dict(task_ir.get("bridge_required_parameters") or {}), ) if task_type == "MotifActivity": return MotifActivity( @@ -1235,6 +1609,9 @@ def _common_activity_kwargs(task_ir: dict[str, Any]) -> dict[str, Any]: "min_retry_interval_millis": task_ir.get("min_retry_interval_millis"), "depends_on": _reconstruct_dependencies(task_ir.get("depends_on")), "cluster": task_ir.get("cluster"), + "existing_cluster_id": task_ir.get("existing_cluster_id"), + "libraries": task_ir.get("libraries"), + "parameter_approximations": list(task_ir.get("parameter_approximations") or []), "required_parameters": dict(task_ir.get("required_parameters") or {}), "compute_mode": task_ir.get("compute_mode"), } diff --git a/src/orchestra/bundler/inner_job_params.py b/src/orchestra/bundler/inner_job_params.py index 20e4e58..ab36e87 100644 --- a/src/orchestra/bundler/inner_job_params.py +++ b/src/orchestra/bundler/inner_job_params.py @@ -38,6 +38,7 @@ def collect_inner_job_params( tasks: list[dict[str, Any]], *, raw_ir_tasks: list[dict[str, Any]] | None = None, + variable_task_keys: dict[str, str] | None = None, ) -> tuple[list[dict[str, Any]], dict[str, str]]: """Scans task dicts for parameter references and return declarations + pass-through map. @@ -46,6 +47,15 @@ def collect_inner_job_params( raw_ir_tasks: Optional raw IR dicts (before DAB conversion) to scan for references in fields that are consumed during conversion (e.g. WebActivity ``url``, ``body``). + variable_task_keys: C-06 (VAREX-004): mapping of pipeline-variable + names to the setter task_key on the parent job that owns the + variable. Names that match a variable do NOT get declared as + inner-job parameters; instead the parent passes the variable's + task-value reference (``{{tasks..values.}}``) + through the ``job_parameters`` map. Without this the parent + would emit ``{{job.parameters.}}`` referring to a name + that's never declared on the parent job, so the inner job + would receive an empty string. Returns: Tuple of: @@ -53,19 +63,28 @@ def collect_inner_job_params( the inner job definition. - ``job_parameters``: dict mapping param name -> parent expression, suitable for the ``run_job_task.job_parameters`` block. ``item`` - always maps to ``"{{input}}"``, pipeline/variable params map to - ``"{{job.parameters.}}"``. + always maps to ``"{{input}}"``, pipeline params map to + ``"{{job.parameters.}}"``, variables resolved via the + *variable_task_keys* map route through ``{{tasks.X.values.Y}}``. """ param_names: set[str] = set() item_field_names: set[str] = set() + variable_names: set[str] = set() - _scan_tasks(tasks, param_names, item_field_names=item_field_names) + _scan_tasks(tasks, param_names, item_field_names=item_field_names, variable_names=variable_names) if raw_ir_tasks: - _scan_ir_tasks(raw_ir_tasks, param_names, item_field_names=item_field_names) + _scan_ir_tasks(raw_ir_tasks, param_names, item_field_names=item_field_names, variable_names=variable_names) + + var_task_keys = variable_task_keys or {} parameters: list[dict[str, Any]] = [] for name in sorted(param_names): + # C-06: variables with a known setter task on the parent job route + # via {{tasks.X.values.Y}} -- they must NOT show up as inner-job + # parameter declarations. + if name in variable_names and name in var_task_keys: + continue param: dict[str, Any] = {"name": name} if name != "item": param["default"] = "" @@ -80,6 +99,9 @@ def collect_inner_job_params( job_parameters[name] = "{{input}}" elif name in item_field_names: job_parameters[name] = "{{input." + name + "}}" + elif name in variable_names and name in var_task_keys: + setter = var_task_keys[name] + job_parameters[name] = "{{tasks." + setter + ".values." + name + "}}" else: job_parameters[name] = "{{job.parameters." + name + "}}" @@ -119,6 +141,7 @@ def _scan_tasks( param_names: set[str], *, item_field_names: set[str] | None = None, + variable_names: set[str] | None = None, ) -> None: """Recursively scan task dicts for ADF parameter references. @@ -126,28 +149,31 @@ def _scan_tasks( tasks: List of task dicts to scan. param_names: Accumulator set of discovered parameter names. item_field_names: Optional accumulator for field names from item().field refs. + variable_names: Optional accumulator for names sourced from + ``variables('X')`` references (separate from pipeline params). """ + kw: dict[str, Any] = {"item_field_names": item_field_names, "variable_names": variable_names} for task in tasks: notebook_task = task.get("notebook_task", {}) params = notebook_task.get("base_parameters", {}) for value in params.values(): - _extract_refs(value, param_names, item_field_names=item_field_names) + _extract_refs(value, param_names, **kw) run_job_task = task.get("run_job_task", {}) for value in run_job_task.get("job_parameters", {}).values(): - _extract_refs(value, param_names, item_field_names=item_field_names) + _extract_refs(value, param_names, **kw) condition_task = task.get("condition_task", {}) if condition_task: - _extract_refs(condition_task.get("left", ""), param_names, item_field_names=item_field_names) - _extract_refs(condition_task.get("right", ""), param_names, item_field_names=item_field_names) - _scan_tasks(condition_task.get("if_true", []), param_names, item_field_names=item_field_names) - _scan_tasks(condition_task.get("if_false", []), param_names, item_field_names=item_field_names) + _extract_refs(condition_task.get("left", ""), param_names, **kw) + _extract_refs(condition_task.get("right", ""), param_names, **kw) + _scan_tasks(condition_task.get("if_true", []), param_names, **kw) + _scan_tasks(condition_task.get("if_false", []), param_names, **kw) for_each_task = task.get("for_each_task", {}) body = for_each_task.get("task") if body: - _scan_tasks([body], param_names, item_field_names=item_field_names) + _scan_tasks([body], param_names, **kw) def _scan_ir_tasks( @@ -155,6 +181,7 @@ def _scan_ir_tasks( param_names: set[str], *, item_field_names: set[str] | None = None, + variable_names: set[str] | None = None, ) -> None: """Scans raw IR task dicts for parameter references in all fields. @@ -162,8 +189,9 @@ def _scan_ir_tasks( ir_tasks: Raw serialised IR task dicts. param_names: Accumulator set of discovered parameter names. item_field_names: Optional accumulator for field names from item().field refs. + variable_names: Optional accumulator for variable names. """ - field_name_kwargs = {"item_field_names": item_field_names} + field_name_kwargs = {"item_field_names": item_field_names, "variable_names": variable_names} for task_dict in ir_tasks: _extract_refs(task_dict.get("url", ""), param_names, **field_name_kwargs) _extract_refs(task_dict.get("body"), param_names, **field_name_kwargs) @@ -195,6 +223,7 @@ def _extract_refs( param_names: set[str], *, item_field_names: set[str] | None = None, + variable_names: set[str] | None = None, ) -> None: """Extracts parameter names from a single value that may be a string or ADF expression dict.""" text = "" @@ -211,6 +240,8 @@ def _extract_refs( for match in _VARIABLES_RE.finditer(text): param_names.add(match.group(1)) + if variable_names is not None: + variable_names.add(match.group(1)) for match in _ITEM_FIELD_RE.finditer(text): field_name = match.group(1) diff --git a/src/orchestra/bundler/prereqs_writer.py b/src/orchestra/bundler/prereqs_writer.py index 934b8bc..9408f5b 100644 --- a/src/orchestra/bundler/prereqs_writer.py +++ b/src/orchestra/bundler/prereqs_writer.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from typing import Any -from flowx.models.dab import DabNotebook +from flowx.models.dab import DabNotebook, ParameterApproximation, SecretInstruction # Regexes used to mine the generated artifacts for external dependencies. # Kept as compiled patterns so :func:`build_prereqs` is cheap to call. @@ -110,6 +110,34 @@ class Prereqs: compute_notes: list[str] = field(default_factory=list) network_endpoints: list[NetworkEndpoint] = field(default_factory=list) manual_parameters: list[ManualParameter] = field(default_factory=list) + parameter_approximations: list[ParameterApproximation] = field(default_factory=list) + # VAREX3-003: variables mutated inside a ForEach inner-job that a + # sibling task reads. Each entry is the SetupTask.config dict shape + # ({variable_name, parent_foreach, message}). + manual_variable_rollups: list[dict[str, Any]] = field(default_factory=list) + # C-28 (NB-ITER4-001): notebook activities whose ADF ``notebookPath`` is + # a runtime expression the translator couldn't resolve. Each entry is + # the SetupTask.config dict ({task_key, activity_name, expression, + # widget_name}). + dynamic_notebook_dispatches: list[dict[str, Any]] = field(default_factory=list) + # C-30 (NB-ITER4-003): library descriptor jar/whl paths the translator + # couldn't resolve to a literal/dab_ref. Each entry is the SetupTask + # config dict ({task_key, library_type, expression, missing}). + unresolved_libraries: list[dict[str, Any]] = field(default_factory=list) + # C-33 (VAREX4-001/CF4-003): SetVariable activities whose ADF + # expression couldn't be lowered. Each entry is the SetupTask config + # dict ({task_key, variable_name, expression}). + manual_variable_inits: list[dict[str, Any]] = field(default_factory=list) + # C-36 (SCHED4-001): scheduled jobs whose recurrence carried + # hours/minutes/weekDays the cron emitter could not encode. + manual_schedule_time_of_day: list[dict[str, Any]] = field(default_factory=list) + # C-39 (LSC4-004): MSI / CredentialReference cluster substitutions. + manual_credentials: list[dict[str, Any]] = field(default_factory=list) + # C-43 (CF5-001 / CF5-002): condition_task operands the bundler had to + # blank because they referenced a task in another job. Each entry is + # {task_key, field, original_ref}. A blanked operand makes the + # predicate always-true, so the user must re-wire the condition. + neutralized_conditions: list[dict[str, str]] = field(default_factory=list) def is_empty(self) -> bool: """Return ``True`` when nothing needs to happen before ``bundle run``.""" @@ -121,6 +149,14 @@ def is_empty(self) -> bool: and not self.compute_notes and not self.network_endpoints and not self.manual_parameters + and not self.parameter_approximations + and not self.manual_variable_rollups + and not self.dynamic_notebook_dispatches + and not self.unresolved_libraries + and not self.manual_variable_inits + and not self.manual_schedule_time_of_day + and not self.manual_credentials + and not self.neutralized_conditions ) @@ -323,6 +359,15 @@ def build_prereqs( cross_bundle_variables: dict[str, str] | None = None, compute_notes: list[str] | None = None, manual_parameters: list[ManualParameter] | None = None, + parameter_approximations: list[ParameterApproximation] | None = None, + manual_variable_rollups: list[dict[str, Any]] | None = None, + secret_instructions: list[SecretInstruction] | None = None, + dynamic_notebook_dispatches: list[dict[str, Any]] | None = None, + unresolved_libraries: list[dict[str, Any]] | None = None, + manual_variable_inits: list[dict[str, Any]] | None = None, + manual_schedule_time_of_day: list[dict[str, Any]] | None = None, + manual_credentials: list[dict[str, Any]] | None = None, + neutralized_conditions: list[dict[str, str]] | None = None, ) -> Prereqs: """Assemble a :class:`Prereqs` from the bundle's generated artifacts. @@ -348,14 +393,30 @@ def build_prereqs( # the tasks (in case upstream still emits them). cross_bundle.extend(collect_cross_bundle_refs(tasks, known_bundle_jobs)) + # LSC3-006: union notebook-scanned secrets with the workflow's typed + # SecretInstruction list so SETUP.md Option A (scope/key checklist) and + # Option B (create_secrets.py from workflow.secrets) reference the same + # set of (scope, key) pairs. De-dupe by hash; later additions don't + # overwrite earlier values. + secrets = scan_notebooks_for_secrets(notebooks) + for instruction in secret_instructions or []: + secrets.setdefault(instruction.scope, set()).add(instruction.key) return Prereqs( - secrets=scan_notebooks_for_secrets(notebooks), + secrets=secrets, missing_notebooks=collect_missing_notebooks(notebooks, tasks), cross_bundle_refs=cross_bundle, empty_parameters=collect_empty_parameters(tasks), compute_notes=list(compute_notes or []), network_endpoints=collect_network_endpoints(notebooks), manual_parameters=list(manual_parameters or []), + parameter_approximations=list(parameter_approximations or []), + manual_variable_rollups=list(manual_variable_rollups or []), + dynamic_notebook_dispatches=list(dynamic_notebook_dispatches or []), + unresolved_libraries=list(unresolved_libraries or []), + manual_variable_inits=list(manual_variable_inits or []), + manual_schedule_time_of_day=list(manual_schedule_time_of_day or []), + manual_credentials=list(manual_credentials or []), + neutralized_conditions=list(neutralized_conditions or []), ) @@ -518,6 +579,179 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: ) lines.append("") + if prereqs.parameter_approximations: + lines.append("## Parameter substitutions") + lines.append("") + lines.append( + "Flowx mapped the ADF expressions below to Databricks dynamic value " + "references so they land directly in the bundle YAML. The substitutions are " + "semantically *close* but not identical to the originals; review the listed " + "caveats and decide whether each replacement is acceptable for your workload." + ) + lines.append("") + lines.append("| Task | Widget | Original ADF expression | Replacement | Caveat |") + lines.append("|---|---|---|---|---|") + for approximation in prereqs.parameter_approximations: + lines.append( + f"| `{approximation.task_key}` | `{approximation.widget_name}` " + f"| `{approximation.raw_expression}` | `{approximation.replacement}` " + f"| {approximation.note} |" + ) + lines.append("") + + if prereqs.dynamic_notebook_dispatches: + lines.append("## Dynamic notebook dispatch") + lines.append("") + lines.append( + "The ADF activities below carried a runtime expression for " + "`notebookPath`. Flowx emitted a dispatch-stub notebook for " + "each one that reads the resolved path from the listed widget and " + "calls `dbutils.notebook.run()`. Supply the widget value at job " + "runtime (via `--params`, a parent task value, or job parameter " + "default) so the stub can dispatch to the correct notebook." + ) + lines.append("") + lines.append("| Task | Activity | Widget | Original ADF expression |") + lines.append("|---|---|---|---|") + for entry in prereqs.dynamic_notebook_dispatches: + task_key = entry.get("task_key", "") + activity_name = entry.get("activity_name", "") + widget_name = entry.get("widget_name", "") + expression = entry.get("expression", "") + lines.append( + f"| `{task_key}` | `{activity_name}` | `dbutils.widgets.get('{widget_name}')` | `{expression}` |" + ) + lines.append("") + + if prereqs.unresolved_libraries: + lines.append("## Unresolved libraries") + lines.append("") + lines.append( + "The library descriptors below carried ADF expressions that " + "couldn't be reduced to a real path or DAB reference. Without " + "resolution the cluster would try to install a file literally " + "named like the expression and fail at job-run time. Either " + "populate the missing identifiers (see the `Missing` column) " + "or replace the entry with a static path before deploying." + ) + lines.append("") + lines.append("| Task | Library type | Expression | Missing |") + lines.append("|---|---|---|---|") + for entry in prereqs.unresolved_libraries: + task_key = entry.get("task_key", "") + lib_type = entry.get("library_type", "") + expression = entry.get("expression", "") + missing = ", ".join(entry.get("missing") or []) or "*(unknown)*" + lines.append(f"| `{task_key}` | `{lib_type}` | `{expression}` | {missing} |") + lines.append("") + + if prereqs.manual_variable_inits: + lines.append("## Manual variable initialisation") + lines.append("") + lines.append( + "The ADF SetVariable activities below carried expressions the " + "translator couldn't lower. Flowx blanked the variable's " + "initial value to keep the bundle YAML valid. Compute the real " + "value yourself (e.g. via a parent task value or runtime widget) " + "before downstream tasks read the variable." + ) + lines.append("") + lines.append("| Task | Variable | Original ADF expression |") + lines.append("|---|---|---|") + for entry in prereqs.manual_variable_inits: + task_key = entry.get("task_key", "") + variable_name = entry.get("variable_name", "") + expression = entry.get("expression", "") + lines.append(f"| `{task_key}` | `{variable_name}` | `{expression}` |") + lines.append("") + + if prereqs.manual_schedule_time_of_day: + lines.append("## Manual schedule time-of-day") + lines.append("") + lines.append( + "The ADF triggers below declared a `schedule` block (hours / " + "minutes / weekDays) the cron emitter couldn't fully encode. " + "Review the spec and add the desired time-of-day to the job's " + "`schedule.quartz_cron_expression` manually." + ) + lines.append("") + lines.append("| Pipeline | Frequency | Interval | Time-of-day spec |") + lines.append("|---|---|---|---|") + for entry in prereqs.manual_schedule_time_of_day: + pipeline = entry.get("pipeline", "") + frequency = entry.get("frequency", "") + interval = entry.get("interval", "") + tod_spec = entry.get("time_of_day_note", "") + lines.append(f"| `{pipeline}` | `{frequency}` | `{interval}` | `{tod_spec}` |") + lines.append("") + + if prereqs.manual_credentials: + lines.append("## Manual credential setup") + lines.append("") + lines.append( + "The cluster compute backing the tasks below was authenticated in " + "ADF via a managed identity / CredentialReference that has no " + "direct Databricks equivalent. Flowx defaulted the bundle's " + "default_cluster to `single_user_name: ${workspace.current_user.userName}` " + "so deployment works for the deploying user, but production runs " + "should swap that for a service principal." + ) + lines.append("") + lines.append("| Source | Linked service | ADF authentication | Suggested Databricks setup |") + lines.append("|---|---|---|---|") + for entry in prereqs.manual_credentials: + source = entry.get("activity_name") or entry.get("source", "") + linked_service = entry.get("linked_service", "") + auth = entry.get("authentication", "") + note = entry.get( + "note", + "Swap `single_user_name` to the SP application ID or set `run_as.service_principal_name` on the job.", + ) + lines.append(f"| `{source}` | `{linked_service}` | `{auth}` | {note} |") + lines.append("") + + if prereqs.neutralized_conditions: + lines.append("## Conditions neutralized to always-true — manual re-wiring required") + lines.append("") + lines.append( + "The IfCondition tasks below referenced a task value that lives only " + "in another job (typically a parent-job init task hoisted out of a " + "split-out ForEach inner job). Databricks task values cannot cross " + "`run_job_task` boundaries, so Flowx blanked the operand. A blanked " + "operand makes the predicate `NOT_EQUAL('', '0')` **always true**, so the " + "branch now runs unconditionally. Re-wire each condition below — either " + "recompute the operand inside this job or pass it as a job parameter." + ) + lines.append("") + lines.append("| Condition task | Operand | Original reference |") + lines.append("|---|---|---|") + for entry in prereqs.neutralized_conditions: + task_key = entry.get("task_key", "") + field_name = entry.get("field", "") + original = entry.get("original_ref", "") + lines.append(f"| `{task_key}` | `{field_name}` | `{original}` |") + lines.append("") + + if prereqs.manual_variable_rollups: + lines.append("## Manual variable roll-ups") + lines.append("") + lines.append( + "These variables are mutated inside a ForEach inner-job but read by a " + "sibling task in the parent. Databricks task values cannot cross " + "`run_job_task` boundaries, so the sibling reads the stale init value. " + "Add a roll-up notebook task that copies the final value back to a " + "parent-scope task value before the sibling task runs." + ) + lines.append("") + lines.append("| Variable | ForEach task | Workaround |") + lines.append("|---|---|---|") + for rollup in prereqs.manual_variable_rollups: + var_name = rollup.get("variable_name", "") + parent_key = rollup.get("parent_foreach", "") + message = rollup.get("message", "") + lines.append(f"| `{var_name}` | `{parent_key}` | {message} |") + lines.append("") + if prereqs.network_endpoints: lines.append("## Networking") lines.append("") diff --git a/src/orchestra/bundler/setup_generator.py b/src/orchestra/bundler/setup_generator.py index e637cf5..76059d0 100644 --- a/src/orchestra/bundler/setup_generator.py +++ b/src/orchestra/bundler/setup_generator.py @@ -74,11 +74,21 @@ def _generate_secrets_setup_notebook(secrets: list[SecretInstruction]) -> DabNot for s in secrets: scopes.setdefault(s.scope, []).append(s) - body_parts: list[str] = [] + # C-46 (LSC5-002): the ``dbutils.secrets`` submodule is read-only + # (get / getBytes / list / listScopes) — ``createScope`` and ``put`` do + # not exist and raise AttributeError on the first cell. Provision via + # the Databricks SDK ``WorkspaceClient`` instead. + init_cell = textwrap.dedent("""\ + from databricks.sdk import WorkspaceClient + + w = WorkspaceClient() + """).rstrip() + + body_parts: list[str] = [init_cell] for scope_name, scope_secrets in sorted(scopes.items()): lines: list[str] = [f"# Create scope: {scope_name}"] lines.append("try:") - lines.append(f' dbutils.secrets.createScope(scope="{scope_name}")') + lines.append(f' w.secrets.create_scope(scope="{scope_name}")') lines.append(f' print("Created scope: {scope_name}")') lines.append("except Exception as e:") lines.append(' if "RESOURCE_ALREADY_EXISTS" in str(e):') @@ -89,7 +99,7 @@ def _generate_secrets_setup_notebook(secrets: list[SecretInstruction]) -> DabNot for secret in scope_secrets: lines.append(f"# {secret.value_source}") - lines.append(f'dbutils.secrets.put(scope="{scope_name}", key="{secret.key}", string_value="PLACEHOLDER")') + lines.append(f'w.secrets.put_secret(scope="{scope_name}", key="{secret.key}", string_value="PLACEHOLDER")') lines.append(f'print("Created secret: {scope_name}/{secret.key}")') lines.append("") diff --git a/src/orchestra/models/adf_ast.py b/src/orchestra/models/adf_ast.py index e25c90e..9e2d2f9 100644 --- a/src/orchestra/models/adf_ast.py +++ b/src/orchestra/models/adf_ast.py @@ -105,10 +105,14 @@ class AdfLinkedServiceReference: Attributes: reference_name: Logical name of the linked service. type: Reference type (always ``"LinkedServiceReference"``). + parameters: Runtime parameter overrides supplied by the activity, + keyed by parameter name. These flow into the resolver as + ``@linkedService().X`` substitutions. """ reference_name: str type: str = "LinkedServiceReference" + parameters: dict[str, Any] | None = None # --------------------------------------------------------------------------- @@ -241,12 +245,46 @@ class AdfDefinitions: datasets: Dataset definitions keyed by name. linked_services: Linked service definitions keyed by name. triggers: Trigger definitions. + global_parameters: Factory-level ``globalParameters`` keyed by name, + with each value parsed into ``{"type": str, "value": Any}``. """ pipelines: list[AdfPipeline] datasets: dict[str, AdfDataset] = field(default_factory=dict) linked_services: dict[str, AdfLinkedService] = field(default_factory=dict) triggers: list[AdfTrigger] = field(default_factory=list) + global_parameters: dict[str, Any] = field(default_factory=dict) + + def get_dataset(self, name: str | None) -> AdfDataset | None: + """Case-insensitive dataset lookup. + + LSC3-005: ADF identifiers are documented as case-insensitive; pipelines + sometimes reference a dataset by a different casing than the source + JSON file declares. Tolerate the mismatch instead of returning None. + """ + if not name: + return None + found = self.datasets.get(name) + if found is not None: + return found + lowered = name.lower() + for key, value in self.datasets.items(): + if key.lower() == lowered: + return value + return None + + def get_linked_service(self, name: str | None) -> AdfLinkedService | None: + """Case-insensitive linked service lookup; see :meth:`get_dataset`.""" + if not name: + return None + found = self.linked_services.get(name) + if found is not None: + return found + lowered = name.lower() + for key, value in self.linked_services.items(): + if key.lower() == lowered: + return value + return None # --------------------------------------------------------------------------- diff --git a/src/orchestra/models/dab.py b/src/orchestra/models/dab.py index 23bd002..ec0b85c 100644 --- a/src/orchestra/models/dab.py +++ b/src/orchestra/models/dab.py @@ -123,6 +123,27 @@ class SetupTask: config: dict[str, Any] = field(default_factory=dict) +@dataclass(slots=True, kw_only=True) +class ParameterApproximation: + """A base_parameter where flowx substituted a DAB dynamic value for an + ADF expression with non-identical semantics (e.g. ``utcnow()`` mapped to + job start time). + + Attributes: + task_key: DAB task key. + widget_name: The base_parameter name. + raw_expression: The original ADF expression text. + replacement: The DAB dynamic value reference flowx emitted. + note: Human-readable caveat explaining the semantic difference. + """ + + task_key: str + widget_name: str + raw_expression: str + replacement: str + note: str + + # --------------------------------------------------------------------------- # Top-level bundle # --------------------------------------------------------------------------- diff --git a/src/orchestra/models/ir.py b/src/orchestra/models/ir.py index 3bb7fa7..2fd263c 100644 --- a/src/orchestra/models/ir.py +++ b/src/orchestra/models/ir.py @@ -12,12 +12,34 @@ @dataclass(slots=True, kw_only=True) class ExpressionResult: - """Result of resolving an ADF expression.""" + """Result of resolving an ADF expression. - kind: str # "literal", "dab_ref", "notebook_code" + Attributes: + kind: One of ``"literal"`` / ``"dab_ref"`` / ``"notebook_code"``. + value: The resolved value text. + imports: Imports the notebook_code value needs. + required_parameters: Widget name -> DAB ref mapping for + base_parameters threading. + notes: Free-form caveats surfaced in SETUP.md. + was_string_literal: C-34 (VAREX4-002): True when the original + ADF token was a quoted string (``'09'``, ``"12"``) so the + function-call codegen path can ``repr()`` it instead of + emitting a bare numeric token that strips quotedness. + was_bool_literal: C-34 (VAREX4-003): True when the original ADF + token was ``true`` / ``false``. ADF Booleans serialise as + the lowercase strings ``'true'``/``'false'`` on the + SetVariable consumer side (post-C-21), so comparisons must + emit ``'true'`` / ``'false'`` Python strings rather than the + bare Python ``True`` / ``False``. + """ + + kind: str value: str imports: list[str] = field(default_factory=list) required_parameters: dict[str, str] = field(default_factory=dict) + notes: list[str] = field(default_factory=list) + was_string_literal: bool = False + was_bool_literal: bool = False @dataclass(slots=True, kw_only=True) @@ -46,6 +68,12 @@ class Activity: min_retry_interval_millis: Minimum delay between retries (ms). depends_on: Upstream task dependencies. cluster: Cluster configuration for the task, if any. + existing_cluster_id: ID of an existing all-purpose cluster the task + should run on. + libraries: Task-scoped library descriptors carried through from ADF. + Each entry is a supported Databricks task library; see + https://docs.databricks.com/aws/en/dev-tools/bundles/library-dependencies + for the supported shapes. """ name: str @@ -56,10 +84,13 @@ class Activity: min_retry_interval_millis: int | None = None depends_on: list[Dependency] | None = None cluster: dict[str, Any] | None = None - # Widget-name → DAB-ref mapping for every `dbutils.widgets.get()` call - # that shows up in any notebook_code the translator produced for this - # activity. Preparers thread these into ``base_parameters`` so DAB - # resolves the refs at job runtime. + existing_cluster_id: str | None = None + libraries: list[dict[str, Any]] | None = None + # Approximate parameter substitutions made at translation time (e.g. + # ``utcnow()`` mapped to ``{{job.start_time.iso_datetime}}``). Each + # entry has keys ``widget_name``, ``raw_expression``, ``replacement``, + # and ``note``; the bundler surfaces these in SETUP.md. + parameter_approximations: list[dict[str, str]] = field(default_factory=list) required_parameters: dict[str, str] = field(default_factory=dict) # Compute mode stamped by the pipeline modifier in response to user # preferences. One of "serverless", "classic_single_node", @@ -75,11 +106,27 @@ class NotebookActivity(Activity): notebook_path: Workspace path to the notebook. base_parameters: Parameters passed to the notebook at runtime. linked_service_definition: Raw linked-service dictionary for cluster config. + notebook_path_unresolved: C-28 (NB-ITER4-001): True when the ADF + ``notebookPath`` is a dynamic expression the translator couldn't + reduce to a literal/dab_ref workspace path. The preparer emits + a dispatch-stub notebook that reads ``notebook_path`` from a + widget and ``dbutils.notebook.run()``s the resolved value. + notebook_path_expression: Raw ADF expression text captured when + ``notebook_path_unresolved`` is True, surfaced in SETUP.md. + unresolved_libraries: C-30 (NB-ITER4-003): library descriptor + entries whose value (jar/whl/egg/requirements path) carried an + ADF expression the resolver couldn't reduce to a literal or + dab_ref. Each entry has ``type`` (library shape key), + ``expression`` (raw ADF text), and ``missing`` (referenced + identifier names not bound in the translation context). """ notebook_path: str base_parameters: dict[str, str] | None = None linked_service_definition: dict[str, Any] | None = None + notebook_path_unresolved: bool = False + notebook_path_expression: str | None = None + unresolved_libraries: list[dict[str, Any]] = field(default_factory=list) @dataclass(slots=True, kw_only=True) @@ -132,11 +179,23 @@ class ForEachActivity(Activity): inner_activities: Translated activities executed for each item. concurrency: Maximum parallel iterations (maps to Databricks ``for_each_task.concurrency``). + inputs_bridge_notebook_code: C-31 (CF4-001): when the items + expression resolves to ``notebook_code`` (e.g. + ``@split(variables('fecha'),',')``), the translator captures + the Python code here while the full TranslationContext is + available. The preparer reads it instead of re-resolving + against an empty context (which silently failed before). + inputs_bridge_notebook_imports: Imports the bridge code needs. + inputs_bridge_required_parameters: Widget name → DAB ref mapping + for the bridge notebook's base_parameters. """ items_expression: str inner_activities: list[Activity] = field(default_factory=list) concurrency: int | None = None + inputs_bridge_notebook_code: str | None = None + inputs_bridge_notebook_imports: list[str] = field(default_factory=list) + inputs_bridge_required_parameters: dict[str, str] = field(default_factory=dict) @dataclass(slots=True, kw_only=True) @@ -149,6 +208,16 @@ class IfConditionActivity(Activity): right: Right-hand operand expression. if_true_activities: Activities for the true branch. if_false_activities: Activities for the false branch. + bridge_notebook_code: C-07 (CF-iter2-001 / VAREX-003): when the + ADF condition expression contained a function call that + couldn't be lowered to a literal/dab_ref operand, + ``bridge_notebook_code`` carries the Python code that + evaluates it. The preparer synthesises a hidden SetVariable + task that runs this code and points ``left`` at the + resulting task value. + bridge_notebook_imports: Imports the bridge notebook code needs. + bridge_required_parameters: Widget name -> DAB ref mapping for + the bridge notebook's base_parameters. """ op: str @@ -156,6 +225,9 @@ class IfConditionActivity(Activity): right: str if_true_activities: list[Activity] = field(default_factory=list) if_false_activities: list[Activity] = field(default_factory=list) + bridge_notebook_code: str | None = None + bridge_notebook_imports: list[str] = field(default_factory=list) + bridge_required_parameters: dict[str, str] = field(default_factory=dict) @dataclass(slots=True, kw_only=True) @@ -165,16 +237,23 @@ class SetVariableActivity(Activity): Attributes: variable_name: Name of the variable being set. variable_value: Expression string that evaluates to the value. - value_kind: Kind of the resolved expression ("literal", "dab_ref", "notebook_code"). + value_kind: Kind of the resolved expression ("literal", "dab_ref", + "notebook_code", "unresolved"). notebook_code: Python code for notebook_code kind values. notebook_imports: Import statements needed for notebook_code. + raw_expression: C-33 (VAREX4-001 / CF4-003): when ``value_kind`` is + ``"unresolved"`` (the resolver returned None for an ADF + ``@``-prefixed value), this carries the original ADF + expression text so SETUP.md can surface the manual + initialisation step. """ variable_name: str variable_value: str - value_kind: str = "literal" # "literal", "dab_ref", "notebook_code" + value_kind: str = "literal" # "literal", "dab_ref", "notebook_code", "unresolved" notebook_code: str | None = None notebook_imports: list[str] = field(default_factory=list) + raw_expression: str | None = None @dataclass(slots=True, kw_only=True) @@ -269,12 +348,10 @@ class SparkJarActivity(Activity): Attributes: main_class_name: Fully qualified main class within the JAR. parameters: Arguments passed to the main class. - libraries: Library descriptors (JARs, wheels, etc.). """ main_class_name: str parameters: list[str] | None = None - libraries: list[dict[str, Any]] | None = None @dataclass(slots=True, kw_only=True) @@ -311,11 +388,21 @@ class SwitchActivity(Activity): on_expression: The ADF expression to evaluate. cases: Ordered list of case branches. default_activities: Activities to run when no case matches. + bridge_notebook_code: C-07 (CF-iter2-001 / CF-iter2-003): when + ``on_expression`` cannot be lowered to a literal/dab_ref, this + field carries the Python code the preparer runs in a bridge + task so the resolved value drives ``condition_task.left``. + bridge_notebook_imports: Imports for the bridge notebook code. + bridge_required_parameters: Widget name -> DAB ref mapping for + the bridge notebook's base_parameters. """ on_expression: str cases: list[SwitchCase] = field(default_factory=list) default_activities: list[Activity] = field(default_factory=list) + bridge_notebook_code: str | None = None + bridge_notebook_imports: list[str] = field(default_factory=list) + bridge_required_parameters: dict[str, str] = field(default_factory=dict) @dataclass(slots=True, kw_only=True) @@ -480,6 +567,10 @@ class TranslationContext: registry: MappingProxyType[str, Any] = field(default_factory=lambda: MappingProxyType({})) variable_cache: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) variable_value_cache: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) + variable_types: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) + variable_default_literals: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) + global_parameters: MappingProxyType[str, Any] = field(default_factory=lambda: MappingProxyType({})) + linked_service_parameters: MappingProxyType[str, Any] = field(default_factory=lambda: MappingProxyType({})) def with_activity(self, name: str, activity: Activity) -> TranslationContext: """Return a new context with *activity* added to the cache. @@ -496,6 +587,10 @@ def with_activity(self, name: str, activity: Activity) -> TranslationContext: registry=self.registry, variable_cache=self.variable_cache, variable_value_cache=self.variable_value_cache, + variable_types=self.variable_types, + variable_default_literals=self.variable_default_literals, + global_parameters=self.global_parameters, + linked_service_parameters=self.linked_service_parameters, ) def get_activity(self, activity_name: str) -> Activity | None: @@ -537,16 +632,102 @@ def with_variable( registry=self.registry, variable_cache=MappingProxyType({**self.variable_cache, variable_name: task_key}), variable_value_cache=new_variable_value_cache, + variable_types=self.variable_types, + variable_default_literals=self.variable_default_literals, + global_parameters=self.global_parameters, + linked_service_parameters=self.linked_service_parameters, + ) + + def with_variable_types( + self, + types: dict[str, str], + *, + default_literals: dict[str, str] | None = None, + ) -> TranslationContext: + """Return a new context seeded with declared variable types. + + Args: + types: Mapping of variable name -> ADF declared type + (``"String"``, ``"Boolean"``, ``"Array"``, ...). Used by + the IfCondition fallback to recognise Boolean variables + whose value is seeded only by a literal init task (and + therefore absent from ``variable_value_cache``). + default_literals: Optional mapping of variable name -> seeded + literal default (e.g. ``"true"``/``"false"``). The + IfCondition bridge (C-43) recomputes a Boolean operand + locally from this literal so an inner-ForEach condition does + not dangle to a parent-job task value. + + Returns: + New context carrying the variable type / default-literal maps. + """ + return TranslationContext( + activity_cache=self.activity_cache, + registry=self.registry, + variable_cache=self.variable_cache, + variable_value_cache=self.variable_value_cache, + variable_types=MappingProxyType({**self.variable_types, **types}), + variable_default_literals=MappingProxyType({**self.variable_default_literals, **(default_literals or {})}), + global_parameters=self.global_parameters, + linked_service_parameters=self.linked_service_parameters, ) def get_variable_task_key(self, variable_name: str) -> str | None: """Look up the task key that sets a variable.""" return self.variable_cache.get(variable_name) + def get_variable_type(self, variable_name: str) -> str | None: + """Look up a variable's declared ADF type, if known.""" + return self.variable_types.get(variable_name) + + def get_variable_default_literal(self, variable_name: str) -> str | None: + """Look up a variable's seeded literal default value, if known.""" + return self.variable_default_literals.get(variable_name) + def get_variable_dab_ref(self, variable_name: str) -> str | None: """Look up the inlined DAB ref value for a variable, if available.""" return self.variable_value_cache.get(variable_name) + def with_linked_service_parameters(self, params: dict[str, Any]) -> TranslationContext: + """Return a new context with linked-service-scoped parameters applied. + + Args: + params: Mapping of LS parameter name -> resolved value. Used + by ``@linkedService().X`` references in LS typeProperties. + + Returns: + New context with the parameters bound for the current activity. + """ + return TranslationContext( + activity_cache=self.activity_cache, + registry=self.registry, + variable_cache=self.variable_cache, + variable_value_cache=self.variable_value_cache, + variable_types=self.variable_types, + variable_default_literals=self.variable_default_literals, + global_parameters=self.global_parameters, + linked_service_parameters=MappingProxyType(dict(params)), + ) + + def get_global_parameter(self, name: str) -> Any: + """Look up a factory-level global parameter value. + + Args: + name: Global parameter name (e.g. ``"env_variable"``). + + Returns: + The parameter value if present, else ``None``. Values may be + scalar or dict-typed (e.g. ``{"type": "string", "value": "t"}``). + """ + raw = self.global_parameters.get(name) + if isinstance(raw, dict) and "value" in raw: + return raw["value"] + return raw + + def get_linked_service_parameter(self, name: str) -> Any: + """Look up an activity-supplied linked-service parameter value.""" + return self.linked_service_parameters.get(name) + TranslationResult: TypeAlias = Activity | UnsupportedActivity @@ -578,7 +759,15 @@ class TranslationReport: agentic_count: Activities requiring agentic translation. unsupported_count: Activities that could not be translated. gaps: List of agentic gaps identified during translation. - warnings: Human-readable warning messages emitted during translation. + warnings: Human-readable warning messages emitted during translation, + including unresolved ``@{...}`` ADF expressions surfaced by the + whole-IR rewriter. + detected_motifs: Multi-activity patterns the detector matched on the + source AST. When the caller did not supply a motif-consolidation + answer the translator collapses every entry into a single + :class:`MotifActivity`; otherwise the list still reports what + was detected so the adapter can prompt the user. The objects + here are :class:`~flowx.models.motifs.DetectedMotif` instances. """ pipeline: Pipeline @@ -587,3 +776,4 @@ class TranslationReport: unsupported_count: int = 0 gaps: list[AgenticGap] = field(default_factory=list) warnings: list[str] = field(default_factory=list) + detected_motifs: list[Any] = field(default_factory=list) diff --git a/src/orchestra/parser/adf_loader.py b/src/orchestra/parser/adf_loader.py index 03e6374..acc3ed9 100644 --- a/src/orchestra/parser/adf_loader.py +++ b/src/orchestra/parser/adf_loader.py @@ -93,6 +93,17 @@ def load_adf_definitions(source_dir: Path) -> AdfDefinitions: datasets: dict[str, AdfDataset] = {} linked_services: dict[str, AdfLinkedService] = {} triggers: list[AdfTrigger] = [] + global_parameters: dict[str, Any] = {} + + factory_dir = _find_json_dir(source_dir, "factory", "factories") + if factory_dir is not None: + for json_file in sorted(factory_dir.glob("*.json")): + try: + data = json.loads(json_file.read_text(encoding="utf-8")) + factory_params = _parse_factory_global_parameters(data) + global_parameters.update(factory_params) + except Exception: + logger.exception("Failed to parse factory file %s", json_file) pipeline_dir = _find_json_dir(source_dir, "pipelines", "pipeline") if pipeline_dir is not None: @@ -137,9 +148,36 @@ def load_adf_definitions(source_dir: Path) -> AdfDefinitions: datasets=datasets, linked_services=linked_services, triggers=triggers, + global_parameters=global_parameters, ) +def _parse_factory_global_parameters(data: dict[str, Any]) -> dict[str, Any]: + """Extracts ``globalParameters`` from a factory JSON payload. + + Args: + data: Raw JSON dictionary loaded from ``factory/.json``. + + Returns: + Mapping of global parameter name -> value. Each value is either + the scalar default (when ADF stores it bare) or the original + ``{"type": ..., "value": ...}`` dict. + """ + data = _normalize_arm(data) + props = data.get("properties", data) + raw = props.get("globalParameters") or {} + if not isinstance(raw, dict): + return {} + + result: dict[str, Any] = {} + for name, value in raw.items(): + if isinstance(value, dict) and "value" in value: + result[name] = value + else: + result[name] = value + return result + + def classify_activity(activity_type: str) -> tuple[TranslationStrategy, str | None]: """Classify an ADF activity type into a translation strategy. @@ -324,6 +362,7 @@ def parse_activity(data: dict[str, Any]) -> AdfActivity: linked_service_name = AdfLinkedServiceReference( reference_name=raw_ls.get("referenceName", ""), type=raw_ls.get("type", "LinkedServiceReference"), + parameters=raw_ls.get("parameters"), ) if_true_activities: list[AdfActivity] | None = None @@ -527,6 +566,7 @@ def _load_arm_template(template_path: Path) -> AdfDefinitions: datasets: dict[str, AdfDataset] = {} linked_services: dict[str, AdfLinkedService] = {} triggers: list[AdfTrigger] = [] + global_parameters: dict[str, Any] = {} for resource in resources: rtype = resource.get("type", "") @@ -561,12 +601,18 @@ def _load_arm_template(template_path: Path) -> AdfDefinitions: triggers.append(_parse_trigger_json(wrapped, fallback_name=name)) except Exception: logger.exception("Failed to parse ARM trigger resource %s", name) + elif rtype.endswith("/factories"): + try: + global_parameters.update(_parse_factory_global_parameters(wrapped)) + except Exception: + logger.exception("Failed to parse ARM factory resource %s", name) return AdfDefinitions( pipelines=pipelines, datasets=datasets, linked_services=linked_services, triggers=triggers, + global_parameters=global_parameters, ) diff --git a/src/orchestra/parser/expression_parser.py b/src/orchestra/parser/expression_parser.py index 1fcb69c..7ca884b 100644 --- a/src/orchestra/parser/expression_parser.py +++ b/src/orchestra/parser/expression_parser.py @@ -10,7 +10,10 @@ _ITEM_RE = re.compile(r"item\(\s*\)$", re.IGNORECASE) -_ITEM_FIELD_RE = re.compile(r"item\(\s*\)\.(\w+)", re.IGNORECASE) +# C-35 (CF4-004): anchor the end-of-string so multi-segment chains like +# ``item().condition.name`` don't match here and silently drop the trailing +# ``.name`` (the previous behaviour mapped to ``{{input.condition}}``). +_ITEM_FIELD_RE = re.compile(r"item\(\s*\)\.(\w+)\s*$", re.IGNORECASE) _ACTIVITY_OUTPUT_RE = re.compile( r"""activity\(\s*'([^']+)'\s*\)\.output(?:\.(.+))?""", @@ -22,11 +25,26 @@ re.IGNORECASE, ) +_PIPELINE_GLOBAL_PARAM_RE = re.compile( + r"""pipeline\(\s*\)\.globalParameters\.(\w+)""", + re.IGNORECASE, +) + _PIPELINE_PROPERTY_RE = re.compile( r"""pipeline\(\s*\)\.(\w+)""", re.IGNORECASE, ) +_LINKED_SERVICE_PARAM_RE = re.compile( + r"""linkedService\(\s*\)\.(\w+)""", + re.IGNORECASE, +) + +_ITEM_SAFE_NAV_RE = re.compile( + r"item\(\s*\)((?:\??\.\w+)+)$", + re.IGNORECASE, +) + _VARIABLE_RE = re.compile( r"""variables\(\s*'([^']+)'\s*\)""", re.IGNORECASE, @@ -67,10 +85,17 @@ _INTERPOLATION_RE = re.compile(r"@\{(.+?)\}") _FUNCTION_CALL_RE = re.compile( - r"([a-zA-Z_]\w*)\((.*)?\)$", + r"([a-zA-Z_]\w*)\((.*)?\)\s*$", re.IGNORECASE | re.DOTALL, ) +# Function names that are no-op wrappers when they appear at the outermost +# position around a single deterministic parameter / variable reference. +# Stripping these lets resolve_expression reach the underlying ref instead +# of falling through to notebook_code for trivial @json(pipeline().parameters.X) +# style wrappers commonly used in ADF for type coercion. +_NOOP_WRAPPER_NAMES: frozenset[str] = frozenset({"json", "string", "array"}) + _DATETIME_IMPORTS = ["from datetime import datetime, timezone, timedelta"] _TIME_UNIT_MAP: dict[str, str] = { @@ -109,7 +134,10 @@ def resolve_expression( return None if isinstance(value, bool): - return ExpressionResult(kind="literal", value=str(value)) + # VAREX3-002: render Python bool as lowercase 'true'/'false' so + # downstream ADF comparisons like @equals(variables('X'), true) + # match ADF's lowercase boolean tokens. + return ExpressionResult(kind="literal", value="true" if value else "false") if isinstance(value, (int, float)): return ExpressionResult(kind="literal", value=str(value)) @@ -119,7 +147,15 @@ def resolve_expression( if not value.startswith("@"): return ExpressionResult(kind="literal", value=value) - expr = value[1:] # strip leading @ + expr = value[1:].rstrip() # strip leading @ and trailing whitespace/newlines + + # Strip no-op wrappers like @json(pipeline().parameters.X) so the inner + # ref resolves to its DAB dynamic value. We only unwrap when the inner + # expression itself resolves cleanly (literal / dab_ref) so we don't + # eat the wrapper's semantics where it actually matters. + unwrapped = _unwrap_noop_call(expr, context, variable_task_keys=variable_task_keys) + if unwrapped is not None: + return unwrapped if _ITEM_RE.match(expr): return ExpressionResult(kind="dab_ref", value="{{input}}") @@ -129,6 +165,18 @@ def resolve_expression( field_name = match.group(1) return ExpressionResult(kind="dab_ref", value="{{input." + field_name + "}}") + result = _resolve_item_safe_nav(expr) + if result is not None: + return result + + result = _resolve_pipeline_global_param(expr, context) + if result is not None: + return result + + result = _resolve_linked_service_param(expr, context) + if result is not None: + return result + result = _resolve_pipeline_param(expr) if result is not None: return result @@ -157,6 +205,21 @@ def resolve_expression( if result is not None: return result + # CF3-004 / fix-attribute-access-on-function-results: handle + # ``....`` chains like + # ``json(pipeline().parameters.items).type`` by resolving the function + # call first then chaining `.get('attr')` onto the resulting code. + result = _resolve_function_call_with_attribute(expr, context, variable_task_keys=variable_task_keys) + if result is not None: + return result + + # C-33 (VAREX4-001): handle ``[N]`` chains so e.g. + # ``@split(pipeline().parameters.referenceDate,'/')[0]`` lowers to + # notebook_code. + result = _resolve_function_call_with_index(expr, context, variable_task_keys=variable_task_keys) + if result is not None: + return result + return None @@ -295,6 +358,114 @@ def _resolve_pipeline_param(expr: str) -> ExpressionResult | None: return ExpressionResult(kind="dab_ref", value="{{" + f"job.parameters.{param_name}" + "}}") +def _resolve_pipeline_global_param(expr: str, context: TranslationContext) -> ExpressionResult | None: + """Resolves ``pipeline().globalParameters.X`` against factory globals. + + When ``context.global_parameters`` carries a concrete value for *X* + the expression collapses to a literal so downstream callers (notably + ``concat`` reductions) get the actual factory value baked in. When + no factory value is available we fall back to a job-parameter DAB + ref so the bundle YAML can supply it. + """ + match = _PIPELINE_GLOBAL_PARAM_RE.match(expr) + if match is None: + return None + param_name = match.group(1) + value = context.get_global_parameter(param_name) + if value is None: + return ExpressionResult(kind="dab_ref", value="{{" + f"job.parameters.{param_name}" + "}}") + return ExpressionResult(kind="literal", value=str(value)) + + +def _resolve_linked_service_param(expr: str, context: TranslationContext) -> ExpressionResult | None: + """Resolves ``linkedService().X`` against activity-supplied LS parameters.""" + match = _LINKED_SERVICE_PARAM_RE.match(expr) + if match is None: + return None + param_name = match.group(1) + if param_name in context.linked_service_parameters: + value = context.get_linked_service_parameter(param_name) + if value is None: + return None + return ExpressionResult(kind="literal", value=str(value)) + return None + + +def _resolve_item_safe_nav(expr: str) -> ExpressionResult | None: + """Resolves ``item()?.X``, ``item().a?.b?.c`` and ``item().a.b`` chains. + + Any chain that uses the ADF safe-navigation operator ``?.`` — even a + single segment ``item()?.X`` — must lower to notebook_code so the bridge + lowering can fire (the trivial ``item().X`` case stays a DAB ref via + ``_ITEM_FIELD_RE``). The returned ``notebook_code`` walks the chain + using ``.get()`` so missing keys do not raise. + + C-16 (CF3-005 / VAREX3-005): the previous ``len(parts) < 2`` guard + blocked Switch on-expressions and SetVariable expressions wrapping + ``item()?.X`` from triggering bridge lowering. + """ + match = _ITEM_SAFE_NAV_RE.match(expr) + if match is None: + return None + chain = match.group(1) + # Collect (operator, field) tuples so single-segment item()?.X still + # lowers to notebook_code (was: skipped when len(parts) < 2). + segments: list[tuple[str, str]] = re.findall(r"(\??\.)(\w+)", chain) + if not segments: + return None + # If the chain has no safe-nav operator at all (purely ``item().a.b``) + # AND only one segment, defer to _ITEM_FIELD_RE's dab_ref path. + # C-35 (CF4-004): multi-segment pure-dotted chains like + # ``item().condition.name`` must lower to notebook_code so the + # downstream consumers can walk both segments instead of mapping to + # ``{{input.condition}}`` and silently dropping ``.name``. + has_safe_nav = any(op == "?." for op, _ in segments) + if not has_safe_nav and len(segments) < 2: + return None + expr_code = "__import__('json').loads(dbutils.widgets.get('item'))" + for _, part in segments: + expr_code = f"({expr_code} or {{}}).get('{part}')" + return ExpressionResult(kind="notebook_code", value=expr_code) + + +def _unwrap_noop_call( + expr: str, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> ExpressionResult | None: + """If *expr* is a no-op wrapper around a deterministic ref, return inner. + + Handles patterns like ``@json(pipeline().parameters.items)``, + ``@string(pipeline().parameters.X)``, and ``@array(...)`` where the + sole argument resolves to a clean literal/dab_ref. When the inner + cannot be deterministically resolved we return ``None`` so the + regular function dispatcher takes over. + """ + match = _FUNCTION_CALL_RE.match(expr) + if match is None: + return None + func_name = match.group(1) + if func_name.lower() not in _NOOP_WRAPPER_NAMES: + return None + inner = (match.group(2) or "").strip() + if not inner: + return None + args = _split_args(inner) + if len(args) != 1: + return None + sole_arg = args[0].strip() + if sole_arg.startswith("'") and sole_arg.endswith("'"): + return None # bare literal, let upstream string() handler decide + sub_expr = sole_arg if sole_arg.startswith("@") else "@" + sole_arg + inner_result = resolve_expression(sub_expr, context, variable_task_keys=variable_task_keys) + if inner_result is None: + return None + if inner_result.kind in ("literal", "dab_ref"): + return inner_result + return None + + def _resolve_pipeline_property(expr: str) -> ExpressionResult | None: """Resolves ``pipeline().PropertyName`` -> DAB ref.""" match = _PIPELINE_PROPERTY_RE.match(expr) @@ -336,38 +507,77 @@ def _resolve_variable( *, variable_task_keys: dict[str, str] | None = None, ) -> ExpressionResult | None: - """Resolves ``variables('name')`` -> task value DAB ref.""" + """Resolves ``variables('name')`` -> task value DAB ref. + + C-05 (VAREX-002): when neither the explicit mapping nor the context's + variable_cache knows a setter for *var_name*, return ``None`` instead + of falling back to ``{{tasks..values.}}`` — that + self-referential placeholder is never satisfied at runtime and + pollutes the bundle with hundreds of dangling refs. Init tasks + synthesised in :func:`engine._build_variable_init_activities` seed + the cache for default-valued variables, so this path now triggers + only for genuinely unset variables (caller logs / surfaces a setup + note). + """ match = _VARIABLE_RE.match(expr) if match is None: return None var_name = match.group(1) - # Always resolve to the task value reference. This preserves the - # explicit task dependency chain — downstream tasks must depend on the - # setter task. Even when the variable was set to a DAB built-in like - # {{job.start_time.iso_datetime}}, the task value is the canonical - # source since the setter notebook may transform the value. variable_task_keys_map = variable_task_keys or {} - setter_key = variable_task_keys_map.get(var_name) or context.get_variable_task_key(var_name) or var_name + setter_key = variable_task_keys_map.get(var_name) or context.get_variable_task_key(var_name) + if setter_key is None: + return None return ExpressionResult(kind="dab_ref", value="{{" + f"tasks.{setter_key}.values.{var_name}" + "}}") +_UTCNOW_APPROXIMATION_NOTE = ( + "Mapped ADF `utcnow()` to the Databricks job start time. " + "Single-task jobs see sub-second skew; multi-task jobs can see minutes of skew " + "between job start and the moment the activity actually runs." +) + +# ADF .NET-style format strings that map cleanly onto a Databricks dynamic +# value reference. Anything not in this table falls back to a notebook_code +# strftime call. +_UTCNOW_FORMAT_TO_DAB_REF: dict[str, str] = { + "yyyy-MM-dd": "{{job.start_time.iso_date}}", + "yyyy-MM-ddTHH:mm:ss": "{{job.start_time.iso_datetime}}", + "yyyy-MM-ddTHH:mm:ssZ": "{{job.start_time.iso_datetime}}", + "o": "{{job.start_time.iso_datetime}}", + "s": "{{job.start_time.iso_datetime}}", +} + + def _resolve_utcnow(expr: str) -> ExpressionResult | None: - """Resolves ``utcNow()`` or ``utcNow('format')`` -> notebook_code.""" + """Resolves ``utcNow()`` / ``utcNow('format')``. + + Bare ``utcnow()`` and ``utcnow('')`` map to a Databricks + dynamic value reference so the result can land directly inside DAB + YAML (``base_parameters`` and similar). Unrecognised format strings + keep the legacy ``notebook_code`` translation. + """ match = _UTCNOW_RE.match(expr) if match is None: return None format_string = match.group(1) - if format_string: - python_format = _convert_date_format(format_string) + if not format_string: return ExpressionResult( - kind="notebook_code", - value=f"datetime.now(timezone.utc).strftime('{python_format}')", - imports=["from datetime import datetime, timezone"], + kind="dab_ref", + value="{{job.start_time.iso_datetime}}", + notes=[_UTCNOW_APPROXIMATION_NOTE], ) + dab_ref = _UTCNOW_FORMAT_TO_DAB_REF.get(format_string) + if dab_ref: + return ExpressionResult( + kind="dab_ref", + value=dab_ref, + notes=[_UTCNOW_APPROXIMATION_NOTE], + ) + python_format = _convert_date_format(format_string) return ExpressionResult( kind="notebook_code", - value="datetime.now(timezone.utc).isoformat()", + value=f"datetime.now(timezone.utc).strftime('{python_format}')", imports=["from datetime import datetime, timezone"], ) @@ -405,6 +615,8 @@ def _resolve_concat( all_imports: list[str] = [] code_parts: list[str] = [] + literal_parts: list[str] = [] + all_literal = True all_required_parameters: dict[str, str] = {} for part in parts: @@ -412,25 +624,36 @@ def _resolve_concat( if not part: continue if part.startswith("'") and part.endswith("'"): - code_parts.append(repr(part[1:-1])) + value_text = part[1:-1] + code_parts.append(repr(value_text)) + literal_parts.append(value_text) else: sub_result = resolve_expression("@" + part, context, variable_task_keys=variable_task_keys) if sub_result is None: return None if sub_result.kind == "literal": code_parts.append(repr(sub_result.value)) + literal_parts.append(sub_result.value) elif sub_result.kind == "dab_ref": code_parts.append(_dab_ref_to_widget_code(sub_result.value)) widget_name, dab_ref = _required_parameter_for_ref(sub_result.value) all_required_parameters.setdefault(widget_name, dab_ref) + all_literal = False elif sub_result.kind == "notebook_code": code_parts.append(f"str({sub_result.value})") all_imports.extend(sub_result.imports) all_required_parameters.update(sub_result.required_parameters) + all_literal = False if not code_parts: return None + # If every part collapsed to a literal value, fold the whole concat into + # a single literal so downstream consumers (notebook library install, + # cluster fields, etc.) get a plain string instead of Python source. + if all_literal: + return ExpressionResult(kind="literal", value="".join(literal_parts)) + value = " + ".join(code_parts) return ExpressionResult( kind="notebook_code", @@ -503,6 +726,113 @@ def _split_args(inner: str) -> list[str]: return parts +_FUNCTION_CALL_WITH_ATTRIBUTE_RE = re.compile( + # Captures `funcName(args).attr.attr...` -- the trailing attribute chain + # must end with a word character so we don't accidentally swallow other + # closing parens / spaces. Used to lower + # ``json(pipeline().parameters.items).type`` to a notebook_code expression + # since the bare function dispatcher requires the function call to be the + # outermost token. + r"^([a-zA-Z_]\w*)\((.*)\)((?:\.\w+)+)\s*$", + re.IGNORECASE | re.DOTALL, +) + +_FUNCTION_CALL_WITH_INDEX_RE = re.compile( + # C-33 (VAREX4-001): ``funcName(args)[N]`` — captures a trailing + # integer subscript so ``split(...)[0]`` and similar ADF expressions + # lower to notebook_code (the bare dispatcher only matched when the + # function call was the outermost token). We support a single + # numeric subscript for now; nested chains (``...[0][1]``) fall + # through to the legacy unsupported path. + r"^([a-zA-Z_]\w*)\((.*)\)\[\s*(-?\d+)\s*\]\s*$", + re.IGNORECASE | re.DOTALL, +) + + +def _resolve_function_call_with_index( + expr: str, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> ExpressionResult | None: + """Lowers ``[N]`` to notebook_code. + + C-33 (VAREX4-001): ADF SetVariable expressions like + ``@split(pipeline().parameters.referenceDate,'/')[0]`` were + previously rejected because the bare function dispatcher matched + only when the call was the outermost token. Resolve the function + call as usual, then append ``[N]`` to the resulting Python code. + """ + match = _FUNCTION_CALL_WITH_INDEX_RE.match(expr) + if match is None: + return None + func_name = match.group(1) + inner = match.group(2) + index = match.group(3) + func_expr = f"@{func_name}({inner})" + base_result = resolve_expression(func_expr, context, variable_task_keys=variable_task_keys) + if base_result is None: + return None + if base_result.kind == "literal": + base_code = repr(base_result.value) + elif base_result.kind == "dab_ref": + base_code = _dab_ref_to_widget_code(base_result.value) + else: + base_code = base_result.value + code = f"({base_code})[{index}]" + return ExpressionResult( + kind="notebook_code", + value=code, + imports=list(base_result.imports), + required_parameters=dict(base_result.required_parameters), + ) + + +def _resolve_function_call_with_attribute( + expr: str, + context: TranslationContext, + *, + variable_task_keys: dict[str, str] | None = None, +) -> ExpressionResult | None: + """Lowers `....` chains to a notebook_code expression. + + CF3-004 / fix-attribute-access-on-function-results: the bare function + dispatcher only matches when the function call is the outermost token, + so an expression like ``json(pipeline().parameters.items).type`` falls + through with ``None`` and ships unmodified into ``condition_task.left``. + We resolve the function call, render it as Python code, then chain + ``.get('attr')`` for each segment of the trailing attribute path so the + bridge lowering picks it up. + """ + match = _FUNCTION_CALL_WITH_ATTRIBUTE_RE.match(expr) + if match is None: + return None + func_name = match.group(1) + inner = match.group(2) + attr_chain = match.group(3) + func_expr = f"@{func_name}({inner})" + base_result = resolve_expression(func_expr, context, variable_task_keys=variable_task_keys) + if base_result is None: + return None + if base_result.kind == "literal": + # The result is a known literal -- render it as a Python expression + # then chain `.get(...)` so the resulting notebook_code is valid. + base_code = repr(base_result.value) + elif base_result.kind == "dab_ref": + base_code = _dab_ref_to_widget_code(base_result.value) + else: + base_code = base_result.value + code = base_code + for segment in attr_chain.strip(".").split("."): + code = f"({code}).get('{segment}')" + return ExpressionResult( + kind="notebook_code", + value=code, + imports=list(base_result.imports), + required_parameters=dict(base_result.required_parameters), + ) + + def _resolve_function_call( expr: str, context: TranslationContext, @@ -535,12 +865,26 @@ def _resolve_function_call( continue if (raw_arg.startswith("'") and raw_arg.endswith("'")) or (raw_arg.startswith('"') and raw_arg.endswith('"')): - resolved_args.append(ExpressionResult(kind="literal", value=raw_arg[1:-1])) + # C-34 (VAREX4-002): preserve the quotedness so the codegen + # downstream emits ``repr(value)`` rather than a bare token — + # otherwise quoted ``'09'`` / ``'12'`` collapse to a bare + # numeric and either raise a SyntaxError (leading zero) or + # silently compare against the wrong value. + resolved_args.append(ExpressionResult(kind="literal", value=raw_arg[1:-1], was_string_literal=True)) elif _is_numeric(raw_arg): resolved_args.append(ExpressionResult(kind="literal", value=raw_arg)) elif raw_arg.lower() in ("true", "false"): + # C-34 (VAREX4-003): ADF Booleans (``true`` / ``false``) match + # lowercase strings on the SetVariable consumer side (C-21). + # Mark the literal so ``_arg_to_code`` emits ``'true'`` / + # ``'false'`` strings rather than the bare Python ``True`` / + # ``False`` (whose ``str()`` is title-case and never matches). resolved_args.append( - ExpressionResult(kind="literal", value="True" if raw_arg.lower() == "true" else "False") + ExpressionResult( + kind="literal", + value="true" if raw_arg.lower() == "true" else "false", + was_bool_literal=True, + ) ) elif raw_arg.lower() == "null": resolved_args.append(ExpressionResult(kind="literal", value="None")) @@ -579,8 +923,18 @@ def _is_numeric(text: str) -> bool: def _arg_to_code(arg: ExpressionResult) -> str: - """Converts a resolved argument to a Python code snippet.""" + """Converts a resolved argument to a Python code snippet. + + C-34 (VAREX4-002/003): quoted-string and Boolean-literal arguments + must emit ``repr()`` of the value (e.g. ``'09'`` rather than the + bare token ``09``) so the resulting code (a) parses (leading-zero + integers are SyntaxErrors in modern Python) and (b) compares against + the right concrete value (Booleans on the SetVariable consumer side + serialise as lowercase strings, not Python bools). + """ if arg.kind == "literal": + if arg.was_string_literal or arg.was_bool_literal: + return repr(arg.value) if arg.value in ("True", "False", "None") or _is_numeric(arg.value): return arg.value return repr(arg.value) @@ -628,6 +982,11 @@ def _collect_required_parameters(*args: ExpressionResult) -> dict[str, str]: return merged +def _collect_notes(*args: ExpressionResult) -> list[str]: + """Collects caveat notes across resolved arguments.""" + return [note for arg in args for note in arg.notes] + + def _result_from_args( value: str, args: list[ExpressionResult], @@ -649,13 +1008,22 @@ def _result_from_args( value=value, imports=imports, required_parameters=_collect_required_parameters(*args), + notes=_collect_notes(*args), ) def _handle_concat(args: list[ExpressionResult]) -> ExpressionResult | None: - """concat(a, b, ...) -> str(a) + str(b) + ...""" + """concat(a, b, ...) -> str(a) + str(b) + ... + + When every argument resolved to a ``literal`` kind, collapse the whole + expression to a single literal so downstream consumers (cluster fields, + library paths, notebook-install jar refs, etc.) get a plain string + instead of Python source. + """ if not args: return None + if all(a.kind == "literal" for a in args): + return ExpressionResult(kind="literal", value="".join(a.value for a in args)) parts = [f"str({_arg_to_code(a)})" for a in args] return _result_from_args(" + ".join(parts), args) @@ -725,7 +1093,17 @@ def _handle_starts_with(args: list[ExpressionResult]) -> ExpressionResult | None def _handle_substring(args: list[ExpressionResult]) -> ExpressionResult | None: - """substring(text, start, length) -> str(text)[int(start):int(start)+int(length)]""" + """substring(text, start[, length]) -> Python slice. + + C-33 (VAREX4-001): ADF accepts the 2-arg form ``substring(x, start)`` + in addition to the documented 3-arg form. Treat the 2-arg case as + ``str(text)[int(start):]`` so SetVariable activities that wrap it can + actually resolve. + """ + if len(args) == 2: + text = _arg_to_code(args[0]) + start = _arg_to_code(args[1]) + return _result_from_args(f"str({text})[int({start}):]", args) if len(args) != 3: return None text = _arg_to_code(args[0]) @@ -1140,6 +1518,18 @@ def _handle_format_date_time(args: list[ExpressionResult]) -> ExpressionResult | """formatDateTime(ts, fmt?) -> datetime.fromisoformat(ts).strftime(converted_fmt)""" if len(args) < 1 or len(args) > 2: return None + # formatDateTime(utcnow(), '') -> remap straight to the + # matching Databricks dynamic value so the result lands in YAML. + if ( + len(args) == 2 + and args[0].kind == "dab_ref" + and args[0].value == "{{job.start_time.iso_datetime}}" + and _UTCNOW_APPROXIMATION_NOTE in args[0].notes + and args[1].kind == "literal" + ): + dab_ref = _UTCNOW_FORMAT_TO_DAB_REF.get(args[1].value) + if dab_ref: + return ExpressionResult(kind="dab_ref", value=dab_ref, notes=[_UTCNOW_APPROXIMATION_NOTE]) timestamp_dt = _datetime_arg_code(args[0]) if len(args) == 2 and args[1].kind == "literal": python_format = _convert_date_format(args[1].value) diff --git a/src/orchestra/parser/ir_rewriter.py b/src/orchestra/parser/ir_rewriter.py new file mode 100644 index 0000000..b3cc0a5 --- /dev/null +++ b/src/orchestra/parser/ir_rewriter.py @@ -0,0 +1,252 @@ +"""Whole-IR expression rewriter. + +Activity translators each call :func:`resolve_interpolated_string` on the +specific string fields they know about (``source_query``, ``url``, +``base_parameters`` values, ...). Anything outside those known fields +-- raw SQL ``WHERE`` clauses inside ``source_properties``, REST request +bodies, dataset folder paths, ``base_parameters`` strings that pass +through a value untouched -- can carry through to the bundle as a +literal ``@{...}`` ADF expression, silently corrupting query semantics +at runtime. + +This module runs a final pass over the translated IR, walks every +string-typed field on every :class:`~flowx.models.ir.Activity` +(including strings nested inside ``dict`` / ``list`` fields and inside +control-flow inner activities), and re-applies +:func:`resolve_interpolated_string` to each. Every ``@{...}`` token +that still remains after the pass is appended to *warnings* so the +caller surfaces the gap in the translation report. + +Fields that intentionally hold raw ADF input (``linked_service_definition``, +``raw_definition``) or that preserve pre-rewrite history +(``original_activities`` on :class:`~flowx.models.ir.MotifActivity`) +are skipped. Identifier fields (``name``, ``task_key``, +``variable_name``) are skipped so that rewriting cannot break +cross-task references. +""" + +from __future__ import annotations + +import dataclasses +import re +from types import MappingProxyType +from typing import Any + +from flowx.models.ir import ( + Activity, + AppendVariableActivity, + Pipeline, + SetVariableActivity, + SwitchActivity, + SwitchCase, + TranslationContext, +) +from flowx.parser.expression_parser import resolve_interpolated_string + +# Fields that must never be rewritten — either they hold raw ADF input +# that downstream consumers parse separately, or they are identifiers +# whose value is used as a reference key elsewhere in the IR. +_FIELDS_TO_SKIP: frozenset[str] = frozenset( + { + "name", + "task_key", + "linked_service_definition", + "raw_definition", + "original_activities", + "variable_name", + "matched_activity_names", + } +) + +_UNRESOLVED_RE = re.compile(r"@\{[^}]+\}") + + +def rewrite_pipeline_expressions( + pipeline: Pipeline, + *, + warnings: list[str] | None = None, +) -> Pipeline: + """Walks every string field in *pipeline* and rewrites ``@{...}`` tokens. + + Args: + pipeline: Translated pipeline IR. Not mutated. + warnings: Optional list to which the rewriter appends a message + for every string field that still contains an unresolved + ``@{...}`` token after the pass. When ``None`` the rewriter + still runs but cannot surface gaps. + + Returns: + A new :class:`Pipeline` whose activities have had their string + fields rewritten through + :func:`~flowx.parser.expression_parser.resolve_interpolated_string`. + + Notes: + - The rewriter builds a single :class:`TranslationContext` from + the pipeline's :class:`SetVariableActivity` and + :class:`AppendVariableActivity` nodes so that + ``@{variables('x')}`` tokens resolve against the same setter + task keys the per-activity translators used. + - Fields listed in ``_FIELDS_TO_SKIP`` -- identifiers and raw + ADF input -- are returned unchanged. + - Unknown field types (ints, bools, None, custom dataclasses + beyond Activity/SwitchCase) pass through unchanged. + """ + context = _build_context_from_pipeline(pipeline) + sink: list[str] = warnings if warnings is not None else [] + rewritten_tasks = [_rewrite_activity(activity, context, sink) for activity in pipeline.tasks] + return dataclasses.replace(pipeline, tasks=rewritten_tasks) + + +def _build_context_from_pipeline(pipeline: Pipeline) -> TranslationContext: + """Builds a TranslationContext whose variable_cache is keyed by every + SetVariable / AppendVariable activity in the pipeline. + + Args: + pipeline: Translated pipeline IR. + + Returns: + A :class:`TranslationContext` with ``variable_cache`` populated. + Activities that nest inside control-flow types are walked too so + a variable set inside a ForEach is still discoverable. + """ + variable_cache: dict[str, str] = {} + + def visit(activities: list[Activity]) -> None: + for activity in activities: + if isinstance(activity, (SetVariableActivity, AppendVariableActivity)): + variable_cache.setdefault(activity.variable_name, activity.task_key) + # Recurse into control-flow branches. + nested = _nested_activities(activity) + if nested: + visit(nested) + + visit(list(pipeline.tasks)) + return TranslationContext( + activity_cache=MappingProxyType({}), + registry=MappingProxyType({}), + variable_cache=MappingProxyType(variable_cache), + ) + + +def _nested_activities(activity: Activity) -> list[Activity]: + """Returns activities nested inside a control-flow activity, or []. + + Args: + activity: Any IR activity. + + Returns: + The control-flow branches' activity lists concatenated, or an + empty list when *activity* is a leaf type. + """ + nested: list[Activity] = [] + if hasattr(activity, "inner_activities"): + nested.extend(getattr(activity, "inner_activities") or []) + if hasattr(activity, "if_true_activities"): + nested.extend(getattr(activity, "if_true_activities") or []) + if hasattr(activity, "if_false_activities"): + nested.extend(getattr(activity, "if_false_activities") or []) + if isinstance(activity, SwitchActivity): + for case in activity.cases: + nested.extend(case.activities) + nested.extend(activity.default_activities) + return nested + + +def _rewrite_activity(activity: Activity, context: TranslationContext, warnings: list[str]) -> Activity: + """Returns a new activity with every safe string field rewritten. + + Args: + activity: Activity to rewrite. Not mutated. + context: Translation context whose ``variable_cache`` resolves + ``@variables('x')`` tokens. + warnings: List to append unresolved-expression warnings to. + + Returns: + A new activity instance with rewritten string fields. Control- + flow activities have their inner branches recursed into. + """ + field_overrides: dict[str, Any] = {} + for f in dataclasses.fields(activity): + if f.name in _FIELDS_TO_SKIP: + continue + original = getattr(activity, f.name) + rewritten = _rewrite_value( + original, + context, + warnings, + field_path=f"{type(activity).__name__}.{activity.task_key}.{f.name}", + ) + if rewritten is not original: + field_overrides[f.name] = rewritten + + if not field_overrides: + return activity + return dataclasses.replace(activity, **field_overrides) + + +def _rewrite_value(value: Any, context: TranslationContext, warnings: list[str], *, field_path: str) -> Any: + """Recursively rewrites every string contained in *value*. + + Args: + value: Any IR value -- str, list, dict, Activity, SwitchCase, or + a primitive. Activities and SwitchCases recurse; primitives + pass through. + context: Translation context. + warnings: List to append unresolved-expression warnings to. + field_path: Dotted path describing where this value sits in the + IR (used in warning messages so the user can find the gap). + + Returns: + The rewritten value, or *value* unchanged when no rewrite + applied. + """ + if isinstance(value, str): + return _rewrite_string(value, context, warnings, field_path=field_path) + if isinstance(value, Activity): + return _rewrite_activity(value, context, warnings) + if isinstance(value, SwitchCase): + new_value = _rewrite_value(value.value, context, warnings, field_path=f"{field_path}.value") + new_activities = [_rewrite_activity(a, context, warnings) for a in value.activities] + if new_value is value.value and all(n is o for n, o in zip(new_activities, value.activities)): + return value + return SwitchCase(value=new_value, activities=new_activities) + if isinstance(value, list): + new_list = [ + _rewrite_value(item, context, warnings, field_path=f"{field_path}[{i}]") for i, item in enumerate(value) + ] + if all(n is o for n, o in zip(new_list, value)): + return value + return new_list + if isinstance(value, dict): + new_dict = { + k: _rewrite_value(v, context, warnings, field_path=f"{field_path}[{k!r}]") for k, v in value.items() + } + if all(new_dict[k] is value[k] for k in value): + return value + return new_dict + return value + + +def _rewrite_string(value: str, context: TranslationContext, warnings: list[str], *, field_path: str) -> str: + """Applies ``resolve_interpolated_string`` and surfaces leftover ``@{...}``. + + Args: + value: String value to rewrite. + context: Translation context. + warnings: List to append unresolved-expression warnings to. + field_path: Dotted path for the warning message. + + Returns: + The rewritten string. When the rewrite cannot resolve every + ``@{...}`` token, the leftover tokens remain in the returned + string (for forensics) and a warning is appended. + """ + if "@{" not in value: + return value + rewritten = resolve_interpolated_string(value, context) + leftovers = _UNRESOLVED_RE.findall(rewritten) + if leftovers: + warnings.append( + f"Unresolved ADF expression at {field_path}: {sorted(set(leftovers))!r} (left in output verbatim)" + ) + return rewritten diff --git a/src/orchestra/preparer/activity_preparers/for_each.py b/src/orchestra/preparer/activity_preparers/for_each.py index a59c117..7be09d1 100644 --- a/src/orchestra/preparer/activity_preparers/for_each.py +++ b/src/orchestra/preparer/activity_preparers/for_each.py @@ -19,6 +19,7 @@ from flowx.preparer.workflow_preparer import ( PreparedActivity, PreparedWorkflow, + _iter_activity_with_descendants, build_common_task_fields, prepare_activity, ) @@ -28,31 +29,116 @@ from flowx.models.ir import ForEachActivity -def _resolve_for_each_inputs(items_expression: str) -> str: - """Converts an ADF items expression to a DAB dynamic value reference. - - Args: - items_expression: The raw ADF expression for ForEach items. - - Returns: - A DAB dynamic value reference string, or the original expression if - it cannot be resolved. +def _resolve_for_each_inputs_with_bridge( + activity: ForEachActivity, +) -> tuple[str, dict[str, Any] | None, list[DabNotebook]]: + """Resolves the ForEach items expression and emits a bridge task when needed. + + C-08 (CF-iter2-002): per Databricks docs, ``for_each_task.inputs`` + accepts a literal JSON array, ``{{tasks.X.values.Y}}``, or + ``{{job.parameters.X}}``. Function calls like ``@split(, ',')`` + are rejected. When the expression resolves to ``notebook_code`` we + synthesise a hidden seed task that computes the array and publishes + it as a task value the ForEach inputs reference. + + C-31 (CF4-001): the translator now stashes the resolved bridge code + on the IR (``inputs_bridge_notebook_code`` and friends) while the + full TranslationContext is available. The preparer reads those + fields rather than re-resolving against an empty TranslationContext + — the latter silently failed for any expression that needed + variable_cache lookups (e.g. ``@split(variables('fecha'),',')``). """ + items_expression = activity.items_expression + task_key = activity.task_key + + # IR-supplied bridge wins (C-31). Falls through to the legacy + # re-resolution path only when no bridge code was captured. + if activity.inputs_bridge_notebook_code: + bridge_key = f"{task_key}_inputs_bridge" + value_key = "items" + notebook_relative_path = f"notebooks/{bridge_key}.py" + base_parameters: dict[str, str] = dict(activity.inputs_bridge_required_parameters) + notebook_source = _render_for_each_inputs_bridge( + activity.inputs_bridge_notebook_code, + list(activity.inputs_bridge_notebook_imports), + list(base_parameters.keys()), + value_key, + ) + bridge_task: dict[str, Any] = { + "task_key": bridge_key, + "notebook_task": { + "notebook_path": f"../src/{notebook_relative_path}", + "base_parameters": base_parameters, + }, + } + bridge_value_ref = f"{{{{tasks.{bridge_key}.values.{value_key}}}}}" + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=notebook_source)] + return bridge_value_ref, bridge_task, notebooks + if items_expression.startswith("{{"): - return items_expression + return items_expression, None, [] context = TranslationContext() result = resolve_expression(items_expression, context) + if result is None and not items_expression.startswith("@"): + result = resolve_expression("@" + items_expression, context) + if result is not None and result.kind in ("dab_ref", "literal"): - return result.value + return result.value, None, [] + + if result is not None and result.kind == "notebook_code": + bridge_key = f"{task_key}_inputs_bridge" + value_key = "items" + notebook_relative_path = f"notebooks/{bridge_key}.py" + base_parameters = dict(result.required_parameters) + notebook_source = _render_for_each_inputs_bridge( + result.value, + result.imports, + list(base_parameters.keys()), + value_key, + ) + bridge_task = { + "task_key": bridge_key, + "notebook_task": { + "notebook_path": f"../src/{notebook_relative_path}", + "base_parameters": base_parameters, + }, + } + bridge_value_ref = f"{{{{tasks.{bridge_key}.values.{value_key}}}}}" + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=notebook_source)] + return bridge_value_ref, bridge_task, notebooks + + return items_expression, None, [] - # Also try with @ prefix if not present - if not items_expression.startswith("@"): - result = resolve_expression("@" + items_expression, context) - if result is not None and result.kind in ("dab_ref", "literal"): - return result.value - return items_expression +def _render_for_each_inputs_bridge( + notebook_code: str, + imports: list[str], + widget_names: list[str], + value_key: str, +) -> str: + """Generates the Python source for a ForEach-inputs bridge notebook. + + The notebook computes the array value and publishes it via + ``dbutils.jobs.taskValues.set`` so the parent ForEach task can + reference it via ``{{tasks..values.items}}``. + """ + lines: list[str] = [] + seen_imports: set[str] = set() + for imp in imports: + if imp in seen_imports: + continue + seen_imports.add(imp) + lines.append(imp) + if seen_imports: + lines.append("") + for widget in widget_names: + lines.append(f"dbutils.widgets.text('{widget}', '')") + if widget_names: + lines.append("") + lines.append(f"_bridge_value = {notebook_code}") + lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") + return "\n".join(lines) + "\n" def _inject_input_parameter(inner_task: dict) -> dict: @@ -73,12 +159,22 @@ def _inject_input_parameter(inner_task: dict) -> dict: return inner_task -def prepare(activity: ForEachActivity, *, scope: str = "") -> PreparedActivity: +def prepare( + activity: ForEachActivity, + *, + scope: str = "", + variable_task_keys: dict[str, str] | None = None, +) -> PreparedActivity: """Converts a ForEachActivity into a DAB for_each_task definition. Args: activity: The translated for-each activity from the IR. scope: Secret scope name (typically the pipeline/job name). + variable_task_keys: C-06 (VAREX-004): parent-job variable->setter + mapping threaded into ``collect_inner_job_params`` so + ``@variables('X')`` references in the inner-job body route + through the variable's task-value rather than fabricating an + undeclared inner-job parameter. Returns: A PreparedActivity with the for_each_task, plus any notebooks, secrets, @@ -86,35 +182,91 @@ def prepare(activity: ForEachActivity, *, scope: str = "") -> PreparedActivity: """ task = build_common_task_fields(activity) concurrency = activity.concurrency if activity.concurrency is not None else 20 - inputs = _resolve_for_each_inputs(activity.items_expression) + inputs, inputs_bridge_task, inputs_bridge_notebooks = _resolve_for_each_inputs_with_bridge(activity) + if inputs_bridge_task is not None: + existing_deps = list(task.get("depends_on") or []) + task["depends_on"] = [*existing_deps, {"task_key": inputs_bridge_task["task_key"]}] inner_activities = activity.inner_activities - all_notebooks: list[DabNotebook] = [] + all_notebooks: list[DabNotebook] = list(inputs_bridge_notebooks) all_secrets: list[SecretInstruction] = [] all_setup_tasks: list[SetupTask] = [] inner_workflows: list[PreparedWorkflow] = [] + extra_tasks: list[dict[str, Any]] = [] + if inputs_bridge_task is not None: + extra_tasks.append(inputs_bridge_task) if len(inner_activities) == 1: inner_prepared = prepare_activity(inner_activities[0], scope=scope) - inner_task = _inject_input_parameter(inner_prepared.task) all_notebooks.extend(inner_prepared.notebooks) all_secrets.extend(inner_prepared.secrets) all_setup_tasks.extend(inner_prepared.setup_tasks) inner_workflows.extend(inner_prepared.inner_workflows) - task["for_each_task"] = { - "inputs": inputs, - "task": inner_task, - "concurrency": concurrency, - } + # If the single child contributed extra_tasks (e.g. IfCondition or + # Switch branch bodies) we cannot inline as for_each_task.task — + # for_each only accepts a single task. Escalate to the sub-job + # path so the entire branch body survives (CF-001). + if inner_prepared.extra_tasks: + inner_job_name = f"{activity.task_key}_inner_tasks" + inner_tasks: list[dict[str, Any]] = [ + inner_prepared.task, + *inner_prepared.extra_tasks, + ] + normalize_inner_task_params(inner_tasks) + parameters, job_parameters = collect_inner_job_params(inner_tasks, variable_task_keys=variable_task_keys) + + # LSC3-001: gather cluster hints from inner activities so the + # inner-job default cluster lifts spark_env_vars / custom_tags / + # driver_node_type_id etc. from the LS-derived cluster spec. + inner_cluster_hints: list[dict[str, Any]] = [] + for nested_activity in _iter_activity_with_descendants(inner_activities[0]): + if nested_activity.cluster: + inner_cluster_hints.append(dict(nested_activity.cluster)) + + inner_workflow = PreparedWorkflow( + name=inner_job_name, + tasks=inner_tasks, + notebooks=[], + secrets=[], + setup_tasks=[], + parameters=parameters, + cluster_hints=inner_cluster_hints, + ) + inner_workflows.append(inner_workflow) + + inner_job_key = normalize_task_key(inner_job_name) + body_task: dict[str, Any] = { + "task_key": f"{activity.task_key}_iteration", + "run_job_task": { + "job_id": f"${{resources.jobs.{inner_job_key}.id}}", + "job_parameters": job_parameters, + }, + } + + task["for_each_task"] = { + "inputs": inputs, + "task": body_task, + "concurrency": concurrency, + } + else: + inner_task = _inject_input_parameter(inner_prepared.task) + task["for_each_task"] = { + "inputs": inputs, + "task": inner_task, + "concurrency": concurrency, + } elif len(inner_activities) > 1: inner_job_name = f"{activity.task_key}_inner_tasks" - inner_tasks: list[dict[str, Any]] = [] + inner_tasks = [] for child in inner_activities: child_prepared = prepare_activity(child, scope=scope) inner_tasks.append(child_prepared.task) + # Carry IfCondition / Switch branch bodies through so the + # nested control flow survives the ForEach wrap (CF-001). + inner_tasks.extend(child_prepared.extra_tasks) all_notebooks.extend(child_prepared.notebooks) all_secrets.extend(child_prepared.secrets) all_setup_tasks.extend(child_prepared.setup_tasks) @@ -122,7 +274,16 @@ def prepare(activity: ForEachActivity, *, scope: str = "") -> PreparedActivity: normalize_inner_task_params(inner_tasks) - parameters, job_parameters = collect_inner_job_params(inner_tasks) + parameters, job_parameters = collect_inner_job_params(inner_tasks, variable_task_keys=variable_task_keys) + + # LSC3-001: gather cluster hints from every nested inner activity + # so the inner-job default cluster picks up LS-derived + # spark_env_vars / custom_tags / driver_node_type_id. + inner_cluster_hints = [] + for child in inner_activities: + for nested_activity in _iter_activity_with_descendants(child): + if nested_activity.cluster: + inner_cluster_hints.append(dict(nested_activity.cluster)) inner_workflow = PreparedWorkflow( name=inner_job_name, @@ -131,11 +292,12 @@ def prepare(activity: ForEachActivity, *, scope: str = "") -> PreparedActivity: secrets=[], setup_tasks=[], parameters=parameters, + cluster_hints=inner_cluster_hints, ) inner_workflows.append(inner_workflow) inner_job_key = normalize_task_key(inner_job_name) - body_task: dict[str, Any] = { + body_task = { "task_key": f"{activity.task_key}_iteration", "run_job_task": { "job_id": f"${{resources.jobs.{inner_job_key}.id}}", @@ -158,6 +320,7 @@ def prepare(activity: ForEachActivity, *, scope: str = "") -> PreparedActivity: return PreparedActivity( task=task, + extra_tasks=extra_tasks, notebooks=all_notebooks, secrets=all_secrets, setup_tasks=all_setup_tasks, diff --git a/src/orchestra/preparer/activity_preparers/if_condition.py b/src/orchestra/preparer/activity_preparers/if_condition.py index 85473d3..f69b8b3 100644 --- a/src/orchestra/preparer/activity_preparers/if_condition.py +++ b/src/orchestra/preparer/activity_preparers/if_condition.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any +from flowx.models.dab import DabNotebook from flowx.preparer.workflow_preparer import ( PreparedActivity, PreparedArtifacts, @@ -20,6 +21,9 @@ if TYPE_CHECKING: from flowx.models.ir import IfConditionActivity +# Placeholder emitted by the translator when an operand requires a bridge. +_BRIDGE_PLACEHOLDER_PREFIX = "__BRIDGE__::" + def inject_outcome_dependency(tasks: list[dict[str, Any]], condition_key: str, outcome: str) -> None: """Gates branch-root tasks on the condition's outcome. @@ -59,12 +63,24 @@ def prepare(activity: IfConditionActivity, *, scope: str = "") -> PreparedActivi and aggregated artifacts from both branches. """ task = build_common_task_fields(activity) + + bridge_task, bridge_value_ref, bridge_notebooks = _build_bridge_task(activity) + left = _rewrite_bridge_placeholder(activity.left, bridge_value_ref) + right = _rewrite_bridge_placeholder(activity.right, bridge_value_ref) + task["condition_task"] = { "op": activity.op, - "left": activity.left, - "right": activity.right, + "left": left, + "right": right, } + # Bridge task runs ahead of the condition task -- the condition must + # depend on the bridge succeeding. + if bridge_task is not None: + bridge_dep = {"task_key": bridge_task["task_key"]} + existing_deps = list(task.get("depends_on") or []) + task["depends_on"] = [*existing_deps, bridge_dep] + artifacts = PreparedArtifacts() if_true_tasks: list[dict[str, Any]] = [] @@ -83,11 +99,96 @@ def prepare(activity: IfConditionActivity, *, scope: str = "") -> PreparedActivi artifacts = merge_prepared_artifacts(artifacts, prepared) inject_outcome_dependency(if_false_tasks, activity.task_key, "false") + extras: list[dict[str, Any]] = [] + if bridge_task is not None: + extras.append(bridge_task) + extras.extend(if_true_tasks + if_false_tasks) + + notebooks = list(artifacts.notebooks) + notebooks.extend(bridge_notebooks) + return PreparedActivity( task=task, - extra_tasks=if_true_tasks + if_false_tasks, - notebooks=list(artifacts.notebooks), + extra_tasks=extras, + notebooks=notebooks, secrets=list(artifacts.secrets), setup_tasks=list(artifacts.setup_tasks), inner_workflows=list(artifacts.inner_workflows), ) + + +def _build_bridge_task( + activity: IfConditionActivity, +) -> tuple[dict[str, Any] | None, str | None, list[DabNotebook]]: + """Synthesises a hidden SetVariable-like task that evaluates a bridged + notebook_code expression for an IfCondition operand. + + C-07 (CF-iter2-001 / CF-iter2-003 / VAREX-003): when the translator + surfaces ``bridge_notebook_code``, the preparer wires it into the job + graph as a Python notebook task that writes a single task value the + condition operand can reference. + """ + if not activity.bridge_notebook_code: + return None, None, [] + + bridge_key = f"{activity.task_key}_bridge" + value_key = "result" + notebook_relative_path = f"notebooks/{bridge_key}.py" + + # Build the bridge notebook source. base_parameters can include widget + # bindings the bridge expression depends on. + base_parameters: dict[str, str] = dict(activity.bridge_required_parameters) + notebook_source = _render_bridge_notebook( + activity.bridge_notebook_code, + activity.bridge_notebook_imports, + list(base_parameters.keys()), + value_key, + ) + + bridge_task: dict[str, Any] = { + "task_key": bridge_key, + "notebook_task": { + "notebook_path": f"../src/{notebook_relative_path}", + "base_parameters": base_parameters, + }, + } + bridge_value_ref = f"{{{{tasks.{bridge_key}.values.{value_key}}}}}" + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=notebook_source)] + return bridge_task, bridge_value_ref, notebooks + + +def _rewrite_bridge_placeholder(operand: str, bridge_value_ref: str | None) -> str: + """Rewrites a translator-side bridge placeholder to the real task value.""" + if not isinstance(operand, str): + return operand + if not operand.startswith(_BRIDGE_PLACEHOLDER_PREFIX): + return operand + if bridge_value_ref is None: + # Defensive: translator surfaced a placeholder but no bridge code. + return operand + return bridge_value_ref + + +def _render_bridge_notebook( + notebook_code: str, + imports: list[str], + widget_names: list[str], + value_key: str, +) -> str: + """Generates the Python source for a condition bridge notebook.""" + lines: list[str] = [] + seen_imports: set[str] = set() + for imp in imports: + if imp in seen_imports: + continue + seen_imports.add(imp) + lines.append(imp) + if seen_imports: + lines.append("") + for widget in widget_names: + lines.append(f"dbutils.widgets.text('{widget}', '')") + if widget_names: + lines.append("") + lines.append(f"_bridge_value = {notebook_code}") + lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") + return "\n".join(lines) + "\n" diff --git a/src/orchestra/preparer/activity_preparers/notebook.py b/src/orchestra/preparer/activity_preparers/notebook.py index a756cec..429a2c9 100644 --- a/src/orchestra/preparer/activity_preparers/notebook.py +++ b/src/orchestra/preparer/activity_preparers/notebook.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING -from flowx.models.dab import DabNotebook +from flowx.models.dab import DabNotebook, ParameterApproximation, SetupTask from flowx.models.ir import TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string from flowx.preparer.activity_preparers.naming import notebook_filename, workspace_notebook_filename @@ -94,6 +94,51 @@ def _resolve_notebook_path(path: str) -> str: return path +_DISPATCH_STUB_WIDGET = "notebook_path" + + +def _dispatch_stub_notebook(activity: NotebookActivity, filename: str) -> str: + """Returns the body of a dynamic-dispatch stub notebook. + + C-28 (NB-ITER4-001): when the ADF ``notebookPath`` is a runtime + expression the translator couldn't reduce (e.g. ``@trim(json(...))``), + the bundle ships a stub that reads ``notebook_path`` from a widget and + calls ``dbutils.notebook.run()`` to dispatch to whatever the workflow + resolved at runtime. The base_parameters dict ferries the rest of the + widgets through. + """ + expression = activity.notebook_path_expression or "(unspecified)" + return ( + "# Databricks notebook source\n" + "# MAGIC %md\n" + f"# MAGIC # Dispatch stub: {activity.name}\n" + "# MAGIC\n" + "# MAGIC The ADF activity's `notebookPath` is a runtime expression that\n" + "# MAGIC flowx could not resolve at translation time.\n" + "# MAGIC\n" + f"# MAGIC **Original expression**: `{expression}`\n" + "# MAGIC\n" + "# MAGIC This stub reads the resolved notebook path from the\n" + f"# MAGIC `{_DISPATCH_STUB_WIDGET}` widget and dispatches via\n" + "# MAGIC `dbutils.notebook.run`. See SETUP.md → *Dynamic notebook dispatch*.\n" + "\n# COMMAND ----------\n\n" + f"dbutils.widgets.text('{_DISPATCH_STUB_WIDGET}', '')\n" + f"target_notebook = dbutils.widgets.get('{_DISPATCH_STUB_WIDGET}')\n" + "if not target_notebook:\n" + " raise ValueError(\n" + f' "Dispatch stub for activity {activity.name!r} requires a runtime "\n' + f" \"value for the '{_DISPATCH_STUB_WIDGET}' widget. See SETUP.md.\"\n" + " )\n" + "\n" + "# Forward every other widget through to the resolved notebook so it\n" + "# receives the same base_parameters the workflow declared.\n" + f"_passthrough_widgets = [w for w in dbutils.widgets.getAll() if w != '{_DISPATCH_STUB_WIDGET}']\n" + "arguments = {name: dbutils.widgets.get(name) for name in _passthrough_widgets}\n" + "\n" + "dbutils.notebook.run(target_notebook, timeout_seconds=0, arguments=arguments)\n" + ) + + def prepare( activity: NotebookActivity, *, @@ -101,6 +146,10 @@ def prepare( variable_task_keys: dict[str, str] | None = None, ) -> PreparedActivity: """Converts a NotebookActivity into a DAB notebook_task definition.""" + # C-28 (NB-ITER4-001): dynamic notebookPath -> emit a dispatch stub. + if activity.notebook_path_unresolved: + return _prepare_dispatch_stub(activity, variable_task_keys=variable_task_keys) + resolved_path = _resolve_notebook_path(activity.notebook_path) task = build_common_task_fields(activity) is_existing_notebook = resolved_path.startswith("/") @@ -113,6 +162,17 @@ def prepare( existing_notebook=is_existing_notebook, ) + approximations = [ + ParameterApproximation( + task_key=activity.task_key, + widget_name=entry["widget_name"], + raw_expression=entry["raw_expression"], + replacement=entry["replacement"], + note=entry["note"], + ) + for entry in activity.parameter_approximations + ] + if is_existing_notebook: downloaded = download_notebook(resolved_path) if workspace_downloads_enabled() else None if downloaded is not None: @@ -121,15 +181,28 @@ def prepare( task["notebook_task"] = {"notebook_path": f"../src/{notebook_relative_path}"} if base_parameters is not None: task["notebook_task"]["base_parameters"] = base_parameters - if activity.compute_mode != "serverless": + if activity.compute_mode != "serverless" and not task.get("existing_cluster_id"): task["job_cluster_key"] = "default_cluster" + if activity.libraries: + task["libraries"] = activity.libraries notebooks = [DabNotebook(relative_path=notebook_relative_path, content=downloaded)] - return PreparedActivity(task=task, notebooks=notebooks) + return PreparedActivity( + task=task, + notebooks=notebooks, + parameter_approximations=approximations, + setup_tasks=_unresolved_library_setup_tasks(activity), + ) task["notebook_task"] = {"notebook_path": resolved_path} if base_parameters is not None: task["notebook_task"]["base_parameters"] = base_parameters - return PreparedActivity(task=task) + if activity.libraries: + task["libraries"] = activity.libraries + return PreparedActivity( + task=task, + parameter_approximations=approximations, + setup_tasks=_unresolved_library_setup_tasks(activity), + ) placeholder_filename = notebook_filename(activity.task_key, activity.name) notebook_relative_path = f"notebooks/{placeholder_filename}" @@ -140,6 +213,106 @@ def prepare( task["notebook_task"] = {"notebook_path": f"../src/{notebook_relative_path}"} if base_parameters is not None: task["notebook_task"]["base_parameters"] = base_parameters + if activity.libraries: + task["libraries"] = activity.libraries + + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=content)] + setup_tasks = _unresolved_library_setup_tasks(activity) + return PreparedActivity( + task=task, + notebooks=notebooks, + parameter_approximations=approximations, + setup_tasks=setup_tasks, + ) + + +def _unresolved_library_setup_tasks(activity: NotebookActivity) -> list[SetupTask]: + """Builds ``unresolved_library`` setup tasks for SETUP.md. + + C-30 (NB-ITER4-003): library entries whose jar/whl path didn't resolve + surface as a SETUP.md section so the user can fix the missing identifier + rather than discovering the failure when the cluster tries to install + a file called ``@concat(...)`` at job-run time. + """ + tasks: list[SetupTask] = [] + for entry in activity.unresolved_libraries: + tasks.append( + SetupTask( + type="unresolved_library", + config={ + "task_key": activity.task_key, + "library_type": entry.get("type", ""), + "expression": entry.get("expression", ""), + "missing": list(entry.get("missing") or []), + }, + ) + ) + return tasks + + +def _prepare_dispatch_stub( + activity: NotebookActivity, + *, + variable_task_keys: dict[str, str] | None = None, +) -> PreparedActivity: + """Builds the bundle artifacts for a dynamic-notebookPath activity. + + C-28 (NB-ITER4-001): emits a dispatch-stub notebook (reads + ``notebook_path`` widget and ``dbutils.notebook.run()``s it), a + SetupTask of kind ``dynamic_notebook_dispatch`` for SETUP.md, and + threads the original base_parameters through. + """ + task = build_common_task_fields(activity) + filename = notebook_filename(activity.task_key, activity.name) + notebook_relative_path = f"notebooks/{filename}" + + base_parameters: dict[str, str] = {} + base_parameters[_DISPATCH_STUB_WIDGET] = "" + if activity.base_parameters: + for key, value in _resolve_base_parameters( + dict(activity.base_parameters), + variable_task_keys=variable_task_keys, + existing_notebook=False, + ).items(): + base_parameters.setdefault(key, value) + + content = _dispatch_stub_notebook(activity, filename) + + task["notebook_task"] = { + "notebook_path": f"../src/{notebook_relative_path}", + "base_parameters": base_parameters, + } + if activity.libraries: + task["libraries"] = activity.libraries + + setup_tasks: list[SetupTask] = [ + SetupTask( + type="dynamic_notebook_dispatch", + config={ + "task_key": activity.task_key, + "activity_name": activity.name, + "expression": activity.notebook_path_expression or "", + "widget_name": _DISPATCH_STUB_WIDGET, + }, + ) + ] + setup_tasks.extend(_unresolved_library_setup_tasks(activity)) + + approximations = [ + ParameterApproximation( + task_key=activity.task_key, + widget_name=entry["widget_name"], + raw_expression=entry["raw_expression"], + replacement=entry["replacement"], + note=entry["note"], + ) + for entry in activity.parameter_approximations + ] notebooks = [DabNotebook(relative_path=notebook_relative_path, content=content)] - return PreparedActivity(task=task, notebooks=notebooks) + return PreparedActivity( + task=task, + notebooks=notebooks, + setup_tasks=setup_tasks, + parameter_approximations=approximations, + ) diff --git a/src/orchestra/preparer/activity_preparers/set_variable.py b/src/orchestra/preparer/activity_preparers/set_variable.py index 5820929..e84e5fe 100644 --- a/src/orchestra/preparer/activity_preparers/set_variable.py +++ b/src/orchestra/preparer/activity_preparers/set_variable.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING +from flowx.models.dab import SetupTask from flowx.preparer.activity_preparers.helpers import build_notebook_activity_task from flowx.preparer.activity_preparers.naming import notebook_filename from flowx.preparer.code_generator import generate_set_variable_notebook @@ -18,6 +19,8 @@ def prepare(activity: SetVariableActivity, *, scope: str = "") -> PreparedActivi base_parameters: dict[str, str] = {"variable_name": activity.variable_name} if activity.value_kind in ("literal", "dab_ref"): base_parameters["value"] = activity.variable_value + elif activity.value_kind == "unresolved": + base_parameters["value"] = "" for widget_name, dab_ref in activity.required_parameters.items(): base_parameters.setdefault(widget_name, dab_ref) @@ -27,4 +30,20 @@ def prepare(activity: SetVariableActivity, *, scope: str = "") -> PreparedActivi notebook_content=generate_set_variable_notebook(activity), base_parameters=base_parameters, ) - return PreparedActivity(task=task, notebooks=notebooks) + + setup_tasks: list[SetupTask] = [] + # C-33 (VAREX4-001 / CF4-003): emit a manual_variable_init SetupTask so + # SETUP.md flags the variable as needing a runtime value. + if activity.value_kind == "unresolved" and activity.raw_expression: + setup_tasks.append( + SetupTask( + type="manual_variable_init", + config={ + "task_key": activity.task_key, + "variable_name": activity.variable_name, + "expression": activity.raw_expression, + }, + ) + ) + + return PreparedActivity(task=task, notebooks=notebooks, setup_tasks=setup_tasks) diff --git a/src/orchestra/preparer/activity_preparers/spark_python.py b/src/orchestra/preparer/activity_preparers/spark_python.py index 9a9c5e7..9156967 100644 --- a/src/orchestra/preparer/activity_preparers/spark_python.py +++ b/src/orchestra/preparer/activity_preparers/spark_python.py @@ -69,4 +69,6 @@ def prepare(activity: SparkPythonActivity, *, scope: str = "") -> PreparedActivi } if activity.parameters: task["spark_python_task"]["parameters"] = list(activity.parameters) + if activity.libraries: + task["libraries"] = activity.libraries return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/orchestra/preparer/activity_preparers/switch.py b/src/orchestra/preparer/activity_preparers/switch.py index a442f0c..bfec286 100644 --- a/src/orchestra/preparer/activity_preparers/switch.py +++ b/src/orchestra/preparer/activity_preparers/switch.py @@ -10,6 +10,7 @@ import re from typing import TYPE_CHECKING, Any +from flowx.models.dab import DabNotebook from flowx.models.ir import TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string from flowx.preparer.activity_preparers.if_condition import inject_outcome_dependency @@ -24,6 +25,8 @@ if TYPE_CHECKING: from flowx.models.ir import SwitchActivity +_BRIDGE_PLACEHOLDER_PREFIX = "__BRIDGE__::" + def sanitize_case_key(value: str) -> str: """Returns a task-key-safe form of a switch case value. @@ -44,14 +47,33 @@ def resolve_switch_on_expression(on_expression: str) -> str: plain literal passes through unchanged. Both the in-process preparer and the JSON-reload path call this so a hand-edited IR with a raw ``@variables(...)`` is still resolved before being written to YAML. + + C-13 (CF-iter2-004): when the input already contains a ``{{...}}`` + DAB dynamic value reference or is not an ``@``-prefixed ADF + expression, return it unchanged. Constructing a bare + :class:`TranslationContext` from this side strips global parameters + and the variable_cache, so re-resolving a previously-lowered ref + would discard the data the translator already populated. The + translator-side bridge placeholder (``__BRIDGE__::``) is likewise + preserved so the bridge rewrite step downstream can fill it. """ + if not isinstance(on_expression, str): + return on_expression + if not on_expression: + return on_expression + # Already a DAB ref / translator placeholder: pass through unchanged. + if "{{" in on_expression or on_expression.startswith(_BRIDGE_PLACEHOLDER_PREFIX): + return on_expression + # Only attempt resolution for bare ADF expressions. Other strings + # (raw literals) pass through. + if not on_expression.startswith("@"): + return on_expression context = TranslationContext() if "@{" in on_expression: return resolve_interpolated_string(on_expression, context) - if on_expression.startswith("@"): - result = resolve_expression(on_expression, context) - if result is not None and result.kind in ("dab_ref", "literal"): - return result.value + result = resolve_expression(on_expression, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value return on_expression @@ -70,6 +92,11 @@ def prepare(activity: SwitchActivity, *, scope: str = "") -> PreparedActivity: extra_tasks: list[dict[str, Any]] = [] resolved_expr = resolve_switch_on_expression(activity.on_expression) + # C-07: if the translator produced a bridge_notebook_code, synthesise + # the bridge task and rewrite the on-expression to its task value. + bridge_task, bridge_value_ref, bridge_notebooks = _build_switch_bridge_task(activity) + if bridge_value_ref is not None and resolved_expr.startswith(_BRIDGE_PLACEHOLDER_PREFIX): + resolved_expr = bridge_value_ref if not activity.cases: task = build_common_task_fields(activity) @@ -83,10 +110,19 @@ def prepare(activity: SwitchActivity, *, scope: str = "") -> PreparedActivity: artifacts = merge_prepared_artifacts(artifacts, prepared) inject_outcome_dependency(default_tasks, activity.task_key, "true") + extras_no_cases: list[dict[str, Any]] = [] + notebooks_no_cases = list(artifacts.notebooks) + if bridge_task is not None: + extras_no_cases.append(bridge_task) + notebooks_no_cases.extend(bridge_notebooks) + existing_deps = list(task.get("depends_on") or []) + task["depends_on"] = [*existing_deps, {"task_key": bridge_task["task_key"]}] + extras_no_cases.extend(default_tasks) + return PreparedActivity( task=task, - extra_tasks=default_tasks, - notebooks=list(artifacts.notebooks), + extra_tasks=extras_no_cases, + notebooks=notebooks_no_cases, secrets=list(artifacts.secrets), setup_tasks=list(artifacts.setup_tasks), inner_workflows=list(artifacts.inner_workflows), @@ -145,12 +181,87 @@ def prepare(activity: SwitchActivity, *, scope: str = "") -> PreparedActivity: # original Switch task_key onto the renamed first case. remap = {activity.task_key: case_keys[0]} + notebooks_out = list(artifacts.notebooks) + extras_out = extra_tasks + if bridge_task is not None: + # Bridge task runs first; the first condition task depends on it. + extras_out = [bridge_task, *extra_tasks] + notebooks_out.extend(bridge_notebooks) + existing_deps = list(first_condition_task.get("depends_on") or []) + first_condition_task["depends_on"] = [ + *existing_deps, + {"task_key": bridge_task["task_key"]}, + ] + return PreparedActivity( task=first_condition_task, - extra_tasks=extra_tasks, - notebooks=list(artifacts.notebooks), + extra_tasks=extras_out, + notebooks=notebooks_out, secrets=list(artifacts.secrets), setup_tasks=list(artifacts.setup_tasks), inner_workflows=list(artifacts.inner_workflows), task_key_remap=remap, ) + + +def _build_switch_bridge_task( + activity: SwitchActivity, +) -> tuple[dict[str, Any] | None, str | None, list[DabNotebook]]: + """Synthesises a bridge SetVariable-style task for a Switch on-expression. + + C-07 (CF-iter2-001 / CF-iter2-003): when ``on_expression`` contains an + ADF function call (e.g. ``@toUpper(coalesce(item()?.type, 'default'))``) + we route the value through a hidden notebook task so the + ``condition_task.left`` operand is a real task-value reference and not + a raw ADF expression string. + """ + if not activity.bridge_notebook_code: + return None, None, [] + + bridge_key = f"{activity.task_key}_bridge" + value_key = "result" + notebook_relative_path = f"notebooks/{bridge_key}.py" + + base_parameters: dict[str, str] = dict(activity.bridge_required_parameters) + notebook_source = _render_bridge_notebook( + activity.bridge_notebook_code, + activity.bridge_notebook_imports, + list(base_parameters.keys()), + value_key, + ) + + bridge_task: dict[str, Any] = { + "task_key": bridge_key, + "notebook_task": { + "notebook_path": f"../src/{notebook_relative_path}", + "base_parameters": base_parameters, + }, + } + bridge_value_ref = f"{{{{tasks.{bridge_key}.values.{value_key}}}}}" + notebooks = [DabNotebook(relative_path=notebook_relative_path, content=notebook_source)] + return bridge_task, bridge_value_ref, notebooks + + +def _render_bridge_notebook( + notebook_code: str, + imports: list[str], + widget_names: list[str], + value_key: str, +) -> str: + """Generates the Python source for a Switch on-expression bridge notebook.""" + lines: list[str] = [] + seen_imports: set[str] = set() + for imp in imports: + if imp in seen_imports: + continue + seen_imports.add(imp) + lines.append(imp) + if seen_imports: + lines.append("") + for widget in widget_names: + lines.append(f"dbutils.widgets.text('{widget}', '')") + if widget_names: + lines.append("") + lines.append(f"_bridge_value = {notebook_code}") + lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") + return "\n".join(lines) + "\n" diff --git a/src/orchestra/preparer/activity_preparers/web_activity.py b/src/orchestra/preparer/activity_preparers/web_activity.py index b1d72f4..bc6245d 100644 --- a/src/orchestra/preparer/activity_preparers/web_activity.py +++ b/src/orchestra/preparer/activity_preparers/web_activity.py @@ -2,9 +2,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any -from flowx.models.dab import SecretInstruction +from flowx.models.dab import SecretInstruction, SetupTask from flowx.preparer.activity_preparers.helpers import ( build_notebook_activity_task, resolve_param_value, @@ -19,25 +19,135 @@ def prepare(activity: WebActivity, *, scope: str = "") -> PreparedActivity: """Converts a WebActivity into a notebook_task with a generated HTTP notebook.""" + secrets, setup_tasks = _extract_secrets_and_setup(activity, scope=scope) + + # C-38 (LSC4-002): when the preparer resolved an AzureKeyVaultSecret + # payload to a real (scope, key) pair, thread it into the notebook + # generator so the rendered ``dbutils.secrets.get`` references the + # real values rather than the hard-coded ``scope=task_key, + # key='auth-credential'`` fallback. + credential_scope: str | None = None + credential_key: str | None = None + if secrets: + first = secrets[0] + credential_scope = first.scope + credential_key = first.key + task, notebooks = build_notebook_activity_task( activity, notebook_relative_path=f"notebooks/{notebook_filename(activity.task_key, activity.name)}", - notebook_content=generate_web_activity_notebook(activity, scope=scope), + notebook_content=generate_web_activity_notebook( + activity, + scope=scope, + credential_scope=credential_scope, + credential_key=credential_key, + ), base_parameters={ "url": resolve_param_value(activity.url), "method": resolve_param_value(activity.method), }, ) + return PreparedActivity( + task=task, + notebooks=notebooks, + secrets=secrets, + setup_tasks=setup_tasks, + ) + + +def _extract_secrets_and_setup( + activity: WebActivity, *, scope: str = "" +) -> tuple[list[SecretInstruction], list[SetupTask]]: + """Inspects the Web activity's authentication payload and emits per-secret refs. + + C-11 (LSC2-005): the legacy implementation always emitted a single + static ``auth-credential`` SecretInstruction regardless of the + underlying ADF auth shape, so AzureKeyVaultSecret payloads lost their + Key Vault scope/secret name and CredentialReference (MSI) payloads + surfaced a placeholder secret that never matches a real secret in + the workspace. + """ secrets: list[SecretInstruction] = [] - if activity.authentication: - auth_type = activity.authentication.get("type", "unknown") + setup_tasks: list[SetupTask] = [] + + auth = activity.authentication or {} + if not auth: + return secrets, setup_tasks + + auth_type = auth.get("type", "unknown") + default_scope = scope or activity.task_key + + # Common nested fields per ADF auth shapes. + for field_name in ("password", "secret", "clientSecret", "pfx", "key"): + field_value = auth.get(field_name) + secret = _materialise_secret(field_value, default_scope=default_scope, role=field_name) + if secret is not None: + secrets.append(secret) + + if auth_type == "MSI" or auth.get("credential"): + # CredentialReference (managed identity) has no static secret -- emit + # a SETUP.md note instead of a fake placeholder secret. + cred = auth.get("credential") or {} + cred_name = cred.get("referenceName") if isinstance(cred, dict) else None + setup_tasks.append( + SetupTask( + type="manual_credential", + config={ + "activity_name": activity.name, + "credential_reference": cred_name or "", + "note": ( + "Web activity uses an Azure managed-identity credential. " + "Configure equivalent OAuth or service-principal auth in Databricks " + "and update the generated notebook." + ), + }, + ) + ) + + if not secrets and auth_type not in ("MSI",) and not auth.get("credential"): + # Fallback for shapes the per-field probe didn't recognise -- preserve + # the legacy behaviour so callers depending on it still get something. secrets.append( SecretInstruction( - scope=scope or activity.task_key, + scope=default_scope, key="auth-credential", value_source=f"Authentication credential ({auth_type}) for web activity '{activity.name}'", ) ) - return PreparedActivity(task=task, notebooks=notebooks, secrets=secrets) + return secrets, setup_tasks + + +def _materialise_secret(value: Any, *, default_scope: str, role: str) -> SecretInstruction | None: + """Builds a :class:`SecretInstruction` from an ADF secret payload. + + Handles the two common shapes: + - ``{"type": "AzureKeyVaultSecret", "store": {"referenceName": ...}, "secretName": ...}`` + - ``{"type": "SecureString", "value": ...}`` + + Returns ``None`` for shapes we cannot map. + """ + if not isinstance(value, dict): + return None + payload_type = value.get("type") + if payload_type == "AzureKeyVaultSecret": + store = value.get("store") or {} + scope_name = store.get("referenceName") or default_scope + secret_name = value.get("secretName") or role + base_url = (value.get("typeProperties") or {}).get("baseUrl", "") + value_source = f"Azure Key Vault secret '{secret_name}'" + if base_url: + value_source += f" at {base_url}" + return SecretInstruction( + scope=str(scope_name), + key=str(secret_name), + value_source=value_source, + ) + if payload_type == "SecureString": + return SecretInstruction( + scope=default_scope, + key=role, + value_source=f"SecureString carried inline in the ADF activity (role={role})", + ) + return None diff --git a/src/orchestra/preparer/code_generator.py b/src/orchestra/preparer/code_generator.py index 4bb12fa..2f2e43a 100644 --- a/src/orchestra/preparer/code_generator.py +++ b/src/orchestra/preparer/code_generator.py @@ -88,6 +88,8 @@ def generate_lookup_notebook(activity: LookupActivity, *, scope: str = "") -> st for col_name, col_value in output.items(): dbutils.jobs.taskValues.set(key=col_name, value=col_value) """) + elif _is_file_lookup(activity): + body = _file_lookup_body(activity) else: body = textwrap.dedent(f"""\ import json @@ -116,12 +118,189 @@ def generate_lookup_notebook(activity: LookupActivity, *, scope: str = "") -> st return header + _command_separator() + body -def generate_web_activity_notebook(activity: WebActivity, *, scope: str = "") -> str: +_DATASET_TYPE_TO_SPARK_FORMAT: dict[str, str] = { + "Json": "json", + "Parquet": "parquet", + "DelimitedText": "csv", + "Avro": "avro", + "Orc": "orc", + "Excel": "com.crealytics.spark.excel", + "Xml": "xml", + "Binary": "binaryFile", +} + + +def _is_file_lookup(activity: LookupActivity) -> bool: + """Return True when the LookupActivity carries a file-source dataset.""" + props = activity.source_properties or {} + dataset_type = props.get("dataset_type") + return bool(dataset_type) and dataset_type in _DATASET_TYPE_TO_SPARK_FORMAT + + +def _coerce_to_str(value: Any) -> str: + """Defensive coercion for file-Lookup path components. + + C-37 (LSC4-001): folder_path / file_name occasionally arrive as ADF + expression dicts (``{"value": ..., "type": "Expression"}``) when the + translator's unwrap pass missed them. ``.strip('/')`` on a dict + crashes the bundler. Coerce to a string so the worst-case outcome + is a missing path component instead of a stack trace that aborts + bundle generation. + """ + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, dict) and "value" in value: + inner = value["value"] + return str(inner) if inner is not None else "" + return str(value) + + +_ADLS_HTTPS_RE = re.compile( + r"^https?://(?P[A-Za-z0-9\-]+)\.(?:dfs|blob)\.core\.windows\.net(?P/.*)?$", + re.IGNORECASE, +) + + +def _rewrite_abfss(url: str, container: str) -> str: + """Rewrites an ``https://.dfs.core.windows.net`` URL to abfss://. + + C-37 (LSC4-003): AzureBlobFS linked services often surface the + HTTPS endpoint instead of the abfss:// form Databricks expects on a + cluster. When the container is known, rewrite to + ``abfss://@.dfs.core.windows.net`` so the + generated lookup notebook can actually read the path. + """ + if not container: + return url + match = _ADLS_HTTPS_RE.match(url) + if match is None: + return url + account = match.group("account") + rest = (match.group("path") or "").lstrip("/") + suffix = f"/{rest}" if rest else "" + return f"abfss://{container}@{account}.dfs.core.windows.net{suffix}" + + +def _assemble_file_lookup_source_path(props: dict[str, Any]) -> str: + """Compose a fully-qualified default source path for a file-source Lookup. + + LSC3-005: when the bound linked service exposes a URL like + ``abfss://container@account.dfs.core.windows.net``, append the dataset's + folder + filename onto the URL so the lookup notebook ships a real + default rather than ``''``. Returns an empty string when no URL is + available -- callers can still override via the widget at runtime. + + C-37 (LSC4-001 + LSC4-003): defensively coerce expression-dict values + to strings before ``.strip('/')`` so 4 pipelines that previously + crashed with AttributeError now emit bundles. Also rewrites + ``https://.dfs.core.windows.net`` URLs to ``abfss://`` when + a container is known so the lookup notebook reads the real ADLS + path rather than the HTTPS REST endpoint. + """ + url = _coerce_to_str(props.get("linked_service_url")) + folder = _coerce_to_str(props.get("folder_path")) + filename = _coerce_to_str(props.get("file_name")) + container = _coerce_to_str(props.get("container")) + # C-47 (LSC5-001): never join a raw ``dataset()`` reference into the + # baked default path. lookup.translate substitutes these from the + # dataset reference's parameter bindings; if one still leaks through + # (e.g. an unbound dataset parameter) drop it so spark.read does not get + # a literal broken ``abfss://.../@dataset().fileName`` path. + if "dataset(" in folder: + folder = "" + if "dataset(" in filename: + filename = "" + if url: + url = _rewrite_abfss(url, container) + if not (url or folder or filename): + return "" + parts: list[str] = [] + if url: + parts.append(url.rstrip("/")) + if folder: + parts.append(folder.strip("/")) + if filename: + parts.append(filename.strip("/")) + return "/".join(p for p in parts if p) + + +def _file_lookup_body(activity: LookupActivity) -> str: + """Render the notebook body for a file-source Lookup. + + Builds a ``spark.read.format(...).option(...).load()`` call + with multiline JSON handling for ``firstRowOnly=False`` over arrayOfObjects. + The source_path is read from a widget so callers can override per run. + + LSC3-005: when the bound linked service supplies a URL (e.g. abfss:// + container@account.dfs.core.windows.net), the default widget value is + pre-populated with the fully-assembled URI so the notebook reads from + the right place without manual SETUP.md fixups. + """ + props = activity.source_properties or {} + dataset_type = props.get("dataset_type", "Json") + spark_format = _DATASET_TYPE_TO_SPARK_FORMAT.get(dataset_type, "json") + + options: list[str] = [] + if dataset_type == "Json": + # ADF Lookup with firstRowOnly=False over a JSON file typically + # walks an array-of-objects, which requires multiline. + if not activity.first_row_only or props.get("multiLineJson"): + options.append('.option("multiline", "true")') + options_block = "\n ".join(options) + options_section = ("\n " + options_block) if options_block else "" + + default_source_path = _assemble_file_lookup_source_path(props) + default_path_literal = repr(default_source_path) if default_source_path else "''" + + body = textwrap.dedent(f"""\ + import json + + # Parameters + first_row_only = dbutils.widgets.get("first_row_only") == "true" + source_path = dbutils.widgets.get("source_path") or {default_path_literal} + + # File-source Lookup + df = ( + spark.read.format({spark_format!r})__OPTIONS__ + .load(source_path) + ) + + if first_row_only: + result = df.first() + output = result.asDict() if result else {{}} + else: + output = [row.asDict() for row in df.collect()] + + dbutils.jobs.taskValues.set(key="result", value=json.dumps(output)) + if first_row_only and isinstance(output, dict): + for col_name, col_value in output.items(): + dbutils.jobs.taskValues.set(key=col_name, value=col_value) + """) + return body.replace("__OPTIONS__", options_section) + + +def generate_web_activity_notebook( + activity: WebActivity, + *, + scope: str = "", + credential_scope: str | None = None, + credential_key: str | None = None, +) -> str: """Generates a Python notebook that makes an HTTP request. Args: activity: The WebActivity IR node. scope: Secret scope name (defaults to task_key if empty). + credential_scope: C-38 (LSC4-002): when the preparer resolved the + auth payload to a real (scope, key) pair (e.g. an + AzureKeyVaultSecret with ``lakeh_ls_keyvault`` / + ``adapp-...-secret``), pass them through so the rendered + ``dbutils.secrets.get`` call references the real values + rather than the hard-coded ``scope=task_key, + key='auth-credential'`` fallback that never matches. + credential_key: See ``credential_scope``. Returns: Complete notebook source code as a string. @@ -137,10 +316,32 @@ def generate_web_activity_notebook(activity: WebActivity, *, scope: str = "") -> if auth: scope = scope or activity.task_key auth_type = auth.get("type", "") - if auth_type in ("ServicePrincipal", "MSI", "ManagedServiceIdentity"): + # C-38 (LSC4-002): prefer the resolved (scope, key) tuple from the + # preparer when supplied. Fall back to the legacy + # ``(task_key, 'auth-credential')`` shape only when the preparer + # didn't (or couldn't) compute one. + resolved_scope = credential_scope or scope + resolved_key = credential_key or "auth-credential" + if auth_type in ("MSI", "ManagedServiceIdentity"): + # LSC3-002: MSI / Managed Identity auth carries no static secret, + # so reading ``auth-credential`` from a secret scope is a fake + # placeholder that fails at runtime. Surface a NotImplementedError + # so the user can implement the credential exchange manually -- + # the manual_credential SetupTask emitted by web_activity preparer + # already flags this in SETUP.md. + auth_block = textwrap.dedent(f"""\ + # Authentication ({auth_type}) - manual implementation required + raise NotImplementedError( + "WebActivity authentication type '{auth_type}' has no static " + "secret to read. See SETUP.md (Manual credential setup) for " + "the Databricks equivalent (e.g. workspace OAuth M2M, " + "service principal token exchange)." + ) + """) + elif auth_type == "ServicePrincipal": auth_block = textwrap.dedent(f"""\ - # Authentication ({auth_type}) - auth_token = dbutils.secrets.get(scope="{scope}", key="auth-credential") + # Authentication (ServicePrincipal) + auth_token = dbutils.secrets.get(scope="{resolved_scope}", key="{resolved_key}") headers["Authorization"] = f"Bearer {{auth_token}}" """) elif auth_type == "Basic": @@ -148,14 +349,14 @@ def generate_web_activity_notebook(activity: WebActivity, *, scope: str = "") -> # Authentication (Basic) import base64 username = dbutils.secrets.get(scope="{scope}", key="auth-username") - password = dbutils.secrets.get(scope="{scope}", key="auth-credential") + password = dbutils.secrets.get(scope="{resolved_scope}", key="{resolved_key}") token = base64.b64encode(f"{{username}}:{{password}}".encode()).decode() headers["Authorization"] = f"Basic {{token}}" """) else: auth_block = textwrap.dedent(f"""\ # Authentication - auth_credential = dbutils.secrets.get(scope="{scope}", key="auth-credential") + auth_credential = dbutils.secrets.get(scope="{resolved_scope}", key="{resolved_key}") headers["Authorization"] = f"Bearer {{auth_credential}}" """) diff --git a/src/orchestra/preparer/workflow_preparer.py b/src/orchestra/preparer/workflow_preparer.py index 815ae4c..57a178f 100644 --- a/src/orchestra/preparer/workflow_preparer.py +++ b/src/orchestra/preparer/workflow_preparer.py @@ -2,10 +2,11 @@ from __future__ import annotations +from collections.abc import Iterable from dataclasses import dataclass, field from typing import Any -from flowx.models.dab import DabNotebook, SecretInstruction, SetupTask +from flowx.models.dab import DabNotebook, ParameterApproximation, SecretInstruction, SetupTask from flowx.models.ir import ( Activity, AppendVariableActivity, @@ -50,6 +51,7 @@ class PreparedActivity: # ``resources/pipelines/.yml``. Each entry is a dict # with ``resource_key`` and ``definition`` keys. pipeline_resources: list[dict[str, Any]] = field(default_factory=list) + parameter_approximations: list[ParameterApproximation] = field(default_factory=list) @dataclass(slots=True, kw_only=True) @@ -65,6 +67,10 @@ class PreparedWorkflow: parameters: list[dict[str, Any]] = field(default_factory=list) cluster_hints: list[dict[str, Any]] = field(default_factory=list) pipeline_resources: list[dict[str, Any]] = field(default_factory=list) + parameter_approximations: list[ParameterApproximation] = field(default_factory=list) + # C-10 (SCHED-001): serialised schedule / trigger spec the bundler + # renders as ``schedule:`` / ``trigger:`` on the emitted DAB job. + schedule: dict[str, Any] | None = None def run_if_from_adf_outcomes(outcomes: list[str | None]) -> str | None: @@ -106,6 +112,9 @@ def build_common_task_fields(activity: Activity) -> dict[str, Any]: if activity.description: task["description"] = activity.description + if activity.existing_cluster_id: + task["existing_cluster_id"] = activity.existing_cluster_id + return task @@ -168,6 +177,12 @@ def prepare_activity( prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) elif type(activity) is AppendVariableActivity: prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) + elif type(activity) is ForEachActivity: + # C-06 (VAREX-004): inner-job parameter collector needs the parent's + # variable -> setter mapping so @variables('X') references inside the + # ForEach body route through {{tasks.X.values.Y}} rather than an + # undeclared {{job.parameters.X}}. + prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) else: prepared = preparer_fn(activity, scope=scope) @@ -243,6 +258,7 @@ class PreparedArtifacts: setup_tasks: tuple[SetupTask, ...] = () inner_workflows: tuple[PreparedWorkflow, ...] = () pipeline_resources: tuple[dict[str, Any], ...] = () + parameter_approximations: tuple[ParameterApproximation, ...] = () def merge_prepared_artifacts( @@ -256,6 +272,7 @@ def merge_prepared_artifacts( setup_tasks=artifacts.setup_tasks + tuple(prepared.setup_tasks), inner_workflows=artifacts.inner_workflows + tuple(prepared.inner_workflows), pipeline_resources=artifacts.pipeline_resources + tuple(prepared.pipeline_resources), + parameter_approximations=artifacts.parameter_approximations + tuple(prepared.parameter_approximations), ) @@ -277,8 +294,14 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: all_tasks.extend(prepared.extra_tasks) artifacts = merge_prepared_artifacts(artifacts, prepared) task_key_remap.update(prepared.task_key_remap) - if activity.cluster: - cluster_hints.append(dict(activity.cluster)) + # C-04 (NB-ITER2-4 / LSC2-001): walk into IfCondition / Switch / + # ForEach branches so the workflow's cluster_hints aggregation + # picks up cluster config on activities nested inside compound + # activities. Without this the default Standard_DS3_v2 / 15.4.x + # fallback ships even when the inner notebook has an explicit LS. + for nested_activity in _iter_activity_with_descendants(activity): + if nested_activity.cluster: + cluster_hints.append(dict(nested_activity.cluster)) if isinstance(activity, (SetVariableActivity, AppendVariableActivity)): variable_task_keys_map[activity.variable_name] = activity.task_key @@ -298,18 +321,194 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: seen_secrets.add(secret_id) unique_secrets.append(secret) + # VAREX3-003: emit a manual_variable_rollup SetupTask whenever a sibling + # IfCondition / Switch / SetVariable reads a variable that is only + # mutated inside a ForEach inner job. ADF semantics treat the post- + # ForEach read as "latest committed value" but that value is unreachable + # across the run_job_task boundary in DAB. Surfacing the warning lets + # the user add a roll-up notebook before the dependent activity runs. + cross_scope_rollups = _detect_cross_foreach_variable_reads(pipeline.tasks) + setup_tasks_out = _dedupe_setup_tasks(artifacts.setup_tasks) + setup_tasks_out.extend(cross_scope_rollups) + # C-36 (SCHED4-001): emit a manual_schedule_time_of_day SetupTask + # whenever the trigger.periodic schedule carries hours/minutes/weekDays + # that the periodic primitive can't encode. SETUP.md picks it up so + # the user can manually add the time-of-day to the cron expression. + if pipeline.schedule and pipeline.schedule.get("time_of_day_note"): + setup_tasks_out.append( + SetupTask( + type="manual_schedule_time_of_day", + config={ + "pipeline": pipeline.name, + "frequency": pipeline.schedule.get("unit", ""), + "interval": pipeline.schedule.get("interval", ""), + "time_of_day_note": pipeline.schedule.get("time_of_day_note"), + }, + ) + ) + + # C-39 (LSC4-004): when any cluster hint references an ADF + # authentication mode that has no direct Databricks equivalent (MSI, + # CredentialReference) the bundle's default_cluster silently uses + # ``single_user_name: ${workspace.current_user.userName}``. Surface + # a manual_credential SetupTask so SETUP.md flags the substitution. + seen_auth: set[tuple[str, str]] = set() + for hint in cluster_hints: + auth = hint.get("_adf_authentication") or "" + cred = hint.get("_adf_credential_reference") or "" + if not auth and not cred: + continue + key = (str(auth), str(cred)) + if key in seen_auth: + continue + seen_auth.add(key) + setup_tasks_out.append( + SetupTask( + type="manual_credential", + config={ + "source": pipeline.name, + "linked_service": cred or "", + "authentication": auth or "CredentialReference", + "note": ( + "ADF cluster auth has no Databricks equivalent. The " + "default_cluster runs as ${workspace.current_user.userName}; " + "swap to a service principal via single_user_name or " + "set run_as.service_principal_name on the job." + ), + }, + ) + ) + return PreparedWorkflow( name=pipeline.name, tasks=all_tasks, notebooks=list(artifacts.notebooks), secrets=unique_secrets, - setup_tasks=_dedupe_setup_tasks(artifacts.setup_tasks), + setup_tasks=setup_tasks_out, inner_workflows=list(artifacts.inner_workflows), cluster_hints=cluster_hints, pipeline_resources=list(artifacts.pipeline_resources), + parameter_approximations=list(artifacts.parameter_approximations), + schedule=pipeline.schedule, ) +def _detect_cross_foreach_variable_reads(activities: list[Activity]) -> list[SetupTask]: + """Returns SetupTasks for variables mutated inside a ForEach but read outside. + + VAREX3-003: when a SetVariable for `X` lives only inside a ForEach + inner-job, a sibling task reading @variables('X') gets the stale init + value (post-ForEach reads cannot cross the run_job_task boundary in + DAB). Surfacing this as a manual_variable_rollup SetupTask gives the + user a documented workaround (add a roll-up notebook that copies the + final value to a parent-scope task value). + """ + import re + + var_ref_pattern = re.compile(r"@?variables\(\s*'([^']+)'\s*\)", re.IGNORECASE) + + # Index variable -> set of ForEach activity names that contain the setter + # so we can name the parent in the warning message. + var_set_inside_foreach: dict[str, list[str]] = {} + for activity in activities: + if isinstance(activity, ForEachActivity): + for inner in activity.inner_activities: + if isinstance(inner, SetVariableActivity): + var_set_inside_foreach.setdefault(inner.variable_name, []).append(activity.task_key) + + if not var_set_inside_foreach: + return [] + + # Identify variables that are also set OUTSIDE any ForEach -- those are + # not cross-scope dangers because the parent always has a fresh setter + # to point at. + set_outside: set[str] = set() + for activity in activities: + if isinstance(activity, SetVariableActivity): + set_outside.add(activity.variable_name) + + dangerous_vars = {name: parents for name, parents in var_set_inside_foreach.items() if name not in set_outside} + if not dangerous_vars: + return [] + + # Now find sibling reads of those variables. Walk every top-level + # activity that is NOT the originating ForEach and collect refs. + flagged: dict[str, str] = {} # variable -> parent task_key + + def _read_refs(text: str) -> set[str]: + if not isinstance(text, str): + return set() + return {m.group(1) for m in var_ref_pattern.finditer(text)} + + def _walk_activity_strings(activity: Activity) -> Iterable[str]: + # Yield every string-like field the variable might appear in. + if isinstance(activity, IfConditionActivity): + yield activity.left or "" + yield activity.right or "" + if isinstance(activity, SwitchActivity): + yield activity.on_expression or "" + if isinstance(activity, SetVariableActivity): + yield activity.variable_value or "" + if isinstance(activity, NotebookActivity): + for value in (activity.base_parameters or {}).values(): + if isinstance(value, str): + yield value + if isinstance(activity, WebActivity): + yield activity.url or "" + if isinstance(activity.body, str): + yield activity.body + + for activity in activities: + # Skip ForEach themselves (siblings only). + if isinstance(activity, ForEachActivity): + continue + for text in _walk_activity_strings(activity): + for var_name in _read_refs(text): + if var_name in dangerous_vars and var_name not in flagged: + flagged[var_name] = dangerous_vars[var_name][0] + + return [ + SetupTask( + type="manual_variable_rollup", + config={ + "variable_name": var_name, + "parent_foreach": parent_key, + "message": ( + f"Variable '{var_name}' is mutated inside ForEach " + f"'{parent_key}' but read in a sibling task. Task " + f"values cannot cross run_job_task boundaries; add a " + f"roll-up notebook that copies the final value to a " + f"parent-scope task value before the sibling runs." + ), + }, + ) + for var_name, parent_key in sorted(flagged.items()) + ] + + +def _iter_activity_with_descendants(activity: Activity) -> Iterable[Activity]: + """Yields *activity* and every nested branch activity (BFS). + + C-04 (NB-ITER2-4 / LSC2-001): IfCondition / Switch / ForEach activities + nest sub-activities in branch fields (``if_true_activities``, + ``if_false_activities``, ``inner_activities``, ``cases[].activities``, + ``default_activities``). ``prepare_workflow`` previously only saw the + top-level tasks, so cluster hints carried by a deeply nested + NotebookActivity were dropped. + """ + queue: list[Activity] = [activity] + while queue: + current = queue.pop(0) + yield current + for attr in ("inner_activities", "if_true_activities", "if_false_activities"): + nested = getattr(current, attr, None) or [] + queue.extend(nested) + if isinstance(current, SwitchActivity): + for case_item in current.cases: + queue.extend(case_item.activities) + queue.extend(current.default_activities) + + def _dedupe_setup_tasks(setup_tasks: tuple[SetupTask, ...]) -> list[SetupTask]: """Returns the setup-task list with duplicates collapsed by identifying config. diff --git a/src/orchestra/translator/activity_translators/execute_pipeline.py b/src/orchestra/translator/activity_translators/execute_pipeline.py index 6c2eef1..89276c9 100644 --- a/src/orchestra/translator/activity_translators/execute_pipeline.py +++ b/src/orchestra/translator/activity_translators/execute_pipeline.py @@ -6,7 +6,8 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, ExecutePipelineActivity, TranslationContext -from flowx.translator.activity_translators.resolve import resolve_dict_values, resolve_field +from flowx.parser.expression_parser import resolve_expression +from flowx.translator.activity_translators.resolve import resolve_field def translate( @@ -35,12 +36,67 @@ def translate( else str(pipeline_ref) ) - parameters = resolve_dict_values(type_properties.get("parameters"), context) or {} + raw_parameters = type_properties.get("parameters") or {} + parameters: dict[str, str] = {} + approximations: list[dict[str, str]] = list(base_kwargs.get("parameter_approximations") or []) + for name, value in raw_parameters.items(): + resolved_value = _resolve_execute_pipeline_parameter(name, value, context, approximations) + if resolved_value is not None: + parameters[name] = resolved_value + wait_on_completion = type_properties.get("waitOnCompletion", True) + if approximations: + # Stamp the approximations onto the activity so the bundler can + # surface them in SETUP.md. + base_kwargs = {**base_kwargs, "parameter_approximations": approximations} + return ExecutePipelineActivity( **base_kwargs, pipeline_name=pipeline_name, parameters=parameters, wait_on_completion=wait_on_completion, ) + + +def _resolve_execute_pipeline_parameter( + name: str, + value: Any, + context: TranslationContext, + approximations: list[dict[str, str]], +) -> str | None: + """Resolves a single ExecutePipeline parameter or drops it with a SETUP note. + + C-09 (VAREX-001): when the value resolves to ``notebook_code`` the + parameter cannot ride through ``job_parameters`` as a literal Python + source string -- the sub-job's widget would receive the code text. + Drop the parameter and record an approximation so the bundler surfaces + it in SETUP.md for the user to supply manually. + """ + if value is None: + return "" + result = resolve_expression(value, context) + if result is None: + # Fallback to the legacy resolve_field for plain strings / dicts. + return resolve_field(value, context) + if result.kind in ("literal", "dab_ref"): + return result.value + if result.kind == "notebook_code": + raw = value + if isinstance(value, dict) and "value" in value: + raw = value["value"] + approximations.append( + { + "widget_name": name, + "raw_expression": str(raw), + "replacement": "", + "note": ( + "ExecutePipeline parameter dropped: the value resolves to a " + "notebook_code expression which cannot ride through DAB " + "job_parameters as a literal. Supply manually in SETUP.md or " + "synthesise a generator task that publishes a task value." + ), + } + ) + return None + return None diff --git a/src/orchestra/translator/activity_translators/for_each.py b/src/orchestra/translator/activity_translators/for_each.py index 54146b3..f13b655 100644 --- a/src/orchestra/translator/activity_translators/for_each.py +++ b/src/orchestra/translator/activity_translators/for_each.py @@ -35,8 +35,27 @@ def translate( items_raw = type_properties.get("items") expr_result = resolve_expression(items_raw, context) if items_raw is not None else None + inputs_bridge_notebook_code: str | None = None + inputs_bridge_notebook_imports: list[str] = [] + inputs_bridge_required_parameters: dict[str, str] = {} if expr_result is not None and expr_result.kind in ("dab_ref", "literal"): items_expression = expr_result.value + elif expr_result is not None and expr_result.kind == "notebook_code": + # C-31 (CF4-001): the preparer used to construct a bare + # TranslationContext() and re-resolve the items expression on the + # JSON-reload path, but ``variable_cache`` is empty there so the + # bridge never fired and DAB rejected the raw @split(...) call. + # Capture the resolved notebook_code here while the full context + # is available; the preparer reads it from these IR fields. + if isinstance(items_raw, dict) and items_raw.get("type") == "Expression": + items_expression = items_raw.get("value", "") + elif isinstance(items_raw, str): + items_expression = items_raw + else: + items_expression = "" + inputs_bridge_notebook_code = expr_result.value + inputs_bridge_notebook_imports = list(expr_result.imports) + inputs_bridge_required_parameters = dict(expr_result.required_parameters) else: # Fallback: extract raw string if isinstance(items_raw, dict) and items_raw.get("type") == "Expression": @@ -64,6 +83,8 @@ def translate( registry=context.registry, variable_cache=context.variable_cache, variable_value_cache=context.variable_value_cache, + global_parameters=context.global_parameters, + linked_service_parameters=context.linked_service_parameters, ) inner_activities, _ = translate_activities_fn(child_adf_activities, child_context, definitions) @@ -72,6 +93,9 @@ def translate( items_expression=items_expression, inner_activities=inner_activities, concurrency=batch_count, + inputs_bridge_notebook_code=inputs_bridge_notebook_code, + inputs_bridge_notebook_imports=inputs_bridge_notebook_imports, + inputs_bridge_required_parameters=inputs_bridge_required_parameters, ) return foreach_activity, context diff --git a/src/orchestra/translator/activity_translators/if_condition.py b/src/orchestra/translator/activity_translators/if_condition.py index c450f1d..86db2ff 100644 --- a/src/orchestra/translator/activity_translators/if_condition.py +++ b/src/orchestra/translator/activity_translators/if_condition.py @@ -11,7 +11,11 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, IfConditionActivity, TranslationContext -from flowx.parser.expression_parser import resolve_expression +from flowx.translator.activity_translators.resolve import ( + BridgeRequest, + lower_to_bridge, + merge_bridge_requests, +) # --------------------------------------------------------------------------- # ADF comparison function -> Databricks condition_task op mapping @@ -69,7 +73,7 @@ def translate( type_properties = activity.type_properties or {} expression_raw = type_properties.get("expression", {}) - op, left, right = _parse_condition(expression_raw, context) + op, left, right, bridge = _parse_condition(expression_raw, context) if_true_activities: list[Activity] = [] if_true_adf = activity.if_true_activities or [] @@ -81,6 +85,14 @@ def translate( if translate_activities_fn and if_false_adf: if_false_activities, _ = translate_activities_fn(if_false_adf, context, definitions) + bridge_kwargs: dict[str, Any] = {} + if bridge is not None: + bridge_kwargs = { + "bridge_notebook_code": bridge.notebook_code, + "bridge_notebook_imports": list(bridge.notebook_imports), + "bridge_required_parameters": dict(bridge.required_parameters), + } + if_activity = IfConditionActivity( **base_kwargs, op=op, @@ -88,20 +100,35 @@ def translate( right=right, if_true_activities=if_true_activities, if_false_activities=if_false_activities, + **bridge_kwargs, ) return if_activity, context -def _parse_condition(expression: dict[str, Any] | str, context: TranslationContext) -> tuple[str, str, str]: - """Parses an ADF IfCondition expression into ``(op, left, right)``. +_BRIDGE_TASK_VALUE_KEY = "result" + + +def _parse_condition( + expression: dict[str, Any] | str, context: TranslationContext +) -> tuple[str, str, str, BridgeRequest | None]: + """Parses an ADF IfCondition expression into ``(op, left, right, bridge)``. + + C-07 (CF-iter2-001 / CF-iter2-003 / VAREX-003): when an operand + resolves to ``notebook_code`` (e.g. ``@empty(X)``, ``@toUpper(...)``), + we package the code into a :class:`BridgeRequest` so the preparer can + emit a hidden SetVariable task whose value drives the + condition_task. The condition operand becomes the bridge's task + value reference. Args: expression: Raw ADF expression dict or string. context: Translation context for resolving variables. Returns: - Tuple of ``(databricks_op, left_operand, right_operand)``. + ``(databricks_op, left_operand, right_operand, bridge_request)`` + where ``bridge_request`` is non-None when the condition required + lowering to a notebook task. """ expr_str = "" if isinstance(expression, dict): @@ -117,9 +144,9 @@ def _parse_condition(expression: dict[str, Any] | str, context: TranslationConte inner_op_name = m_not.group(1).lower() op = _NEGATE_OP_MAP.get(inner_op_name, "NOT_EQUAL") args = _split_args(m_not.group(2).strip()) - left = _resolve_operand(args[0], context) if len(args) > 0 else "" - right = _resolve_operand(args[1], context) if len(args) > 1 else "" - return op, left, right + left, left_bridge = _resolve_operand(args[0], context) if len(args) > 0 else ("", None) + right, right_bridge = _resolve_operand(args[1], context) if len(args) > 1 else ("", None) + return op, left, right, merge_bridge_requests(left_bridge, right_bridge) m = _COMPARISON_RE.match(expr_str.strip()) if m: @@ -128,65 +155,159 @@ def _parse_condition(expression: dict[str, Any] | str, context: TranslationConte if adf_op == "not": inner = m.group(2).strip() - resolved = _resolve_operand(inner, context) - return "NOT_EQUAL", resolved, "" + resolved, bridge = _resolve_operand(inner, context) + # C-15 (CF3-003 / VAREX3-004): when the operand bridges to a + # Python bool task value, compare against 'False' (not '') so + # the IfCondition can actually evaluate to FALSE. + right_operand = "False" if bridge is not None else "" + return "NOT_EQUAL", resolved, right_operand, bridge args = _split_args(m.group(2).strip()) - left = _resolve_operand(args[0], context) if len(args) > 0 else "" - right = _resolve_operand(args[1], context) if len(args) > 1 else "" - return op, left, right - - # Fallback: treat the whole expression as a truthy check - resolved = _resolve_operand(expr_str, context) - return "NOT_EQUAL", resolved, "0" - - -def _resolve_operand(operand: str, context: TranslationContext) -> str: - """Converts an ADF expression operand to a Databricks task value reference. - - Examples:: - - activity('Lookup').output.firstRow.cnt - -> {{tasks.Lookup.values.cnt}} - - activity('Lookup').output.value - -> {{tasks.Lookup.values.result}} - - 0 -> 0 (literal) - 'active' -> active (string literal) - null -> "" (null literal) + left, left_bridge = _resolve_operand(args[0], context) if len(args) > 0 else ("", None) + right, right_bridge = _resolve_operand(args[1], context) if len(args) > 1 else ("", None) + return op, left, right, merge_bridge_requests(left_bridge, right_bridge) + + # Fallback: treat the whole expression as a truthy check. C-07: route + # through the bridge path when the expression is an ADF function call + # so the operand ends up as a real task-value reference rather than + # the legacy NOT_EQUAL '0' against a raw expression string. + resolved, bridge = _resolve_operand(expr_str, context) + if bridge is not None: + return "NOT_EQUAL", _bridge_task_value_placeholder(), "False", bridge + # C-15 (CF3-003 / VAREX3-004): when the truthy operand resolves to a + # task-value ref backed by a SetVariable that writes a Python bool + # (e.g. a previously-cached @variables('continue') with bridge-set + # value), compare against 'False' so the legacy truthy path doesn't + # silently invert behaviour. Detected by the presence of a + # __BRIDGE__:: placeholder or the lowercase 'true'/'false' literal + # body of the upstream SetVariable. + if isinstance(resolved, str) and "__BRIDGE__" in resolved: + return "NOT_EQUAL", resolved, "False", None + # C-43 (CF5-001 / LSC5-001): when the operand is a known-Boolean + # variable that resolves to a parent-job task-value ref + # (``{{tasks._init_X.values.X}}``), prefer recomputing the boolean + # locally via a BridgeRequest, mirroring the Switch path. Without this + # an inner-ForEach IfCondition references a task that lives only in the + # parent job; the bundler then blanks the operand to '' and + # NOT_EQUAL('', '0') is always TRUE, running the true branch + # unconditionally with no SETUP.md signal. The bridge keeps the + # operand local so it survives the dangling-ref safety net. + if _operand_is_known_boolean(expr_str, context): + bridge = _boolean_variable_bridge(expr_str, resolved, context) + if bridge is not None: + return "NOT_EQUAL", _bridge_task_value_placeholder(), "False", bridge + # C-32 (CF4-002): compare against lowercase ``'false'`` (matching + # C-21 SetVariable rendering) instead of the legacy ``'0'`` — the + # latter is always true for a Boolean-string operand so the false + # branch becomes dead code. + return "NOT_EQUAL", resolved, "false", None + return "NOT_EQUAL", resolved, "0", None + + +def _boolean_variable_bridge(expr: str, resolved: str, context: TranslationContext) -> BridgeRequest | None: + """Builds a local-recompute BridgeRequest for a Boolean-variable operand. + + C-43 (CF5-001): the bridge re-derives the boolean inside whatever job + the IfCondition lands in (parent or split-out inner ForEach job), so + the condition operand is a *local* task value rather than a parent-job + ref the bundler would blank. The recomputed value is the variable's + seeded literal default (``true``/``false``); when no literal default is + cached the caller falls back to the in-place ``'false'`` comparison. + + Returns ``None`` when there is no literal default to recompute from + (e.g. the variable is set dynamically), leaving the legacy path intact. + """ + expr = expr.strip() + if expr.startswith("@"): + expr = expr[1:] + var_match = re.match(r"variables\(\s*'([^']+)'\s*\)\s*$", expr, re.IGNORECASE) + if not var_match: + return None + var_name = var_match.group(1) + literal = context.get_variable_default_literal(var_name) + if literal is None or literal.lower() not in ("true", "false"): + return None + python_bool = "True" if literal.lower() == "true" else "False" + return BridgeRequest(notebook_code=python_bool) + + +def _operand_is_known_boolean(expr: str, context: TranslationContext) -> bool: + """Returns True when *expr* references a Boolean-typed variable / parameter. + + Inspects ``context.variable_value_cache`` (populated by C-05 init + SetVariable activities with lowercase ``'true'/'false'`` defaults), the + declared ``context.variable_types`` map (C-41), and bare + ``@pipeline().parameters.`` references when the context carries + Boolean type hints. When the type is known to be Boolean we return + True so the IfCondition fallback uses ``'false'`` as the right operand. + """ + expr = expr.strip() + if expr.startswith("@"): + expr = expr[1:] + # @variables('X') -> look up the cached lowercase value + var_match = re.match(r"variables\(\s*'([^']+)'\s*\)\s*$", expr, re.IGNORECASE) + if var_match: + var_name = var_match.group(1) + cached = context.get_variable_dab_ref(var_name) + if isinstance(cached, str) and cached.lower() in ("true", "false"): + return True + # C-41 (CF5-001): a Boolean variable seeded only by a literal + # default init task never populates variable_value_cache as a + # dab_ref, so fall back to its declared ADF type. + declared = context.get_variable_type(var_name) + if isinstance(declared, str) and declared.lower() in ("boolean", "bool"): + return True + # @pipeline().parameters.X -- without parameter type metadata we + # cannot prove Booleanness; return False conservatively. + return False + + +def _bridge_task_value_placeholder() -> str: + """Sentinel left-operand the preparer rewrites to the bridge task value.""" + return f"__BRIDGE__::{_BRIDGE_TASK_VALUE_KEY}" + + +def _resolve_operand(operand: str, context: TranslationContext) -> tuple[str, BridgeRequest | None]: + """Converts an ADF expression operand to a Databricks task value reference + or a :class:`BridgeRequest` when the operand requires a bridge task. Args: operand: A single operand string from the parsed condition. context: Translation context for resolving variables. Returns: - A DAB dynamic value reference or literal string. + ``(operand, bridge_request)`` where ``bridge_request`` is None for + literals / DAB refs. """ operand = operand.strip() if operand.lower() == "null": - return "" + return "", None if operand.startswith("'") and operand.endswith("'"): - return operand[1:-1] + return operand[1:-1], None if operand.lstrip("-").replace(".", "", 1).isdigit(): - return operand + return operand, None inner = _unwrap_functions(operand) - # Try unified expression resolution with @ prefix - result = resolve_expression("@" + inner, context) - if result is not None and result.kind in ("dab_ref", "literal"): - return result.value + sub_expr = inner if inner.startswith("@") else "@" + inner + operand_value, bridge = lower_to_bridge(sub_expr, context) + if operand_value is not None: + return operand_value, None + if bridge is not None: + return _bridge_task_value_placeholder(), bridge if inner != operand: - result = resolve_expression("@" + operand, context) - if result is not None and result.kind in ("dab_ref", "literal"): - return result.value - - return operand + sub_expr_full = operand if operand.startswith("@") else "@" + operand + operand_value, bridge = lower_to_bridge(sub_expr_full, context) + if operand_value is not None: + return operand_value, None + if bridge is not None: + return _bridge_task_value_placeholder(), bridge + + return operand, None def _unwrap_functions(expr: str) -> str: diff --git a/src/orchestra/translator/activity_translators/lookup.py b/src/orchestra/translator/activity_translators/lookup.py index 3620df8..674ad00 100644 --- a/src/orchestra/translator/activity_translators/lookup.py +++ b/src/orchestra/translator/activity_translators/lookup.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from typing import Any from flowx.models.adf_ast import AdfActivity, AdfDefinitions @@ -9,6 +10,76 @@ from flowx.translator.activity_translators.resolve import resolve_field +def _dataset_parameter_scope(activity: AdfActivity, context: TranslationContext) -> dict[str, str]: + """Resolve the Lookup dataset reference's ``parameters`` binding. + + C-47 (LSC5-001): a file-source dataset's ``folderPath`` / ``fileName`` + can reference its own parameters via ``dataset().X``. The Lookup's + ``typeProperties.dataset.parameters`` block binds each dataset parameter + (e.g. ``digitalCase``) to a pipeline-scoped value (e.g. + ``@pipeline().parameters.digitalCaseCode``). Resolve each binding + against the pipeline ``TranslationContext`` so ``dataset().digitalCase`` + can be substituted with the resolved ``{{job.parameters.X}}`` ref / literal. + + Returns a ``{dataset_param_name: resolved_value}`` map (empty when the + reference carries no parameters). + """ + type_props = activity.type_properties or {} + ref = type_props.get("dataset") + if not isinstance(ref, dict): + return {} + params = ref.get("parameters") + if not isinstance(params, dict): + return {} + return {name: resolve_field(value, context) for name, value in params.items()} + + +def _substitute_dataset_refs(value: Any, scope: dict[str, str]) -> Any: + """Replace ``dataset().X`` occurrences in *value* with the resolved binding. + + C-47 (LSC5-001): without this, ``folderPath`` of + ``@toLower(dataset().digitalCase)`` and ``fileName`` of + ``@dataset().fileName`` pass through verbatim and the code generator + bakes a literal broken ``abfss://.../@toLower(dataset().digitalCase)`` + default path that ``spark.read`` cannot load. + + Only string values carrying a ``dataset().`` reference are rewritten; + everything else is returned unchanged. + """ + if not isinstance(value, str) or "dataset(" not in value: + return value + result = value + for name, resolved in scope.items(): + # dataset().X and dataset()['X'] / dataset()["X"] forms. + result = re.sub( + r"dataset\(\)\s*(?:\.\s*" + re.escape(name) + r"\b|\[\s*['\"]" + re.escape(name) + r"['\"]\s*\])", + resolved, + result, + ) + return result + + +def _unwrap_expression(value: Any) -> Any: + """Unwrap a ``{"value": X, "type": "Expression"}`` dict-wrapper. + + C-37 (LSC4-001): folder_path / file_name on file-source datasets + sometimes ship as the ADF expression dict shape. Without unwrapping, + downstream code (notably ``_assemble_file_lookup_source_path``) calls + ``.strip('/')`` on the dict and crashes with AttributeError, which + has bundled 4 pipelines into "empty bundle directory" outcomes. + """ + if isinstance(value, dict) and "value" in value and value.get("type") == "Expression": + return value["value"] + return value + + +# File-source dataset types that a Lookup can read directly. Keeping +# this list local avoids tugging the broader copy translator in. +_FILE_DATASET_TYPES: frozenset[str] = frozenset( + {"Json", "Parquet", "DelimitedText", "Avro", "Orc", "Excel", "Xml", "Binary"} +) + + def translate( activity: AdfActivity, base_kwargs: dict[str, Any], @@ -39,6 +110,59 @@ def translate( first_row_only = type_properties.get("firstRowOnly", True) + # Resolve the Lookup's dataset reference (lookup-translator-ignores-dataset-reference): + # typeProperties.dataset is the canonical place for ADF; activity.inputs + # is the legacy fall-back used by the loader for flattened activity shapes. + dataset_ref = _resolve_lookup_dataset(activity, definitions) + if dataset_ref is not None: + dataset_props = dataset_ref["properties"] + dataset_type = dataset_ref["type"] + type_props = dataset_props.get("typeProperties") or {} + location = type_props.get("location") or {} + if dataset_type in _FILE_DATASET_TYPES: + source_properties.setdefault("dataset_type", dataset_type) + # Stash the dataset path components so the code generator can + # build the right spark.read call. Avoid pulling in the full + # copy translator dataset-path machinery — we only need the + # raw container + folder + filename to surface to the user. + # C-37 (LSC4-001): unwrap any ADF expression dict shapes so + # downstream code can treat these as plain strings. + container = _unwrap_expression( + location.get("container") or location.get("fileSystem") or location.get("bucketName") + ) + folder = _unwrap_expression(location.get("folderPath")) + filename = _unwrap_expression(location.get("fileName")) + # C-47 (LSC5-001): substitute dataset().X param refs using the + # Lookup dataset reference's parameter bindings, then resolve the + # result so the path default is a real literal / interpolated + # {{job.parameters.X}} string rather than a verbatim dataset() + # expression the code generator would bake into a broken path. + ds_scope = _dataset_parameter_scope(activity, context) + if ds_scope: + if isinstance(folder, str) and folder: + folder = resolve_field(_substitute_dataset_refs(folder, ds_scope), context) + if isinstance(filename, str) and filename: + filename = resolve_field(_substitute_dataset_refs(filename, ds_scope), context) + if container: + source_properties.setdefault("container", container) + if folder: + source_properties.setdefault("folder_path", folder) + if filename: + source_properties.setdefault("file_name", filename) + # Forward type-specific format options (multiline, encoding, etc.) + # so the generator can pass them as spark.read.option(...). + format_settings = type_props.get("formatSettings") or {} + if isinstance(format_settings, dict): + for key in ("multiLineJson", "filePattern"): + if key in format_settings: + source_properties.setdefault(key, format_settings[key]) + # LSC3-005: surface the linked service URL when present so the + # generator can assemble the abfss:// path for AzureBlobFS / ADLS + # backed file datasets. + ls_url = dataset_props.get("linked_service_url") + if ls_url: + source_properties.setdefault("linked_service_url", ls_url) + return LookupActivity( **base_kwargs, source_type=source_type, @@ -46,3 +170,44 @@ def translate( first_row_only=first_row_only, source_query=source_query, ) + + +def _resolve_lookup_dataset( + activity: AdfActivity, + definitions: AdfDefinitions, +) -> dict[str, Any] | None: + """Resolves the Lookup's dataset reference to its full dataset record. + + Args: + activity: The ADF Lookup activity AST node. + definitions: Full ADF definitions for dataset lookup. + + Returns: + Dict with keys ``type`` and ``properties`` describing the bound + dataset, or ``None`` when no dataset is referenced. + """ + type_props = activity.type_properties or {} + ref = type_props.get("dataset") + dataset_name: str | None = None + if isinstance(ref, dict): + dataset_name = ref.get("referenceName") or ref.get("dataset", {}).get("referenceName") + if dataset_name is None and activity.inputs: + dataset_name = activity.inputs[0].reference_name + if dataset_name is None: + return None + # LSC3-005: ADF identifiers are case-insensitive; tolerate casing drift + # between the pipeline's dataset reference and the source JSON filename. + dataset = definitions.get_dataset(dataset_name) + if dataset is None: + return None + properties = dict(dataset.properties or {}) + # Thread linkedService typeProperties.url through onto the properties so + # the lookup notebook can assemble the abfss:// file path for file-source + # datasets where the URL is only known on the linked service. + linked_service = definitions.get_linked_service(dataset.linked_service_name) + if linked_service is not None: + ls_props = linked_service.properties or {} + ls_type_props = ls_props.get("typeProperties") if isinstance(ls_props, dict) else None + if isinstance(ls_type_props, dict) and "url" in ls_type_props: + properties.setdefault("linked_service_url", ls_type_props["url"]) + return {"type": dataset.type, "properties": properties} diff --git a/src/orchestra/translator/activity_translators/notebook.py b/src/orchestra/translator/activity_translators/notebook.py index 3fc5897..986de03 100644 --- a/src/orchestra/translator/activity_translators/notebook.py +++ b/src/orchestra/translator/activity_translators/notebook.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from typing import Any from flowx.models.adf_ast import AdfActivity, AdfDefinitions @@ -29,17 +30,38 @@ def translate( """ type_properties = activity.type_properties or {} - notebook_path = resolve_field(type_properties.get("notebookPath", ""), context) + # C-28 (NB-ITER4-001): when notebookPath is an ADF expression that lowers + # to notebook_code (e.g. @trim(json(...).notebook_path)), preserve the raw + # expression and mark the activity so the preparer emits a dispatch stub + # rather than inlining Python source as the workspace path. + notebook_path_raw = type_properties.get("notebookPath", "") + notebook_path, notebook_path_unresolved, notebook_path_expression = _resolve_notebook_path_field( + notebook_path_raw, context + ) raw_params = type_properties.get("baseParameters") or {} + libraries, unresolved_libraries = _resolve_libraries(type_properties.get("libraries"), context) # Resolve base_parameters at translate time so ADF expressions like # @variables('runTimestamp') are inlined to DAB refs while the full - # translation context (with variable_value_cache) is available. + # translation context (with variable_value_cache) is available. Any + # caveat notes the resolver emits (e.g. utcnow() approximations) are + # captured into parameter_approximations so the bundler can surface + # them in SETUP.md. resolved_params: dict[str, Any] = {} + approximations: list[dict[str, str]] = [] for key, value in raw_params.items(): result = resolve_expression(value, context) if result is not None and result.kind in ("dab_ref", "literal"): resolved_params[key] = result.value + for note in result.notes: + approximations.append( + { + "widget_name": key, + "raw_expression": _raw_expression_text(value), + "replacement": result.value, + "note": note, + } + ) else: # Keep original for downstream handling (notebook_code or unresolvable) resolved_params[key] = value @@ -47,5 +69,147 @@ def translate( return NotebookActivity( **base_kwargs, notebook_path=notebook_path, + notebook_path_unresolved=notebook_path_unresolved, + notebook_path_expression=notebook_path_expression, base_parameters=resolved_params, + libraries=libraries, + unresolved_libraries=unresolved_libraries, + parameter_approximations=approximations, ) + + +def _resolve_notebook_path_field( + value: Any, + context: TranslationContext, +) -> tuple[str, bool, str | None]: + """Resolves the ADF ``notebookPath`` field, preserving dynamic dispatch shapes. + + C-28 (NB-ITER4-001): the legacy ``resolve_field`` returns ``result.value`` + for every kind including ``notebook_code``, which means an ADF expression + like ``@trim(json(activity('cfg').output.firstRow).notebook_path)`` ends up + as Python source text in ``notebook_path``. Bundle SETUP.md then + mis-documents the source as a workspace path. + + Returns ``(notebook_path, notebook_path_unresolved, raw_expression)``. + When the expression cannot be reduced to a workspace path we set + ``notebook_path_unresolved=True`` so the preparer emits a dispatch-stub + notebook and SETUP.md flags the dynamic dispatch. + """ + if value is None: + return "", False, None + if isinstance(value, dict): + if value.get("type") == "Expression" and "value" in value: + raw_text = str(value["value"]) + result = resolve_expression(value, context) + if result is not None and result.kind in ("literal", "dab_ref"): + return result.value, False, None + return "", True, raw_text + return resolve_field(value, context), False, None + if isinstance(value, str): + if value.startswith("@"): + result = resolve_expression(value, context) + if result is not None and result.kind in ("literal", "dab_ref"): + return result.value, False, None + return "", True, value + return value, False, None + return resolve_field(value, context), False, None + + +def _raw_expression_text(value: Any) -> str: + """Returns the original ADF expression text from a base_parameter value.""" + if isinstance(value, dict) and value.get("type") == "Expression" and "value" in value: + return str(value["value"]) + return str(value) + + +# Library entry keys that may carry ADF expressions (jar/whl paths, +# maven coordinates with @concat, etc). PyPI uses ``package`` and CRAN +# uses ``package``; we walk all of them through the resolver and only +# emit the entry when every expression resolves to a clean literal/dab_ref. +_LIBRARY_VALUE_KEYS: tuple[str, ...] = ("jar", "whl", "egg", "requirements") + + +_GLOBAL_PARAM_REF_RE = re.compile(r"pipeline\(\s*\)\.globalParameters\.(\w+)", re.IGNORECASE) +_PIPELINE_PARAM_REF_RE = re.compile(r"pipeline\(\s*\)\.parameters\.(\w+)", re.IGNORECASE) +_VARIABLE_REF_RE = re.compile(r"variables\(\s*'([^']+)'\s*\)", re.IGNORECASE) + + +def _extract_missing_identifiers(expression_text: str, context: TranslationContext) -> list[str]: + """Returns identifier names referenced by *expression_text* that aren't + bound in *context*. + + Helps surface concrete root causes in SETUP.md when a library expression + fails to resolve (e.g. ``@concat(...proj4jLibFileName)`` referencing a + global parameter that the factory doesn't declare). + """ + missing: list[str] = [] + for match in _GLOBAL_PARAM_REF_RE.finditer(expression_text): + name = match.group(1) + if context.get_global_parameter(name) is None and name not in missing: + missing.append(name) + for match in _PIPELINE_PARAM_REF_RE.finditer(expression_text): + name = match.group(1) + if name not in missing: + missing.append(name) + for match in _VARIABLE_REF_RE.finditer(expression_text): + name = match.group(1) + if context.get_variable_task_key(name) is None and name not in missing: + missing.append(name) + return missing + + +def _resolve_libraries( + libraries: list[dict[str, Any]] | None, + context: TranslationContext, +) -> tuple[list[dict[str, Any]] | None, list[dict[str, Any]]]: + """Pipes library descriptor values through the expression resolver. + + Library entries whose ``jar``/``whl``/``egg``/``requirements`` value is + an ADF expression that resolves cleanly to a literal get the literal + substituted in place. Entries whose expression is unresolved (e.g. + references a missing globalParameter) are passed through unchanged + so downstream bundler tooling can flag them in SETUP.md. + + C-30 (NB-ITER4-003): also returns a list of unresolved library entries + so the preparer can render an ``Unresolved libraries`` section in + SETUP.md instead of shipping a broken ``@concat(...)`` literal jar path + that the cluster cannot install. + """ + unresolved: list[dict[str, Any]] = [] + if not libraries: + return libraries, unresolved + + resolved: list[dict[str, Any]] = [] + for lib in libraries: + if not isinstance(lib, dict): + resolved.append(lib) + continue + resolved_entry: dict[str, Any] = {} + for key, value in lib.items(): + if key in _LIBRARY_VALUE_KEYS and isinstance(value, (str, dict)): + result = resolve_expression(value, context) + # C-13 (NB-ITER3-004): accept both literal and dab_ref so a + # jar path like @pipeline().parameters.libName collapses to + # {{job.parameters.libName}} (symmetric with custom_tags + # resolution in _resolve_ls_parameters). + if result is not None and result.kind in ("literal", "dab_ref"): + resolved_entry[key] = result.value + else: + expression_text = _raw_expression_text(value) + resolved_entry[key] = value + # Only surface library entries whose value carried an + # ADF expression (starts with ``@``). Bare literal + # paths that already resolved successfully don't need a + # SETUP.md callout. + if isinstance(expression_text, str) and expression_text.startswith("@"): + unresolved.append( + { + "type": key, + "expression": expression_text, + "missing": _extract_missing_identifiers(expression_text, context), + } + ) + else: + resolved_entry[key] = value + resolved.append(resolved_entry) + return resolved, unresolved diff --git a/src/orchestra/translator/activity_translators/resolve.py b/src/orchestra/translator/activity_translators/resolve.py index fd6f496..7f8d434 100644 --- a/src/orchestra/translator/activity_translators/resolve.py +++ b/src/orchestra/translator/activity_translators/resolve.py @@ -2,12 +2,92 @@ from __future__ import annotations +from dataclasses import dataclass, field from typing import Any -from flowx.models.ir import TranslationContext +from flowx.models.ir import ExpressionResult, TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string +@dataclass(slots=True) +class BridgeRequest: + """Carrier for a notebook_code expression that must run as a bridge task. + + C-07 (CF-iter2-001 / CF-iter2-003 / VAREX-003): when an operand of an + IfCondition / Switch condition_task resolves to ``notebook_code`` (e.g. + ``@empty(...)``, ``@toUpper(coalesce(...))``), the preparer must + synthesise a hidden SetVariable task ahead of the condition and + rewrite the operand to point at the bridge task's value. Returning a + structured request keeps the translator free of preparer-side + concerns. + """ + + notebook_code: str + notebook_imports: list[str] = field(default_factory=list) + required_parameters: dict[str, str] = field(default_factory=dict) + + +def lower_to_bridge(value: Any, context: TranslationContext) -> tuple[str | None, BridgeRequest | None]: + """Lowers *value* to a condition-task-safe operand or a BridgeRequest. + + Returns: + ``(operand_value, bridge_request)`` -- exactly one of which is + populated. When ``operand_value`` is a string, it's a literal + or ``{{...}}`` DAB ref that can be dropped directly into + ``condition_task.left`` / ``.right``. When ``bridge_request`` is + populated, the caller must emit a hidden SetVariable task that + runs the notebook code and reference its task value in the + operand. When both are None the expression is unresolvable. + """ + if value is None: + return None, None + result = resolve_expression(value, context) + if result is None: + return None, None + if result.kind in ("literal", "dab_ref"): + return result.value, None + if result.kind == "notebook_code": + return None, BridgeRequest( + notebook_code=result.value, + notebook_imports=list(result.imports), + required_parameters=dict(result.required_parameters), + ) + return None, None + + +def merge_bridge_requests(*requests: BridgeRequest | None) -> BridgeRequest | None: + """Combines several bridge requests into one merged Python expression. + + The bridges are joined with ``and`` so call-sites that compose two + operand-level bridges (e.g. ``@and(empty(X), empty(Y))``) produce a + single bridge task with a boolean truthiness result. Returns ``None`` + when no non-None requests are supplied. + """ + populated = [r for r in requests if r is not None] + if not populated: + return None + if len(populated) == 1: + return populated[0] + expression = " and ".join(f"({r.notebook_code})" for r in populated) + imports: list[str] = [] + required: dict[str, str] = {} + for req in populated: + for imp in req.notebook_imports: + if imp not in imports: + imports.append(imp) + required.update(req.required_parameters) + return BridgeRequest( + notebook_code=expression, + notebook_imports=imports, + required_parameters=required, + ) + + +def expression_kind(value: Any, context: TranslationContext) -> ExpressionResult | None: + """Convenience wrapper that returns the raw ExpressionResult for *value*.""" + return resolve_expression(value, context) + + def resolve_field(value: Any, context: TranslationContext) -> str: """Resolves a field value that may contain an ADF expression. diff --git a/src/orchestra/translator/activity_translators/set_variable.py b/src/orchestra/translator/activity_translators/set_variable.py index 6ddc3e1..6f127ea 100644 --- a/src/orchestra/translator/activity_translators/set_variable.py +++ b/src/orchestra/translator/activity_translators/set_variable.py @@ -9,6 +9,51 @@ from flowx.parser.expression_parser import resolve_expression +def _unwrap_return_value_pairs(value: Any) -> Any: + """Unwrap a Set Pipeline Return Value list-of-pairs to a resolvable value. + + A ``pipelineReturnValue`` value is shaped as a list of + ``{'key': ..., 'value': }`` + entries. ADF's expression dicts here use the ``content`` key (not + ``value``). We normalise a single pair's inner value into the + ``{'type': 'Expression', 'value': ...}`` shape (or a bare literal) that + :func:`resolve_expression` understands, so the inner ``@variables('X')`` + reference is preserved instead of stringifying the whole list. + + When the list is empty or carries more than one pair (no single + canonical result), the original value is returned unchanged so the + legacy unresolved/blanking path still applies. + """ + if not isinstance(value, list) or len(value) != 1: + return value + entry = value[0] + if not isinstance(entry, dict) or "value" not in entry: + return value + inner = entry["value"] + if isinstance(inner, dict) and inner.get("type") == "Expression" and "content" in inner: + return {"type": "Expression", "value": inner["content"]} + if isinstance(inner, dict) and inner.get("type") == "Expression" and "value" in inner: + return inner + return inner + + +def _raw_expression_text(value: Any) -> str: + """Returns the original ADF expression text for *value*.""" + if isinstance(value, dict) and value.get("type") == "Expression" and "value" in value: + return str(value["value"]) + return "" if value is None else str(value) + + +def _is_adf_expression(value: Any) -> bool: + """Returns True when *value* is an ADF expression we can't pass through verbatim.""" + if isinstance(value, dict) and value.get("type") == "Expression" and "value" in value: + inner = value["value"] + return isinstance(inner, str) and inner.startswith("@") + if isinstance(value, str): + return value.startswith("@") + return False + + def translate( activity: AdfActivity, base_kwargs: dict[str, Any], @@ -32,21 +77,49 @@ def translate( variable_name = type_properties.get("variableName", "") value_raw = type_properties.get("value", "") + # C-42 (VAREX5-001): a Set Pipeline Return Value activity carries a + # list of {key, value} pairs (e.g. + # [{'key': 'result', 'value': {'type': 'Expression', + # 'content': "@variables('executionOutputs')"}}]). The legacy path + # fails _is_adf_expression and stringifies the whole list, which the + # bundler then blanks. The inner expression is resolvable, so unwrap a + # single pair's value and route it through the normal resolution + # pipeline instead of losing the reference. + value_raw = _unwrap_return_value_pairs(value_raw) + expr_result = resolve_expression(value_raw, context) required_parameters: dict[str, str] = {} + raw_expression_text = _raw_expression_text(value_raw) if expr_result is not None: variable_value = expr_result.value value_kind = expr_result.kind notebook_code = expr_result.value if expr_result.kind == "notebook_code" else None notebook_imports = list(expr_result.imports) if expr_result.kind == "notebook_code" else [] required_parameters = dict(expr_result.required_parameters) + elif _is_adf_expression(value_raw): + # C-33 (VAREX4-001 / CF4-003): when the value is an ADF expression + # the resolver couldn't handle (e.g. a nested function call we + # don't model), do NOT stamp value_kind='literal' with the raw + # @concat text — that ships uninterpretable Python source through + # SETUP.md. Blank the value and mark it unresolved so the bundler + # emits a manual_variable_init SetupTask the user can act on. + variable_value = "" + value_kind = "unresolved" + notebook_code = None + notebook_imports = [] else: # Fallback: unwrap expression-type dicts to at least preserve the string if isinstance(value_raw, dict) and value_raw.get("type") == "Expression": variable_value = value_raw.get("value", "") elif isinstance(value_raw, str): variable_value = value_raw + elif isinstance(value_raw, bool): + # VAREX3-002: render Python bool as lowercase 'true'/'false' so + # downstream ADF comparisons like @equals(variables('X'), true) + # match consistently. ``str(True)`` would emit 'True' and silently + # invert the comparison. + variable_value = "true" if value_raw else "false" else: variable_value = str(value_raw) value_kind = "literal" @@ -61,6 +134,7 @@ def translate( notebook_code=notebook_code, notebook_imports=notebook_imports, required_parameters=required_parameters, + raw_expression=raw_expression_text if value_kind == "unresolved" else None, ) # Register variable -> task_key mapping in context. diff --git a/src/orchestra/translator/activity_translators/spark_python.py b/src/orchestra/translator/activity_translators/spark_python.py index 8dc8892..b0b230a 100644 --- a/src/orchestra/translator/activity_translators/spark_python.py +++ b/src/orchestra/translator/activity_translators/spark_python.py @@ -55,6 +55,7 @@ def translate( python_file = resolve_field(type_properties.get("pythonFile", ""), context) raw_parameters = type_properties.get("parameters") or [] + libraries = type_properties.get("libraries") parameters = [_resolve_parameter(p, context) for p in raw_parameters] @@ -62,4 +63,5 @@ def translate( **base_kwargs, python_file=python_file, parameters=parameters, + libraries=libraries, ) diff --git a/src/orchestra/translator/activity_translators/switch.py b/src/orchestra/translator/activity_translators/switch.py index 095840d..470407c 100644 --- a/src/orchestra/translator/activity_translators/switch.py +++ b/src/orchestra/translator/activity_translators/switch.py @@ -7,29 +7,45 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, SwitchActivity, SwitchCase, TranslationContext from flowx.parser.adf_loader import parse_activity -from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string -from flowx.translator.activity_translators.resolve import resolve_field +from flowx.parser.expression_parser import resolve_interpolated_string +from flowx.translator.activity_translators.resolve import ( + BridgeRequest, + lower_to_bridge, + resolve_field, +) +_BRIDGE_PLACEHOLDER = "__BRIDGE__::result" -def _resolve_on_expression(on_expression: str, context: TranslationContext) -> str: - """Resolves the ``on`` expression to a DAB dynamic value ref. + +def _resolve_on_expression(on_expression: str, context: TranslationContext) -> tuple[str, BridgeRequest | None]: + """Resolves the ``on`` expression to a DAB dynamic value ref or a bridge request. + + C-07 (CF-iter2-001 / CF-iter2-003): when the expression involves an + ADF function call (e.g. ``@toUpper(coalesce(...))``), lower it to a + bridge SetVariable task instead of shipping the raw ADF string into + the condition_task operand. Args: on_expression: Raw ADF on-expression string. context: Translation context for resolving variables. Returns: - Resolved DAB ref string, or the original if unresolvable. + Tuple of ``(resolved_value_or_placeholder, bridge_request_or_None)``. """ if "@{" in on_expression: - return resolve_interpolated_string(on_expression, context) + return resolve_interpolated_string(on_expression, context), None if on_expression.startswith("@"): - result = resolve_expression(on_expression, context) - if result is not None and result.kind in ("dab_ref", "literal"): - return result.value + operand, bridge = lower_to_bridge(on_expression, context) + if operand is not None: + return operand, None + if bridge is not None: + return _BRIDGE_PLACEHOLDER, bridge + # Resolution failed entirely -- preserve the raw string so the + # preparer can flag it via SETUP.md. + return on_expression, None - return on_expression + return on_expression, None def translate( @@ -63,7 +79,7 @@ def translate( else: on_expression_raw = str(on_raw) if on_raw else "" - on_expression = _resolve_on_expression(on_expression_raw, context) + on_expression, bridge = _resolve_on_expression(on_expression_raw, context) cases: list[SwitchCase] = [] raw_cases = type_properties.get("cases", []) @@ -93,11 +109,20 @@ def translate( definitions, ) + bridge_kwargs: dict[str, Any] = {} + if bridge is not None: + bridge_kwargs = { + "bridge_notebook_code": bridge.notebook_code, + "bridge_notebook_imports": list(bridge.notebook_imports), + "bridge_required_parameters": dict(bridge.required_parameters), + } + switch_activity = SwitchActivity( **base_kwargs, on_expression=on_expression, cases=cases, default_activities=default_activities, + **bridge_kwargs, ) return switch_activity, context diff --git a/src/orchestra/translator/engine.py b/src/orchestra/translator/engine.py index 038e2d9..cb90617 100644 --- a/src/orchestra/translator/engine.py +++ b/src/orchestra/translator/engine.py @@ -8,6 +8,7 @@ import re from collections import defaultdict from dataclasses import asdict +from datetime import datetime from pathlib import Path from types import MappingProxyType from typing import Any, Callable @@ -48,6 +49,7 @@ from flowx.motifs.collapser import collapse_motifs from flowx.motifs.detector import detect_motifs from flowx.parser.adf_loader import classify_activity, load_adf_definitions +from flowx.parser.ir_rewriter import rewrite_pipeline_expressions from flowx.translator.activity_translators import ( append_variable, copy, @@ -84,29 +86,77 @@ } -def translate_pipeline(pipeline: AdfPipeline, definitions: AdfDefinitions) -> TranslationReport: +def translate_pipeline( + pipeline: AdfPipeline, + definitions: AdfDefinitions, + *, + motif_consolidations: dict[str, str] | None = None, +) -> TranslationReport: """Translates an ADF pipeline into a Databricks pipeline IR. Args: pipeline: Parsed ADF pipeline AST. definitions: Full ADF definitions for cross-referencing datasets, linked services, etc. + motif_consolidations: Optional mapping of ``motif_id`` -> + ``"keep"`` / ``"consolidate"`` answers gathered from the + adapter. When ``None`` (back-compat default) the translator + consolidates every detected motif. When provided, only + motifs whose id maps to ``"consolidate"`` are collapsed; the + rest remain as the original activity-by-activity translation. Returns: - :class:`TranslationReport` containing the translated :class:`Pipeline` - and any gaps encountered. + :class:`TranslationReport` containing the translated :class:`Pipeline`, + the list of detected motifs, and any gaps encountered. + + Notes: + After dispatching individual activities the translator runs + :func:`~flowx.parser.ir_rewriter.rewrite_pipeline_expressions` + over the whole IR so that ``@{...}`` ADF expressions embedded in + SQL bodies, REST payloads, dataset paths, and other string-typed + fields are rewritten through the same parser the per-activity + translators use. Tokens that cannot be resolved are recorded + as translation warnings instead of shipping into the bundle + verbatim. """ context = TranslationContext( activity_cache=MappingProxyType({}), registry=MappingProxyType(TRANSLATOR_REGISTRY), variable_cache=MappingProxyType({}), + global_parameters=MappingProxyType(dict(definitions.global_parameters)), ) + # C-41 (CF5-001): seed declared variable types so the IfCondition + # fallback can recognise Boolean variables that are backed only by a + # literal default init task (and thus never populate + # variable_value_cache). Without this a `continue`-style Boolean + # condition emits NOT_EQUAL(left, '0'), always true for a + # 'true'/'false' string, making the false branch dead code. + if pipeline.variables: + default_literals: dict[str, str] = {} + for name, var in pipeline.variables.items(): + default = var.default_value + if isinstance(default, bool): + default_literals[name] = "true" if default else "false" + elif isinstance(default, str) and default.lower() in ("true", "false"): + default_literals[name] = default.lower() + context = context.with_variable_types( + {name: var.type for name, var in pipeline.variables.items()}, + default_literals=default_literals, + ) + gaps: list[AgenticGap] = [] warnings: list[str] = [] + # C-05 (VAREX-002): synthesise init SetVariable tasks for pipeline + # variables carrying a defaultValue. This seeds variable_cache so + # downstream @variables('X') references resolve to the init task's + # value reference instead of falling back to a self-referential + # {{tasks.X.values.X}} dangler. + init_variable_activities, context = _build_variable_init_activities(pipeline, context) + translated_activities: list[Activity] = list(init_variable_activities) + sorted_activities = _topological_visit(pipeline.activities) - translated_activities: list[Activity] = [] deterministic_count = 0 agentic_count = 0 unsupported_count = 0 @@ -140,32 +190,53 @@ def translate_pipeline(pipeline: AdfPipeline, definitions: AdfDefinitions) -> Tr ) warnings.append(f"Activity '{adf_activity.name}' (type={adf_activity.type}) has no translation path.") - parameters: dict[str, Any] = {} + parameter_entries: list[dict[str, Any]] = [] if pipeline.parameters: for param_name, param_def in pipeline.parameters.items(): - parameters[param_name] = param_def.default_value + entry: dict[str, Any] = {"name": param_name, "type": param_def.type} + if param_def.default_value is not None: + entry["default"] = _coerce_parameter_default(param_def.default_value, param_def.type) + parameter_entries.append(entry) + + schedule = _compile_pipeline_schedule(pipeline, definitions) pipeline_ir = Pipeline( name=pipeline.name, - parameters=[{"name": param_name, "default": param_value} for param_name, param_value in parameters.items()] - if parameters - else None, + parameters=parameter_entries or None, tasks=translated_activities, tags={"source": "adf", "pipeline": pipeline.name}, + schedule=schedule, ) - # Motif detection and collapsing: scan for known multi-activity patterns - # and replace matched groups with single MotifActivity nodes. + # Whole-IR expression rewrite: catches @{...} tokens the per-activity + # translators didn't address (raw SQL WHERE clauses inside source_properties, + # REST request bodies, dataset folder paths, ...). Unresolved tokens are + # surfaced as translation warnings. + pipeline_ir = rewrite_pipeline_expressions(pipeline_ir, warnings=warnings) + + # Motif detection: scan for known multi-activity patterns. Collapsing is + # gated on the per-motif preference -- when *motif_consolidations* is + # ``None`` we preserve back-compat behaviour and collapse every detected + # motif; otherwise only motifs whose motif_id maps to ``"consolidate"`` are + # collapsed. detected_motifs = detect_motifs(pipeline, definitions) - if detected_motifs: - pipeline_ir = collapse_motifs(pipeline_ir, detected_motifs) - for motif in detected_motifs: + motifs_to_collapse = _filter_motifs_for_collapse(detected_motifs, motif_consolidations) + if motifs_to_collapse: + pipeline_ir = collapse_motifs(pipeline_ir, motifs_to_collapse) + for motif in motifs_to_collapse: logger.info( "Collapsed motif '%s': %d activities -> %s", motif.definition.display_name, len(motif.matched_activities), motif.definition.databricks_replacement, ) + for motif in detected_motifs: + if motif not in motifs_to_collapse: + logger.info( + "Detected motif '%s' left expanded: matched %d activities (user opted to keep)", + motif.definition.display_name, + len(motif.matched_activities), + ) return TranslationReport( pipeline=pipeline_ir, @@ -174,9 +245,32 @@ def translate_pipeline(pipeline: AdfPipeline, definitions: AdfDefinitions) -> Tr unsupported_count=unsupported_count, gaps=gaps, warnings=warnings, + detected_motifs=list(detected_motifs), ) +def _filter_motifs_for_collapse( + detected_motifs: list, + motif_consolidations: dict[str, str] | None, +) -> list: + """Returns the subset of detected motifs the caller asked to collapse. + + Args: + detected_motifs: Output of + :func:`flowx.motifs.detector.detect_motifs`. + motif_consolidations: Caller-supplied answers. ``None`` means + "collapse all" (back-compat). Otherwise only motifs whose + ``motif_id`` maps to ``"consolidate"`` are collapsed. + + Returns: + The subset of motifs to pass to + :func:`flowx.motifs.collapser.collapse_motifs`. + """ + if motif_consolidations is None: + return list(detected_motifs) + return [m for m in detected_motifs if motif_consolidations.get(m.definition.motif_id) == "consolidate"] + + def _dispatch_activity( activity: AdfActivity, context: TranslationContext, @@ -192,7 +286,7 @@ def _dispatch_activity( Returns: Tuple of ``(translated_activity, updated_context)``. """ - base_kwargs = _build_base_kwargs(activity, definitions) + base_kwargs = _build_base_kwargs(activity, definitions, context=context) match activity.type: case "ForEach": @@ -291,6 +385,420 @@ def _translate_activity_list( return results, context +# C-10 (SCHED-001): map Windows timezone names ADF emits onto IANA names +# the Databricks DAB ``schedule.timezone_id`` field expects. Only the +# ones observed in the corpus are mapped explicitly; anything else passes +# through unchanged (Databricks accepts any IANA zone). +_ADF_TIMEZONE_TO_IANA: dict[str, str] = { + "UTC": "UTC", + "Coordinated Universal Time": "UTC", + "Romance Standard Time": "Europe/Madrid", + "Central Europe Standard Time": "Europe/Budapest", + "Central European Standard Time": "Europe/Warsaw", + "W. Europe Standard Time": "Europe/Berlin", + "GMT Standard Time": "Europe/London", + "Eastern Standard Time": "America/New_York", + "Central Standard Time": "America/Chicago", + "Pacific Standard Time": "America/Los_Angeles", + "Mountain Standard Time": "America/Denver", + "Tokyo Standard Time": "Asia/Tokyo", + "China Standard Time": "Asia/Shanghai", + "India Standard Time": "Asia/Kolkata", + "AUS Eastern Standard Time": "Australia/Sydney", +} + +_DAYS_OF_WEEK_MAP: dict[str, str] = { + "Sunday": "SUN", + "Monday": "MON", + "Tuesday": "TUE", + "Wednesday": "WED", + "Thursday": "THU", + "Friday": "FRI", + "Saturday": "SAT", +} + + +def _compile_pipeline_schedule( + pipeline: AdfPipeline, + definitions: AdfDefinitions, +) -> dict[str, Any] | None: + """Compiles the first matching ADF trigger into a Pipeline.schedule dict. + + C-10 (SCHED-001): translates ScheduleTrigger recurrence into a + quartz_cron_expression + timezone_id pair the DAB writer can emit + as the job's ``schedule:`` block. BlobEventsTrigger maps to a + ``trigger.file_arrival`` spec. TumblingWindowTrigger and + CustomEventsTrigger are best-effort: they emit a SETUP-style hint + so the user can finish wiring them manually. + """ + triggers = getattr(definitions, "triggers", None) or [] + pipeline_name = pipeline.name + matching_triggers = [t for t in triggers if _trigger_references(t, pipeline_name)] + if not matching_triggers: + return None + + # First matching trigger wins -- ADF allows multiple triggers per + # pipeline but DAB schedules are 1:1. Subsequent triggers can be + # surfaced via SETUP.md by downstream tooling. + trigger = matching_triggers[0] + spec = _adf_trigger_to_schedule(trigger) + if spec is not None: + # SCHED3-003: pull per-pipeline parameter overrides off the + # matching pipelineReference so trigger-injected params (e.g. + # ``{applicationName: 'app0001', negocio: 'GLP'}``) propagate to + # the job's default parameter values. + overrides = _extract_trigger_parameter_overrides(trigger, pipeline_name) + if overrides: + spec["parameter_overrides"] = overrides + return spec + + +def _trigger_references(trigger: Any, pipeline_name: str) -> bool: + """Returns True if *trigger* references the named pipeline.""" + refs = trigger.pipelines or [] + for ref in refs: + if not isinstance(ref, dict): + continue + pipeline_ref = ref.get("pipelineReference") or {} + if isinstance(pipeline_ref, dict) and pipeline_ref.get("referenceName") == pipeline_name: + return True + return False + + +def _extract_trigger_parameter_overrides(trigger: Any, pipeline_name: str) -> dict[str, Any]: + """Returns the parameters block on the trigger's pipelineReference entry. + + SCHED3-003: ADF triggers attach per-pipeline parameter overrides at the + ``triggers[].pipelines[].parameters`` level so scheduled runs receive + deterministic values for pipeline parameters. Without surfacing them, + scheduled invocations would receive the pipeline parameter defaults + only. + """ + refs = trigger.pipelines or [] + for ref in refs: + if not isinstance(ref, dict): + continue + pipeline_ref = ref.get("pipelineReference") or {} + if not isinstance(pipeline_ref, dict): + continue + if pipeline_ref.get("referenceName") != pipeline_name: + continue + params = ref.get("parameters") or {} + if isinstance(params, dict) and params: + return dict(params) + return {} + + +def _adf_trigger_to_schedule(trigger: Any) -> dict[str, Any] | None: + """Compiles an :class:`AdfTrigger` into a Pipeline.schedule spec dict.""" + props = trigger.properties or {} + type_properties = props.get("typeProperties") or {} + runtime_state = props.get("runtimeState", "Started") + pause_status = "PAUSED" if runtime_state == "Stopped" else "UNPAUSED" + + trigger_type = trigger.type + if trigger_type == "ScheduleTrigger": + recurrence = type_properties.get("recurrence") or {} + # SCHED3-002: Day/Week/Month with interval > 1 cannot be represented + # in quartz cron without enumerating every Nth occurrence; use the + # trigger.periodic primitive so it ships correctly. + periodic = _recurrence_to_periodic(recurrence) + if periodic is not None: + spec: dict[str, Any] = { + "kind": "periodic", + "interval": periodic["interval"], + "unit": periodic["unit"], + "pause_status": pause_status, + } + # C-36 (SCHED4-001): forward the captured time-of-day so the + # bundler can flag it in SETUP.md. + if "time_of_day_note" in periodic: + spec["time_of_day_note"] = periodic["time_of_day_note"] + return spec + # C-45 (SCHED5-002): an interval > 1 Month recurrence has no + # monthly-cron-expressible form (cron fires every month, ignoring the + # interval) and the DAB periodic enum has no MONTHS unit, so surface a + # manual setup note instead of silently emitting a monthly cron. + if _is_multi_month_recurrence(recurrence): + return { + "kind": "manual_setup", + "trigger_type": "ScheduleTrigger", + "pause_status": pause_status, + "note": ( + "Month-frequency trigger with interval > 1 has no DAB " + "equivalent (PeriodicTriggerConfigurationTimeUnit lacks " + "MONTHS and quartz cron cannot encode every-Nth-month). " + "Configure the schedule manually." + ), + } + cron = _recurrence_to_quartz_cron(recurrence) + if cron is None: + return None + timezone_id = _normalize_timezone(recurrence.get("timeZone")) + spec = { + "kind": "schedule", + "quartz_cron_expression": cron, + "timezone_id": timezone_id, + "pause_status": pause_status, + } + return spec + if trigger_type == "TumblingWindowTrigger": + # Approximate as a periodic schedule -- the user should review. + frequency = type_properties.get("frequency", "Hour") + interval = type_properties.get("interval", 1) + spec = { + "kind": "schedule", + "tumbling": True, + "frequency": frequency, + "interval": interval, + "pause_status": pause_status, + "note": "Approximated from TumblingWindowTrigger; review window boundaries.", + } + return spec + if trigger_type == "BlobEventsTrigger": + scope = type_properties.get("scope", "") + events = type_properties.get("events") or [] + spec = { + "kind": "file_arrival", + "url": scope, + "events": list(events), + "pause_status": pause_status, + } + return spec + if trigger_type == "CustomEventsTrigger": + spec = { + "kind": "manual_setup", + "trigger_type": "CustomEventsTrigger", + "pause_status": pause_status, + "note": "CustomEventsTrigger has no direct DAB equivalent; configure in SETUP.md.", + } + return spec + return None + + +def _recurrence_to_periodic(recurrence: dict[str, Any]) -> dict[str, Any] | None: + """Returns a {interval, unit} dict when the recurrence requires periodic. + + SCHED3-002: Day/Week/Month with ``interval > 1`` cannot be modelled in + quartz cron without enumerating every Nth occurrence. The DAB + ``trigger.periodic`` primitive accepts ``{interval, unit}`` directly, + so we emit a periodic spec instead. Minute/Hour with interval > 1 are + expressible in cron (``0/N``) so we still leave them to the cron path. + + C-36 (SCHED4-001): when the recurrence carries a non-empty + ``schedule`` block (hours / minutes / weekDays / monthDays), the + ``trigger.periodic`` primitive can't encode the time-of-day so the + schedule silently fires at midnight instead. Capture the original + schedule on a ``time_of_day_note`` field so the bundler can emit a + ``manual_schedule_time_of_day`` SetupTask (SETUP.md). + """ + frequency = recurrence.get("frequency") + interval = recurrence.get("interval", 1) + if isinstance(interval, str) and interval.isdigit(): + interval = int(interval) + if not isinstance(interval, int) or interval <= 1: + return None + # C-45 (SCHED5-002): the DAB PeriodicTriggerConfigurationTimeUnit enum + # only defines DAYS / HOURS / WEEKS — emitting MONTHS makes bundle + # validate/deploy reject the trigger. Month frequencies are routed to + # the quartz cron path (monthDays) instead; an interval > 1 Month, which + # is not monthly-cron-expressible, is surfaced as a setup note by the + # caller. + unit_map = {"Day": "DAYS", "Week": "WEEKS"} + unit = unit_map.get(frequency or "") + if unit is None: + return None + spec: dict[str, Any] = {"interval": interval, "unit": unit} + schedule = recurrence.get("schedule") or {} + if isinstance(schedule, dict): + time_of_day = { + key: schedule.get(key) for key in ("hours", "minutes", "weekDays", "monthDays") if schedule.get(key) + } + if time_of_day: + spec["time_of_day_note"] = time_of_day + return spec + + +def _is_multi_month_recurrence(recurrence: dict[str, Any]) -> bool: + """Returns True for a Month-frequency recurrence with ``interval > 1``. + + C-45 (SCHED5-002): these triggers cannot ship as either a periodic spec + (no MONTHS unit in the DAB enum) or a quartz cron (cron has no + every-Nth-month form), so the caller emits a manual setup note. + """ + if recurrence.get("frequency") != "Month": + return False + interval = recurrence.get("interval", 1) + if isinstance(interval, str) and interval.isdigit(): + interval = int(interval) + return isinstance(interval, int) and interval > 1 + + +def _recurrence_to_quartz_cron(recurrence: dict[str, Any]) -> str | None: + """Compiles a ScheduleTrigger recurrence block into a quartz cron expression. + + ADF recurrence has ``frequency`` + ``interval`` + ``schedule``. The + quartz format expected by DAB is + ``second minute hour day-of-month month day-of-week``. + """ + frequency = recurrence.get("frequency") + interval = recurrence.get("interval", 1) + schedule = recurrence.get("schedule") or {} + minutes = schedule.get("minutes") + hours = schedule.get("hours") + week_days = schedule.get("weekDays") or [] + month_days = schedule.get("monthDays") or [] + + # C-44 (SCHED5-001): when the schedule block carries no explicit + # time-of-day, ADF defaults it to the first-execution time derived from + # ``startTime``. Reading only ``schedule.minutes/hours`` (falling back + # to '0'/'0') silently shifts a ``startTime`` of 21:00 to midnight. + # Derive the hour/minute from ``startTime`` so the cron fires at the + # ADF-intended time. + start_hour, start_minute = _start_time_hour_minute(recurrence.get("startTime")) + minute_default = str(start_minute) if start_minute is not None else "0" + hour_default = str(start_hour) if start_hour is not None else "0" + + minute_field = _list_or_default(minutes, minute_default) + hour_field = _list_or_default(hours, hour_default) + if isinstance(interval, str) and interval.isdigit(): + interval = int(interval) + + if frequency == "Minute": + if not isinstance(interval, int) or interval <= 0: + interval = 1 + return f"0 0/{interval} * * * ?" + if frequency == "Hour": + if not isinstance(interval, int) or interval <= 0: + interval = 1 + return f"0 {minute_field} 0/{interval} * * ?" + if frequency == "Day": + return f"0 {minute_field} {hour_field} * * ?" + if frequency == "Week": + days = ",".join(_DAYS_OF_WEEK_MAP.get(d, d) for d in week_days) or "MON" + return f"0 {minute_field} {hour_field} ? * {days}" + if frequency == "Month": + dom_field = _list_or_default(month_days, "1") + return f"0 {minute_field} {hour_field} {dom_field} * ?" + return None + + +def _list_or_default(value: Any, default: str) -> str: + """Renders a recurrence list/scalar as a cron-field string.""" + if value is None: + return default + if isinstance(value, list): + if not value: + return default + return ",".join(str(v) for v in value) + return str(value) + + +def _start_time_hour_minute(start_time: Any) -> tuple[int | None, int | None]: + """Parses an ISO 8601 ``startTime`` into ``(hour, minute)``. + + C-44 (SCHED5-001): ADF uses the trigger's first-execution time (from + ``startTime``) as the default time-of-day when the recurrence carries no + explicit ``schedule.hours/minutes``. Returns ``(None, None)`` when the + value is missing or unparseable so the caller keeps the midnight + fallback. + """ + if not isinstance(start_time, str) or not start_time.strip(): + return None, None + value = start_time.strip() + # ``datetime.fromisoformat`` rejects a trailing 'Z' before 3.11; map it + # to the explicit UTC offset so older interpreters parse it too. + if value.endswith("Z"): + value = value[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return None, None + return parsed.hour, parsed.minute + + +def _normalize_timezone(tz: Any) -> str: + """Maps an ADF timezone string to an IANA zone the DAB writer accepts.""" + if not tz or not isinstance(tz, str): + return "UTC" + return _ADF_TIMEZONE_TO_IANA.get(tz, tz) + + +def _build_variable_init_activities( + pipeline: AdfPipeline, + context: TranslationContext, +) -> tuple[list[Activity], TranslationContext]: + """Synthesise init SetVariable IR tasks for variables carrying defaultValue. + + C-05 (VAREX-002): without an explicit ADF SetVariable activity, a + variable's defaultValue is never materialised, so downstream + ``@variables('X')`` references fall back to a dangling + ``{{tasks.X.values.X}}`` reference. This helper emits an init task + per default-valued variable so the variable_cache carries a real + setter task_key. + """ + from flowx.parser.expression_parser import resolve_expression + + if not pipeline.variables: + return [], context + + init_tasks: list[Activity] = [] + for var_name, var_def in pipeline.variables.items(): + default = var_def.default_value + if default is None: + return_default = False + else: + return_default = True + if not return_default: + continue + task_key = f"_init_{_sanitize_task_key(var_name)}" + expr_result = resolve_expression(default, context) + if expr_result is not None: + variable_value = expr_result.value + value_kind = expr_result.kind + notebook_code = expr_result.value if expr_result.kind == "notebook_code" else None + notebook_imports = list(expr_result.imports) if expr_result.kind == "notebook_code" else [] + required_parameters = dict(expr_result.required_parameters) + else: + # VAREX3-002: Boolean defaults must render lowercase ('true'/'false') + # so downstream ``@equals(variables('continue'), true)`` evaluates + # consistently with ADF semantics. Python ``str(True)`` would + # produce title-case 'True' and silently invert the comparison. + if isinstance(default, bool): + variable_value = "true" if default else "false" + else: + variable_value = str(default) if not isinstance(default, str) else default + value_kind = "literal" + notebook_code = None + notebook_imports = [] + required_parameters = {} + + init_activity = SetVariableActivity( + name=f"_init_{var_name}", + task_key=task_key, + description=None, + timeout_seconds=None, + max_retries=None, + min_retry_interval_millis=None, + depends_on=None, + cluster=None, + variable_name=var_name, + variable_value=variable_value, + value_kind=value_kind, + notebook_code=notebook_code, + notebook_imports=notebook_imports, + required_parameters=required_parameters, + ) + init_tasks.append(init_activity) + # Register the synthesised setter so @variables('X') resolves to + # {{tasks._init_X.values.X}}. When the value is itself a DAB ref + # (e.g. from @utcNow()), inline it directly per existing semantics. + dab_ref_value = variable_value if value_kind == "dab_ref" else None + context = context.with_variable(var_name, task_key, dab_ref_value=dab_ref_value) + context = context.with_activity(init_activity.name, init_activity) + return init_tasks, context + + def _topological_visit(activities: list[AdfActivity]) -> list[AdfActivity]: """Return activities in dependency-first (topological) order. @@ -340,7 +848,12 @@ def _topological_visit(activities: list[AdfActivity]) -> list[AdfActivity]: _TIMEOUT_RE = re.compile(r"(?:(\d+)\.)?(\d{2}):(\d{2}):(\d{2})") -def _build_base_kwargs(activity: AdfActivity, definitions: AdfDefinitions) -> dict[str, Any]: +def _build_base_kwargs( + activity: AdfActivity, + definitions: AdfDefinitions, + *, + context: TranslationContext | None = None, +) -> dict[str, Any]: """Extracts common fields shared by all Activity IR subclasses. Args: @@ -369,7 +882,7 @@ def _build_base_kwargs(activity: AdfActivity, definitions: AdfDefinitions) -> di if activity.depends_on: depends_on = [] for dependency in activity.depends_on: - outcome = dependency.dependency_conditions[0] if dependency.dependency_conditions else None + outcome = _map_dependency_conditions(dependency.dependency_conditions) depends_on.append( Dependency( task_key=_sanitize_task_key(dependency.activity), @@ -378,11 +891,22 @@ def _build_base_kwargs(activity: AdfActivity, definitions: AdfDefinitions) -> di ) cluster: dict[str, Any] | None = None + existing_cluster_id: str | None = None if activity.linked_service_name: linked_service_name = activity.linked_service_name.reference_name linked_service_def = definitions.linked_services.get(linked_service_name) if linked_service_def: - cluster = _extract_cluster_config(linked_service_def.properties) + ls_param_overrides = _resolve_ls_parameters( + linked_service_def.properties, + activity.linked_service_name.parameters, + context=context, + ) + cluster = _extract_cluster_config( + linked_service_def.properties, + ls_param_overrides, + ) + if cluster: + existing_cluster_id = cluster.get("existing_cluster_id") return { "name": activity.name, @@ -393,6 +917,7 @@ def _build_base_kwargs(activity: AdfActivity, definitions: AdfDefinitions) -> di "min_retry_interval_millis": min_retry_interval_millis, "depends_on": depends_on, "cluster": cluster, + "existing_cluster_id": existing_cluster_id, } @@ -411,6 +936,46 @@ def _sanitize_task_key(name: str) -> str: return key or "unnamed" +def _map_dependency_conditions(conditions: list[str] | None) -> str | None: + """Map an ADF dependsOn[].dependencyConditions list to a single outcome. + + ADF accepts multiple conditions on a single edge — e.g. + ``['Succeeded', 'Failed']`` means "run regardless of upstream + success/failure". Databricks Workflows encodes the same semantics + by combining ``run_if`` and per-edge outcomes; the encoding that + propagates correctly through this codebase is to pick a single + representative outcome that the downstream + ``run_if_from_adf_outcomes`` reducer can interpret. + + Mapping rules (single condition): + Succeeded -> "Succeeded" + Failed -> "Failed" + Completed -> "Completed" (run regardless of upstream result) + Skipped -> "Skipped" + + Mapping rules (multi): + Any list that includes ``Failed`` AND ``Succeeded`` -> "Completed" + Any list that includes ``Skipped`` -> "Skipped" + Multi-element list including ``Failed`` only -> "Failed" + Anything else -> first item + """ + if not conditions: + return None + normalized = [c for c in conditions if c] + if not normalized: + return None + if len(normalized) == 1: + return normalized[0] + cset = set(normalized) + if "Failed" in cset and "Succeeded" in cset: + return "Completed" + if "Skipped" in cset: + return "Skipped" + if "Failed" in cset: + return "Failed" + return normalized[0] + + def _parse_adf_timeout(timeout_str: str) -> int | None: """Parses an ADF timeout string to total seconds. @@ -430,16 +995,183 @@ def _parse_adf_timeout(timeout_str: str) -> int | None: return days * 86400 + hours * 3600 + minutes * 60 + seconds -def _extract_cluster_config(ls_properties: dict[str, Any]) -> dict[str, Any] | None: +def _resolve_ls_parameters( + ls_properties: dict[str, Any], + activity_supplied: dict[str, Any] | None, + context: TranslationContext | None = None, +) -> dict[str, Any]: + """Builds the effective LS parameter map for cluster-config resolution. + + Args: + ls_properties: Full properties bag from the linked service JSON. + activity_supplied: Per-activity parameter overrides from the + ``linkedServiceName.parameters`` block (may be ``None``). + context: Translation context used to resolve ``@``-prefixed + activity-supplied values against factory global parameters. + When omitted, ADF expressions are left as raw strings. + + Returns: + Mapping of parameter name -> resolved value. Activity-supplied + overrides win over LS defaultValue. Wrapped ``{"value": ..., "type": + "Expression"}`` dicts are unwrapped and (when ``context`` is set) + passed through :func:`resolve_expression` so ``@pipeline(). + globalParameters.X`` collapses to the factory value. + """ + from flowx.parser.expression_parser import resolve_expression + + resolved: dict[str, Any] = {} + declared = ls_properties.get("parameters") or {} + if isinstance(declared, dict): + for pname, pdef in declared.items(): + if isinstance(pdef, dict) and "defaultValue" in pdef: + resolved[pname] = _unwrap_expression_value(pdef["defaultValue"]) + if isinstance(activity_supplied, dict): + for pname, pval in activity_supplied.items(): + raw = _unwrap_expression_value(pval) + # C-03: route @-prefixed activity-supplied values through the + # expression parser so @pipeline().globalParameters.X collapses + # to the factory value when one is set. + if context is not None and isinstance(raw, str) and raw.startswith("@"): + result = resolve_expression(raw, context) + # C-13 (NB-ITER3-002 / LSC3-003 / VAREX3-006): accept both + # literal and dab_ref so @pipeline().parameters.X collapses + # to {{job.parameters.X}} (valid in custom_tags map values). + if result is not None and result.kind in ("literal", "dab_ref"): + raw = result.value + resolved[pname] = raw + return resolved + + +def _unwrap_expression_value(value: Any) -> Any: + """Unwrap a ``{"value": ..., "type": "Expression"}`` ADF dict-wrapper. + + C-02 (NB-ITER2-2 / LSC2-003): activity-supplied LS parameter values and + LS-derived cluster fields (``custom_tags``, ``spark_env_vars`` entries, + ...) sometimes ship as the ADF expression-dict shape. Databricks + cluster YAML rejects nested dicts in ``custom_tags``; recursive unwrap + flattens them to scalars while leaving regular dicts untouched. + """ + if isinstance(value, dict): + # Bare {"value": X, "type": "Expression"} -- collapse to inner X. + if "value" in value and value.get("type") == "Expression": + return _unwrap_expression_value(value["value"]) + # Some payloads omit the explicit type marker but follow the same + # single-key shape. Conservatively unwrap only when the dict has + # the exact two keys {"value", "type"} so we don't corrupt regular + # nested config blocks like {"workspace": {"destination": ...}}. + if set(value.keys()) == {"value", "type"}: + return _unwrap_expression_value(value["value"]) + return {k: _unwrap_expression_value(v) for k, v in value.items()} + if isinstance(value, list): + return [_unwrap_expression_value(v) for v in value] + return value + + +_LS_PARAM_REF_RE = re.compile(r"@linkedService\(\s*\)\.(\w+)", re.IGNORECASE) + + +def _substitute_ls_params(value: Any, params: dict[str, Any]) -> Any: + """Replaces ``@linkedService().X`` tokens in *value* with bound params. + + Handles scalar strings and nested dicts/lists. Returns the value + unchanged when no substitution is possible. + """ + if isinstance(value, str): + if "@linkedService()" not in value: + return value + # Full-string single-reference: drop in the resolved value with its + # original type so e.g. integer params don't get stringified. + full = _LS_PARAM_REF_RE.fullmatch(value) + if full is not None: + name = full.group(1) + if name in params: + return params[name] + return value + + def _sub(match: re.Match[str]) -> str: + name = match.group(1) + if name in params: + return str(params[name]) + return match.group(0) + + return _LS_PARAM_REF_RE.sub(_sub, value) + if isinstance(value, dict): + return {k: _substitute_ls_params(v, params) for k, v in value.items()} + if isinstance(value, list): + return [_substitute_ls_params(v, params) for v in value] + return value + + +def _coerce_int(value: Any) -> Any: + """Coerce numeric strings (``"1"``) to ``int``; leave other values alone.""" + if isinstance(value, bool): + return value + if isinstance(value, int): + return value + try: + return int(value) + except (TypeError, ValueError): + return value + + +def _coerce_parameter_default(value: Any, declared_type: str) -> Any: + """Coerce an ADF parameter default into a Python type matching its declared type. + + The declared type (``"Bool"`` / ``"Int"`` / ``"Float"`` / ``"String"`` / + ``"Array"`` / ``"Object"``) is what ADF stores in the pipeline JSON. + Defaults round-trip as strings through JSON, so a Bool parameter with + default ``false`` arrives as the literal ``"False"``. The fix re-types + each default per the declared type so the emitted YAML carries a real + bool / int / float, not a quoted string. + """ + t = (declared_type or "String").lower() + if t in ("bool", "boolean"): + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in ("true", "false"): + return lowered == "true" + return value + if t in ("int", "integer"): + return _coerce_int(value) + if t == "float": + try: + return float(value) + except (TypeError, ValueError): + return value + return value + + +def _extract_cluster_config( + ls_properties: dict[str, Any], + ls_param_overrides: dict[str, Any] | None = None, +) -> dict[str, Any] | None: """Extracts Databricks cluster configuration from a linked-service properties dict. Args: ls_properties: Full properties bag from the linked service JSON. + ls_param_overrides: Optional map of ``@linkedService().X`` -> value + overrides to substitute before extraction. When supplied, + every string value in the LS payload is rewritten through + ``_substitute_ls_params`` so cluster fields like + ``newClusterVersion: '@linkedService().clusterVersion'`` + resolve to a real Spark version string. Returns: Cluster configuration dict, or ``None`` if no Databricks cluster details are present. """ + overrides = ls_param_overrides or {} + if overrides: + ls_properties = _substitute_ls_params(ls_properties, overrides) + + # C-02 (NB-ITER2-2 / LSC2-003): unwrap any {value, type:'Expression'} + # dicts that survived the substitution pass. Map fields like + # custom_tags and spark_env_vars must be plain Map[String, String] for + # Databricks to accept the cluster YAML. + ls_properties = _unwrap_expression_value(ls_properties) + nested = ls_properties.get("typeProperties") or {} # Merge: nested values win over flat ones when both exist (matches ARM # template precedence). @@ -454,16 +1186,43 @@ def _extract_cluster_config(ls_properties: dict[str, Any]) -> dict[str, Any] | N new_cluster = fields.get("newClusterVersion") or fields.get("newClusterSparkVersion") if new_cluster: config["spark_version"] = new_cluster - num_workers_raw = fields.get("newClusterNumOfWorker", 1) - try: - config["num_workers"] = int(num_workers_raw) - except (TypeError, ValueError): - config["num_workers"] = num_workers_raw + config["num_workers"] = _coerce_int(fields.get("newClusterNumOfWorker", 1)) config["node_type_id"] = fields.get("newClusterNodeType", "Standard_DS3_v2") spark_conf = fields.get("newClusterSparkConf") if spark_conf: config["spark_conf"] = spark_conf + # Extended cluster fields (LSC-003, NB-3). + driver_node = fields.get("newClusterDriverNodeType") + if driver_node: + config["driver_node_type_id"] = driver_node + spark_env_vars = fields.get("newClusterSparkEnvVars") + if spark_env_vars: + config["spark_env_vars"] = spark_env_vars + custom_tags = fields.get("newClusterCustomTags") + if custom_tags: + config["custom_tags"] = custom_tags + init_scripts = fields.get("newClusterInitScripts") + if init_scripts: + config["init_scripts"] = init_scripts + data_security_mode = fields.get("dataSecurityMode") or fields.get("newClusterDataSecurityMode") + if data_security_mode: + config["data_security_mode"] = data_security_mode + cluster_log_conf = fields.get("clusterLogConf") or fields.get("newClusterLogDestination") + if cluster_log_conf: + config["cluster_log_conf"] = cluster_log_conf + + # C-39 (LSC4-004): capture the ADF authentication shape (e.g. "MSI" or + # any CredentialReference) so the bundler can emit a manual_credential + # SetupTask warning that ``single_user_name`` was rewritten to the + # deploying user. + authentication = fields.get("authentication") + if authentication: + config["_adf_authentication"] = authentication + credential = fields.get("credential") + if isinstance(credential, dict) and credential.get("type") == "CredentialReference": + config["_adf_credential_reference"] = credential.get("referenceName") or "" + return config if config else None @@ -495,15 +1254,17 @@ def _preferences_to_dict(preferences: Any) -> dict[str, Any]: preferences: The :class:`TranslationPreferences` snapshot to serialise. Returns: - Dictionary with the four StrEnum fields rendered as their string - values and per-task overrides preserved verbatim. + Dictionary with each StrEnum field rendered as its string value + and per-task overrides preserved verbatim. """ return { "copy_activity_paradigm": str(preferences.copy_activity_paradigm), "non_databricks_task_compute": str(preferences.non_databricks_task_compute), "use_lakeflow_connectors": str(preferences.use_lakeflow_connectors), - "databricks_task_compute": str(preferences.databricks_task_compute), "lakeflow_connector_type": str(preferences.lakeflow_connector_type), + "motif_consolidations": { + motif_id: str(choice) for motif_id, choice in preferences.motif_consolidations.items() + }, "per_task": dict(preferences.per_task), } @@ -536,8 +1297,14 @@ def _activity_to_dict(task: Activity) -> dict[str, Any]: ] if task.cluster: task_dict["cluster"] = task.cluster + if task.existing_cluster_id: + task_dict["existing_cluster_id"] = task.existing_cluster_id if task.compute_mode: task_dict["compute_mode"] = task.compute_mode + if task.libraries: + task_dict["libraries"] = task.libraries + if task.parameter_approximations: + task_dict["parameter_approximations"] = task.parameter_approximations extra = _activity_extra_fields(task) task_dict.update(extra) @@ -560,6 +1327,12 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["notebook_path"] = activity.notebook_path if activity.base_parameters: extra["base_parameters"] = activity.base_parameters + if activity.notebook_path_unresolved: + extra["notebook_path_unresolved"] = True + if activity.notebook_path_expression is not None: + extra["notebook_path_expression"] = activity.notebook_path_expression + if activity.unresolved_libraries: + extra["unresolved_libraries"] = list(activity.unresolved_libraries) case CopyActivity(): extra["source_type"] = activity.source_type extra["sink_type"] = activity.sink_type @@ -585,12 +1358,24 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["items_expression"] = activity.items_expression extra["concurrency"] = activity.concurrency extra["inner_activities"] = [_activity_to_dict(inner) for inner in activity.inner_activities] + if activity.inputs_bridge_notebook_code: + extra["inputs_bridge_notebook_code"] = activity.inputs_bridge_notebook_code + if activity.inputs_bridge_notebook_imports: + extra["inputs_bridge_notebook_imports"] = list(activity.inputs_bridge_notebook_imports) + if activity.inputs_bridge_required_parameters: + extra["inputs_bridge_required_parameters"] = dict(activity.inputs_bridge_required_parameters) case IfConditionActivity(): extra["op"] = activity.op extra["left"] = activity.left extra["right"] = activity.right extra["if_true_activities"] = [_activity_to_dict(inner) for inner in activity.if_true_activities] extra["if_false_activities"] = [_activity_to_dict(inner) for inner in activity.if_false_activities] + if activity.bridge_notebook_code: + extra["bridge_notebook_code"] = activity.bridge_notebook_code + if activity.bridge_notebook_imports: + extra["bridge_notebook_imports"] = list(activity.bridge_notebook_imports) + if activity.bridge_required_parameters: + extra["bridge_required_parameters"] = dict(activity.bridge_required_parameters) case LookupActivity(): extra["source_type"] = activity.source_type if activity.source_properties: @@ -608,6 +1393,8 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["notebook_imports"] = activity.notebook_imports if activity.required_parameters: extra["required_parameters"] = dict(activity.required_parameters) + if activity.raw_expression: + extra["raw_expression"] = activity.raw_expression case FilterActivity(): extra["items_expression"] = activity.items_expression extra["condition_expression"] = activity.condition_expression @@ -632,14 +1419,18 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: for case_item in activity.cases ] extra["default_activities"] = [_activity_to_dict(inner) for inner in activity.default_activities] + if activity.bridge_notebook_code: + extra["bridge_notebook_code"] = activity.bridge_notebook_code + if activity.bridge_notebook_imports: + extra["bridge_notebook_imports"] = list(activity.bridge_notebook_imports) + if activity.bridge_required_parameters: + extra["bridge_required_parameters"] = dict(activity.bridge_required_parameters) case WaitActivity(): extra["wait_time_seconds"] = activity.wait_time_seconds case SparkJarActivity(): extra["main_class_name"] = activity.main_class_name if activity.parameters: extra["parameters"] = activity.parameters - if activity.libraries: - extra["libraries"] = activity.libraries case SparkPythonActivity(): extra["python_file"] = activity.python_file if activity.parameters: diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 0247e30..92c2615 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -10,7 +10,6 @@ from flowx.adapter import ( CopyActivityParadigm, - DatabricksTaskCompute, NonDatabricksTaskCompute, TranslationInputRequired, TranslationPreferences, @@ -29,7 +28,6 @@ COMPUTE_MODE_SERVERLESS, LAKEFLOW_CONNECT_REPLACEMENT, QUESTION_COPY_ACTIVITY_PARADIGM, - QUESTION_DATABRICKS_TASK_COMPUTE, QUESTION_LAKEFLOW_CONNECTOR_TYPE, QUESTION_METADATA_DRIVEN_ACCESS, QUESTION_METADATA_DRIVEN_CONSOLIDATE, @@ -129,7 +127,6 @@ def test_default_preferences_are_conservative(self): assert prefs.copy_activity_paradigm is CopyActivityParadigm.NOTEBOOK assert prefs.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS assert prefs.use_lakeflow_connectors is UseLakeflowConnectors.EXISTING - assert prefs.databricks_task_compute is DatabricksTaskCompute.EXISTING def test_string_values_coerce_to_enums(self): prefs = TranslationPreferences( @@ -235,21 +232,23 @@ def test_lakeflow_connect_question_surfaces_for_database_motif(self): lfc_question = next(q for q in pending.questions if q.question_id == QUESTION_USE_LAKEFLOW_CONNECTORS) assert "motif_incremental_load_watermark" in lfc_question.affected_task_keys - def test_databricks_task_compute_question_when_notebook_present(self): + def test_no_databricks_task_compute_question_for_notebook(self): + """The serverless-replacement question for Databricks tasks was removed.""" pipeline = Pipeline( name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")], ) ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_DATABRICKS_TASK_COMPUTE in ids + assert "databricks_task_compute" not in ids - def test_databricks_task_compute_question_for_spark_python(self): + def test_no_databricks_task_compute_question_for_spark_python(self): + """The serverless-replacement question for Databricks tasks was removed.""" pipeline = Pipeline( name="p", tasks=[SparkPythonActivity(**_make_base("py"), python_file="dbfs:/scripts/etl.py")], ) ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_DATABRICKS_TASK_COMPUTE in ids + assert "databricks_task_compute" not in ids def test_already_answered_filters_pending(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) @@ -272,6 +271,56 @@ def test_walks_into_for_each_inner_activities(self): assert question is not None assert "inner_copy" in question.affected_task_keys + def test_motif_consolidation_question_emitted_per_detected_motif(self): + """Each detected motif produces a ``consolidate_motif:`` question.""" + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + motifs = [ + DetectedMotif( + definition=MOTIF_INCREMENTAL_LOAD_WATERMARK, + matched_activities=["WatermarkLookup", "DeltaCopy"], + source_type_hint="database", + confidence_notes=["Detector matched Lookup→Copy→SP chain"], + ) + ] + pending = gather_questions(pipeline, motifs) + ids = {q.question_id for q in pending.questions} + assert "consolidate_motif:incremental_load_watermark" in ids + motif_question = next( + q for q in pending.questions if q.question_id == "consolidate_motif:incremental_load_watermark" + ) + assert motif_question.default == "keep" + assert {opt.value for opt in motif_question.options} == {"keep", "consolidate"} + assert "WatermarkLookup" in motif_question.affected_task_keys + # Confidence note must surface in the rationale so the agent can quote it + assert "Detector matched Lookup→Copy→SP chain" in motif_question.rationale + + def test_motif_consolidation_question_filtered_by_answer(self): + """Once answered the per-motif question must drop out of pending.""" + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + motifs = [ + DetectedMotif( + definition=MOTIF_INCREMENTAL_LOAD_WATERMARK, + matched_activities=["WatermarkLookup"], + source_type_hint=None, + confidence_notes=[], + ) + ] + pending = gather_questions( + pipeline, + motifs, + answers={"consolidate_motif:incremental_load_watermark": "consolidate"}, + ) + ids = {q.question_id for q in pending.questions} + assert "consolidate_motif:incremental_load_watermark" not in ids + + def test_motif_consolidation_validate_answer_accepts_keep_or_consolidate(self): + assert validate_answer("consolidate_motif:rest_api_pagination", "keep") == "keep" + assert validate_answer("consolidate_motif:rest_api_pagination", "consolidate") == "consolidate" + + def test_motif_consolidation_validate_answer_rejects_unknown_value(self): + with pytest.raises(ValueError, match="Invalid answer"): + validate_answer("consolidate_motif:rest_api_pagination", "merge") + class TestValidateAnswer: def test_accepts_allowed_value(self): @@ -302,14 +351,19 @@ def test_classic_compute_routes_copy_to_multi_node_cluster(self): assert modified.tasks[0].compute_mode == COMPUTE_MODE_CLASSIC_MULTI_NODE assert modified.tasks[1].compute_mode == COMPUTE_MODE_CLASSIC_SINGLE_NODE - def test_databricks_task_serverless_stamps_serverless(self): + def test_databricks_task_always_inherits_linked_service_cluster(self): + """DatabricksNotebook activities always inherit the source linked-service cluster + binding; the serverless replacement option was removed.""" pipeline = Pipeline(name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")]) - prefs = TranslationPreferences(databricks_task_compute="serverless") - modified = apply_preferences(pipeline, prefs) - assert modified.tasks[0].compute_mode == COMPUTE_MODE_SERVERLESS + modified = apply_preferences(pipeline, TranslationPreferences()) + assert modified.tasks[0].compute_mode == COMPUTE_MODE_INHERIT - def test_databricks_task_existing_stamps_inherit(self): - pipeline = Pipeline(name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")]) + def test_spark_python_always_inherits_linked_service_cluster(self): + """DatabricksSparkPython activities always inherit the source linked-service cluster + binding; the serverless replacement option was removed.""" + pipeline = Pipeline( + name="p", tasks=[SparkPythonActivity(**_make_base("py"), python_file="dbfs:/scripts/etl.py")] + ) modified = apply_preferences(pipeline, TranslationPreferences()) assert modified.tasks[0].compute_mode == COMPUTE_MODE_INHERIT @@ -459,7 +513,6 @@ def test_preferences_survive_json_roundtrip(self): copy_activity_paradigm="sdp", non_databricks_task_compute="classic", use_lakeflow_connectors="lakeflow_connect", - databricks_task_compute="serverless", ) stamped = apply_preferences(pipeline, prefs) roundtripped, _ = pipeline_dict_to_ir(json.loads(json.dumps(_pipeline_to_dict(stamped), default=str))) @@ -467,7 +520,9 @@ def test_preferences_survive_json_roundtrip(self): assert roundtripped.tasks[0].target_format == "sdp" assert roundtripped.tasks[0].use_lakeflow_connector is True assert roundtripped.tasks[0].compute_mode == COMPUTE_MODE_CLASSIC_MULTI_NODE - assert roundtripped.tasks[1].compute_mode == COMPUTE_MODE_SERVERLESS + # NotebookActivity always inherits the linked-service cluster binding now + # that the serverless replacement option has been removed. + assert roundtripped.tasks[1].compute_mode == COMPUTE_MODE_INHERIT class TestMigrationInputSession: @@ -1123,23 +1178,6 @@ def test_lakeflow_connect_emits_connection_setup_notebook(self, tmp_path: Path): assert "orchestra_copy_a_connection" in body assert "SQLSERVER" in body - def test_serverless_existing_notebook_skips_default_cluster_bind(self, tmp_path: Path): - import yaml - - from flowx.bundler.dab_writer import write_bundle - from flowx.preparer.workflow_preparer import prepare_workflow - - pipeline = Pipeline( - name="job", - tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/existing")], - ) - stamped = apply_preferences(pipeline, TranslationPreferences(databricks_task_compute="serverless")) - workflow = prepare_workflow(stamped) - write_bundle(workflow, tmp_path) - job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) - task = job_yml["resources"]["jobs"]["job"]["tasks"][0] - assert "job_cluster_key" not in task - def test_existing_default_binds_to_default_cluster(self, tmp_path: Path): import yaml diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index 605dd89..4cfa416 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -156,7 +156,9 @@ def test_setup_notebooks_for_secrets(self, tmp_path): secrets_nb = setup_dir / "create_secrets.py" if secrets_nb.exists(): content = secrets_nb.read_text() - assert "createScope" in content + # C-46 (LSC5-002): provision via the SDK WorkspaceClient, not + # the non-existent dbutils.secrets write API. + assert "create_scope" in content def test_write_bundle_returns_created_files(self, tmp_path): """write_bundle returns a list of all created file paths.""" @@ -261,6 +263,645 @@ def test_load_report_handles_aggregated_translations_format(self, tmp_path): assert task_keys == {"pause", "run_nb"} +class TestScheduleEmission: + """C-10 (SCHED-001): schedule spec on PreparedWorkflow lands in job YAML.""" + + def test_schedule_block_emitted(self, tmp_path): + pipeline = Pipeline( + name="scheduled_job", + tasks=[ + WaitActivity(name="Pause", task_key="pause", wait_time_seconds=10), + ], + schedule={ + "kind": "schedule", + "quartz_cron_expression": "0 0 8 * * ?", + "timezone_id": "Europe/Madrid", + "pause_status": "UNPAUSED", + }, + ) + wf = prepare_workflow(pipeline) + write_bundle(wf, tmp_path) + resource_file = list((tmp_path / "resources").glob("*.yml"))[0] + content = yaml.safe_load(resource_file.read_text()) + job_key = list(content["resources"]["jobs"].keys())[0] + job = content["resources"]["jobs"][job_key] + assert job["schedule"]["quartz_cron_expression"] == "0 0 8 * * ?" + assert job["schedule"]["timezone_id"] == "Europe/Madrid" + assert job["schedule"]["pause_status"] == "UNPAUSED" + + def test_periodic_trigger_emitted(self, tmp_path): + """SCHED3-002: periodic schedule spec renders as trigger.periodic.""" + pipeline = Pipeline( + name="periodic_job", + tasks=[ + WaitActivity(name="Pause", task_key="pause", wait_time_seconds=10), + ], + schedule={ + "kind": "periodic", + "interval": 3, + "unit": "DAYS", + "pause_status": "UNPAUSED", + }, + ) + wf = prepare_workflow(pipeline) + write_bundle(wf, tmp_path) + resource_file = list((tmp_path / "resources").glob("*.yml"))[0] + content = yaml.safe_load(resource_file.read_text()) + job_key = list(content["resources"]["jobs"].keys())[0] + job = content["resources"]["jobs"][job_key] + assert job["trigger"]["periodic"]["interval"] == 3 + assert job["trigger"]["periodic"]["unit"] == "DAYS" + # The cron-style schedule block must NOT appear for periodic specs. + assert "schedule" not in job + + def test_trigger_parameter_overrides_mutate_job_parameter_defaults(self, tmp_path): + """SCHED3-003: schedule.parameter_overrides mutates matching + job.parameters entries' default values.""" + pipeline = Pipeline( + name="trg_override_job", + tasks=[ + WaitActivity(name="Pause", task_key="pause", wait_time_seconds=10), + ], + schedule={ + "kind": "schedule", + "quartz_cron_expression": "0 0 2 * * ?", + "timezone_id": "UTC", + "pause_status": "UNPAUSED", + "parameter_overrides": { + "negocio": "GLP", + "applicationName": "app0001", + }, + }, + ) + wf = prepare_workflow(pipeline) + # Pipeline parameters land on PreparedWorkflow via the report + # round-trip; emulate that here so the bundler has parameters to + # mutate. + wf.parameters = [ + {"name": "negocio", "default": "DEFAULT"}, + {"name": "applicationName", "default": "DEFAULT"}, + ] + write_bundle(wf, tmp_path) + resource_file = list((tmp_path / "resources").glob("*.yml"))[0] + content = yaml.safe_load(resource_file.read_text()) + job_key = list(content["resources"]["jobs"].keys())[0] + job = content["resources"]["jobs"][job_key] + params = {p["name"]: p["default"] for p in job["parameters"]} + assert params["negocio"] == "GLP" + assert params["applicationName"] == "app0001" + + def test_file_arrival_trigger_emitted(self, tmp_path): + pipeline = Pipeline( + name="blob_triggered_job", + tasks=[ + WaitActivity(name="Pause", task_key="pause", wait_time_seconds=10), + ], + schedule={ + "kind": "file_arrival", + "url": "/subscriptions/x/y", + "pause_status": "UNPAUSED", + }, + ) + wf = prepare_workflow(pipeline) + write_bundle(wf, tmp_path) + resource_file = list((tmp_path / "resources").glob("*.yml"))[0] + content = yaml.safe_load(resource_file.read_text()) + job_key = list(content["resources"]["jobs"].keys())[0] + job = content["resources"]["jobs"][job_key] + assert job["trigger"]["file_arrival"]["url"] == "/subscriptions/x/y" + + +class TestStripDanglingTaskValueRefs: + """C-12 (VAREX-005): safety net widens to job_parameters and condition operands.""" + + def test_strips_dangling_run_job_task_job_parameters(self): + from flowx.bundler.dab_writer import _strip_dangling_task_value_refs + + tasks = [ + { + "task_key": "outer", + "run_job_task": { + "job_id": "${resources.jobs.inner.id}", + "job_parameters": { + "valid": "{{tasks.outer.values.something}}", + "dangling": "{{tasks.gone.values.x}}", + }, + }, + }, + ] + _strip_dangling_task_value_refs(tasks, {"outer"}) + assert tasks[0]["run_job_task"]["job_parameters"]["valid"] == "{{tasks.outer.values.something}}" + assert tasks[0]["run_job_task"]["job_parameters"]["dangling"] == "" + + def test_strips_dangling_condition_task_operands(self): + from flowx.bundler.dab_writer import _strip_dangling_task_value_refs + + tasks = [ + { + "task_key": "branch", + "condition_task": { + "op": "EQUAL_TO", + "left": "{{tasks.missing.values.x}}", + "right": "1", + }, + }, + ] + neutralized = _strip_dangling_task_value_refs(tasks, {"branch"}) + # Dangling ref blanked; right operand untouched. + assert tasks[0]["condition_task"]["left"] == "" + assert tasks[0]["condition_task"]["right"] == "1" + # C-43 (CF5-001 / CF5-002): the blanked condition operand is + # recorded so SETUP.md can flag the always-true predicate. + assert neutralized == [ + { + "task_key": "branch", + "field": "left", + "original_ref": "{{tasks.missing.values.x}}", + } + ] + + def test_neutralized_condition_renders_setup_section(self): + """C-43 (CF5-001 / CF5-002): a blanked condition operand surfaces a + 'Conditions neutralized to always-true' section in SETUP.md so the + always-true predicate is never silent.""" + from flowx.bundler.prereqs_writer import build_prereqs, render_setup_md + + prereqs = build_prereqs( + notebooks=[], + tasks=[], + known_bundle_jobs=set(), + neutralized_conditions=[ + { + "task_key": "branch", + "field": "left", + "original_ref": "{{tasks._init_continue.values.continue}}", + } + ], + ) + assert not prereqs.is_empty() + md = render_setup_md(prereqs, bundle_name="b") + assert "Conditions neutralized to always-true" in md + assert "{{tasks._init_continue.values.continue}}" in md + assert "`branch`" in md + + def test_recurses_into_for_each_task_body(self): + from flowx.bundler.dab_writer import _strip_dangling_task_value_refs + + tasks = [ + { + "task_key": "loop", + "for_each_task": { + "inputs": "[1, 2, 3]", + "task": { + "task_key": "loop_body", + "run_job_task": { + "job_id": "inner", + "job_parameters": {"x": "{{tasks.absent.values.x}}"}, + }, + }, + }, + }, + ] + _strip_dangling_task_value_refs(tasks, {"loop", "loop_body"}) + assert tasks[0]["for_each_task"]["task"]["run_job_task"]["job_parameters"]["x"] == "" + + +class TestAggregatedReportPipelineParameters: + """Change pipeline-parameters-and-variables-round-trip (P0): VAR-001.""" + + def test_load_report_carries_pipeline_parameters(self, tmp_path): + import json + + from flowx.bundler.dab_writer import _load_report + + report = { + "translations": [ + { + "pipeline": "p1", + "status": "translated", + "parameters": [{"name": "env", "default": "dev"}], + "ir": { + "type": "WaitActivity", + "name": "Pause", + "task_key": "pause", + "wait_time_seconds": 5, + }, + }, + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + + workflows = _load_report(report_path) + assert len(workflows) == 1 + wf = workflows[0] + # Pipeline-level parameters must survive round-trip. + assert wf.parameters + env_param = next(p for p in wf.parameters if p["name"] == "env") + assert env_param["default"] == "dev" + + +class TestAggregatedReportSchedule: + """Change fix-aggregated-report-propagates-schedule (P0): SCHED3-001.""" + + def test_load_report_carries_pipeline_schedule(self, tmp_path): + import json + + from flowx.bundler.dab_writer import _load_report + + schedule_spec = { + "kind": "cron", + "quartz_cron_expression": "0 0 2 ? * * *", + "timezone_id": "UTC", + "pause_status": "UNPAUSED", + } + report = { + "translations": [ + { + "pipeline": "p_with_schedule", + "status": "translated", + "schedule": schedule_spec, + "ir": { + "type": "WaitActivity", + "name": "Pause", + "task_key": "pause", + "wait_time_seconds": 5, + }, + }, + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + + workflows = _load_report(report_path) + assert len(workflows) == 1 + wf = workflows[0] + assert wf.schedule is not None + assert wf.schedule["kind"] == "cron" + assert wf.schedule["quartz_cron_expression"] == "0 0 2 ? * * *" + + def test_load_report_carries_pipeline_schedule_from_ir(self, tmp_path): + """Older single-pipeline reports nest schedule under ``ir.schedule``.""" + import json + + from flowx.bundler.dab_writer import _load_report + + schedule_spec = { + "kind": "cron", + "quartz_cron_expression": "0 0 4 ? * MON,TUE,WED,THU,FRI *", + "timezone_id": "Europe/Madrid", + "pause_status": "UNPAUSED", + } + report = { + "translations": [ + { + "pipeline": "p_with_ir_schedule", + "status": "translated", + "ir": { + "type": "WaitActivity", + "name": "Pause", + "task_key": "pause", + "wait_time_seconds": 5, + "schedule": schedule_spec, + }, + }, + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + + workflows = _load_report(report_path) + assert len(workflows) == 1 + assert workflows[0].schedule is not None + assert workflows[0].schedule["quartz_cron_expression"].startswith("0 0 4") + + +class TestStubBaseParameterCleanup: + """Change base-parameters-cleanup-and-stub-widgets (P1): NB-5.""" + + def test_stub_notebook_strips_unresolvable_adf_expression(self): + from flowx.bundler.dab_writer import ( + _extract_manual_parameters_from_existing_notebook_tasks, + ) + + tasks = [ + { + "task_key": "lakeh_custom_notebook", + "notebook_task": { + "notebook_path": "../src/notebooks/x.py", + "base_parameters": { + "appName": "@string(coalesce(json(activity('X').output).mail_app_name, ''))", + "kept": "literal-value", + }, + }, + } + ] + manual = _extract_manual_parameters_from_existing_notebook_tasks(tasks) + assert len(manual) == 1 + assert manual[0].task_key == "lakeh_custom_notebook" + assert manual[0].widget_name == "appName" + assert "@string" in manual[0].raw_expression + # The ADF expression value must be dropped from base_parameters. + bp = tasks[0]["notebook_task"]["base_parameters"] + assert "appName" not in bp + assert bp["kept"] == "literal-value" + + +class TestStubLibraryBinding: + """Change library-resolution-and-stub-binding (P0): NB-2.""" + + def test_stub_notebook_with_jar_library_binds_to_default_cluster(self): + from flowx.bundler.constants import DEFAULT_JOB_CLUSTER_KEY + from flowx.bundler.dab_writer import _bind_cluster_to_notebook_tasks + + tasks = [ + { + "task_key": "lakeh_custom_notebook", + "notebook_task": {"notebook_path": "../src/notebooks/x.py"}, + "libraries": [{"jar": "/Volumes/x/my.jar"}], + } + ] + _bind_cluster_to_notebook_tasks(tasks) + # Stub path that ships libraries must bind to the default cluster. + assert tasks[0]["job_cluster_key"] == DEFAULT_JOB_CLUSTER_KEY + + def test_stub_notebook_no_libraries_stays_unbound(self): + from flowx.bundler.dab_writer import _bind_cluster_to_notebook_tasks + + tasks = [ + { + "task_key": "k", + "notebook_task": {"notebook_path": "../src/notebooks/x.py"}, + } + ] + _bind_cluster_to_notebook_tasks(tasks) + assert "job_cluster_key" not in tasks[0] + + def test_serverless_compute_mode_with_jar_library_binds_classic(self): + from flowx.bundler.constants import DEFAULT_JOB_CLUSTER_KEY + from flowx.bundler.dab_writer import _bind_cluster_to_notebook_tasks + + tasks = [ + { + "task_key": "k", + "notebook_task": {"notebook_path": "/Shared/nb"}, + "libraries": [{"whl": "/Volumes/x/wheel.whl"}], + "_compute_mode": "serverless", + } + ] + _bind_cluster_to_notebook_tasks(tasks) + # Serverless can't host whl libraries -> classic cluster bind. + assert tasks[0]["job_cluster_key"] == DEFAULT_JOB_CLUSTER_KEY + + +class TestClusterExtrasPropagation: + """Change linked-service-cluster-field-coverage (P1): NB-3, LSC-003.""" + + def test_extras_merged_into_default_cluster(self, tmp_path): + from flowx.bundler.constants import DEFAULT_JOB_CLUSTER_KEY + from flowx.bundler.dab_writer import ( + _build_default_cluster, + _build_default_job_clusters, + _infer_bundle_cluster_extras, + ) + + # Hint set with consistent extras. + wf = _simple_workflow() + wf.cluster_hints = [ + { + "spark_version": "16.4.x-scala2.12", + "node_type_id": "Standard_D4s_v3", + "driver_node_type_id": "Standard_D8s_v3", + "spark_env_vars": {"PYSPARK_PYTHON": "/databricks/python3/bin/python3"}, + "custom_tags": {"DigitalCase": "X"}, + "data_security_mode": "SINGLE_USER", + } + ] + extras = _infer_bundle_cluster_extras(wf) + assert extras["driver_node_type_id"] == "Standard_D8s_v3" + assert extras["spark_env_vars"]["PYSPARK_PYTHON"] == "/databricks/python3/bin/python3" + assert extras["custom_tags"]["DigitalCase"] == "X" + + clusters = _build_default_job_clusters({DEFAULT_JOB_CLUSTER_KEY}, extras=extras) + assert len(clusters) == 1 + new_cluster = clusters[0]["new_cluster"] + assert new_cluster["driver_node_type_id"] == "Standard_D8s_v3" + assert new_cluster["spark_env_vars"]["PYSPARK_PYTHON"] == "/databricks/python3/bin/python3" + assert new_cluster["custom_tags"]["DigitalCase"] == "X" + + # _build_default_cluster() with no extras keeps the legacy shape. + baseline = _build_default_cluster() + assert "driver_node_type_id" not in baseline["new_cluster"] + + def test_num_workers_mined_into_default_cluster(self): + """C-40 (NB-ITER5-001): cluster_hints carrying num_workers!=1 must + flow into the default job_cluster instead of the hardcoded 1.""" + from flowx.bundler.constants import DEFAULT_JOB_CLUSTER_KEY + from flowx.bundler.dab_writer import ( + _build_default_cluster, + _build_default_job_clusters, + _infer_bundle_cluster_extras, + ) + + wf = _simple_workflow() + wf.cluster_hints = [ + { + "spark_version": "16.4.x-scala2.12", + "node_type_id": "Standard_D4s_v3", + "num_workers": 2, + } + ] + extras = _infer_bundle_cluster_extras(wf) + assert extras["num_workers"] == 2 + + clusters = _build_default_job_clusters({DEFAULT_JOB_CLUSTER_KEY}, extras=extras) + assert clusters[0]["new_cluster"]["num_workers"] == 2 + + # No hint -> legacy single-worker default preserved. + baseline = _build_default_cluster() + assert baseline["new_cluster"]["num_workers"] == 1 + + +class TestManualCredentialFromMsiLinkedService: + """C-39 (LSC4-004): when an ADF linked service authenticates via MSI + (or a CredentialReference), the bundle's default_cluster silently + uses ``single_user_name: ${workspace.current_user.userName}``. The + workflow_preparer must surface a manual_credential SetupTask so + SETUP.md flags the substitution.""" + + def test_msi_authentication_emits_manual_credential_setup_task(self): + from flowx.models.ir import NotebookActivity, Pipeline + from flowx.preparer.workflow_preparer import prepare_workflow + + activity = NotebookActivity( + name="Notebook1", + task_key="notebook1", + notebook_path="/Shared/x", + cluster={ + "spark_version": "15.4.x-scala2.12", + "node_type_id": "Standard_D4s_v3", + "data_security_mode": "SINGLE_USER", + "_adf_authentication": "MSI", + }, + ) + pipeline = Pipeline(name="msi_pipe", tasks=[activity]) + wf = prepare_workflow(pipeline) + manual = [st for st in wf.setup_tasks if st.type == "manual_credential"] + assert len(manual) == 1 + config = manual[0].config + assert config["authentication"] == "MSI" + assert "service principal" in config["note"].lower() + + +class TestUnparseableClusterHintsFiltered: + """C-29 (NB-ITER4-002): unparseable spark_version / node_type_id values + are filtered before Counter so the bundle default stays deployable.""" + + def test_unparseable_spark_version_falls_back_to_default(self): + from flowx.bundler.dab_writer import ( + _DEFAULT_SPARK_VERSION, + _infer_bundle_cluster_defaults, + ) + + wf = _simple_workflow() + wf.cluster_hints = [ + { + "spark_version": "@if(equals(item()?.photon,true),'15.4.x-photon-scala2.12','15.4.x-scala2.12')", + "node_type_id": "Standard_DS3_v2", + }, + ] + spark_version, node_type_id = _infer_bundle_cluster_defaults(wf) + assert spark_version == _DEFAULT_SPARK_VERSION + assert node_type_id == "Standard_DS3_v2" + + def test_unparseable_node_type_falls_back_to_default(self): + from flowx.bundler.dab_writer import ( + _DEFAULT_NODE_TYPE_ID, + _infer_bundle_cluster_defaults, + ) + + wf = _simple_workflow() + wf.cluster_hints = [ + { + "spark_version": "15.4.x-scala2.12", + "node_type_id": "@pipeline().parameters.unresolved", + }, + ] + spark_version, node_type_id = _infer_bundle_cluster_defaults(wf) + assert spark_version == "15.4.x-scala2.12" + assert node_type_id == _DEFAULT_NODE_TYPE_ID + + def test_real_spark_version_still_wins(self): + from flowx.bundler.dab_writer import _infer_bundle_cluster_defaults + + wf = _simple_workflow() + wf.cluster_hints = [ + {"spark_version": "15.4.x-photon-scala2.12", "node_type_id": "Standard_D4s_v3"}, + {"spark_version": "15.4.x-photon-scala2.12", "node_type_id": "Standard_D4s_v3"}, + {"spark_version": "@if(equals(item()?.photon,true),X,Y)", "node_type_id": "Standard_D4s_v3"}, + ] + spark_version, _ = _infer_bundle_cluster_defaults(wf) + assert spark_version == "15.4.x-photon-scala2.12" + + +class TestSingleUserNameOnSingleUserClusters: + """Change fix-single-user-cluster-requires-single-user-name (P0): NB-ITER3-003.""" + + def test_default_cluster_includes_single_user_name(self): + from flowx.bundler.dab_writer import _build_default_cluster + + cluster = _build_default_cluster()["new_cluster"] + assert cluster["data_security_mode"] == "SINGLE_USER" + assert cluster["single_user_name"] == "${workspace.current_user.userName}" + + def test_single_node_cluster_includes_single_user_name(self): + from flowx.bundler.dab_writer import _build_single_node_cluster + + cluster = _build_single_node_cluster()["new_cluster"] + assert cluster["data_security_mode"] == "SINGLE_USER" + assert cluster["single_user_name"] == "${workspace.current_user.userName}" + + def test_multi_node_cluster_includes_single_user_name(self): + from flowx.bundler.dab_writer import _build_multi_node_cluster + + cluster = _build_multi_node_cluster()["new_cluster"] + assert cluster["data_security_mode"] == "SINGLE_USER" + assert cluster["single_user_name"] == "${workspace.current_user.userName}" + + +class TestSetupMd: + def test_parameter_approximations_render_to_setup_md(self, tmp_path): + pipeline = Pipeline( + name="approx_pipeline", + tasks=[ + NotebookActivity( + name="Score", + task_key="score", + notebook_path="/Shared/score", + base_parameters={"scoring_date": "{{job.start_time.iso_date}}"}, + parameter_approximations=[ + { + "widget_name": "scoring_date", + "raw_expression": "@formatDateTime(utcnow(), 'yyyy-MM-dd')", + "replacement": "{{job.start_time.iso_date}}", + "note": "Mapped ADF `utcnow()` to the Databricks job start time.", + } + ], + ), + ], + ) + wf = prepare_workflow(pipeline) + write_bundle(wf, tmp_path) + setup_md = (tmp_path / "SETUP.md").read_text() + assert "## Parameter substitutions" in setup_md + assert "`score`" in setup_md + assert "`scoring_date`" in setup_md + assert "@formatDateTime(utcnow(), 'yyyy-MM-dd')" in setup_md + assert "{{job.start_time.iso_date}}" in setup_md + assert "Mapped ADF `utcnow()`" in setup_md + + +class TestManualVariableRollupSetupMd: + """Change fix-cross-foreach-variable-read-warning (P1): VAREX3-003.""" + + def test_setup_md_surfaces_manual_variable_rollup(self, tmp_path): + from flowx.models.ir import ForEachActivity, IfConditionActivity, SetVariableActivity + + # Mirror the preparer-side detection by building the same pipeline. + inner_set = SetVariableActivity( + name="MarkStop", + task_key="mark_stop", + variable_name="continue", + variable_value="false", + ) + loop = ForEachActivity( + name="Loop", + task_key="loop", + items_expression="@output.value", + inner_activities=[inner_set], + concurrency=2, + ) + sibling = IfConditionActivity( + name="CheckCont", + task_key="check_cont", + op="EQUAL_TO", + left="@variables('continue')", + right="true", + if_true_activities=[], + if_false_activities=[], + ) + pipeline = Pipeline( + name="rollup_pipeline", + tasks=[loop, sibling], + ) + wf = prepare_workflow(pipeline) + write_bundle(wf, tmp_path) + setup_md = (tmp_path / "SETUP.md").read_text() + assert "## Manual variable roll-ups" in setup_md + assert "`continue`" in setup_md + assert "`loop`" in setup_md + + class TestSetupGenerator: def test_secrets_setup_notebook_content(self): from flowx.bundler.setup_generator import generate_setup_tasks @@ -273,7 +914,13 @@ def test_secrets_setup_notebook_content(self): assert len(notebooks) == 1 nb = notebooks[0] assert nb.relative_path == "setup/create_secrets.py" - assert "createScope" in nb.content + # C-46 (LSC5-002): generated against the SDK WorkspaceClient, since + # dbutils.secrets is read-only (no createScope / put). + assert "w.secrets.create_scope" in nb.content + assert "w.secrets.put_secret" in nb.content + assert "WorkspaceClient" in nb.content + assert "dbutils.secrets.createScope" not in nb.content + assert "dbutils.secrets.put" not in nb.content assert "my-scope" in nb.content assert "jdbc-url" in nb.content assert "jdbc-password" in nb.content @@ -310,3 +957,64 @@ def test_no_setup_when_empty(self): notebooks = generate_setup_tasks(secrets=[], setup_tasks=[], catalog="main", schema="default") assert len(notebooks) == 0 + + +class TestPipelineDictToIrBridgeFields: + """C-14 (CF3-001 / VAREX3-001): bridge fields survive JSON roundtrip.""" + + def test_if_condition_bridge_fields_preserved(self): + from flowx.bundler.dab_writer import pipeline_dict_to_ir + from flowx.models.ir import IfConditionActivity + + pipeline_dict = { + "name": "p", + "parameters": [], + "tasks": [ + { + "type": "IfConditionActivity", + "name": "Branch", + "task_key": "branch", + "op": "EQUAL_TO", + "left": "__BRIDGE__::result", + "right": "True", + "if_true_activities": [], + "if_false_activities": [], + "bridge_notebook_code": "result = not bool(some_param)", + "bridge_notebook_imports": ["import os"], + "bridge_required_parameters": {"some_param": "{{job.parameters.x}}"}, + } + ], + } + pipeline, _ = pipeline_dict_to_ir(pipeline_dict) + task = pipeline.tasks[0] + assert isinstance(task, IfConditionActivity) + assert task.bridge_notebook_code == "result = not bool(some_param)" + assert task.bridge_notebook_imports == ["import os"] + assert task.bridge_required_parameters == {"some_param": "{{job.parameters.x}}"} + + def test_switch_bridge_fields_preserved(self): + from flowx.bundler.dab_writer import pipeline_dict_to_ir + from flowx.models.ir import SwitchActivity + + pipeline_dict = { + "name": "p", + "parameters": [], + "tasks": [ + { + "type": "SwitchActivity", + "name": "Sw", + "task_key": "sw", + "on_expression": "__BRIDGE__::result", + "cases": [], + "default_activities": [], + "bridge_notebook_code": "result = item.get('type', 'default').upper()", + "bridge_notebook_imports": [], + "bridge_required_parameters": {"item": "{{tasks.upstream.values.row}}"}, + } + ], + } + pipeline, _ = pipeline_dict_to_ir(pipeline_dict) + task = pipeline.tasks[0] + assert isinstance(task, SwitchActivity) + assert task.bridge_notebook_code == "result = item.get('type', 'default').upper()" + assert task.bridge_required_parameters == {"item": "{{tasks.upstream.values.row}}"} diff --git a/tests/unit/test_code_generator.py b/tests/unit/test_code_generator.py index 84a07d0..eb98fc1 100644 --- a/tests/unit/test_code_generator.py +++ b/tests/unit/test_code_generator.py @@ -225,6 +225,68 @@ def test_unparseable_dynamic_query_falls_back_to_literal(self): # ("dbutils.widgets.get(..."), not as executable Python. assert 'query = "dbutils.widgets.get(' in content + def test_file_lookup_coerces_expression_dict_path_components(self): + """C-37 (LSC4-001): folder_path / file_name that arrive as ADF + expression dicts must not crash ``_assemble_file_lookup_source_path`` + with AttributeError on ``.strip('/')``.""" + activity = LookupActivity( + **_make_base("Read_Cfg", "read_cfg"), + source_type="JsonSource", + first_row_only=False, + source_properties={ + "dataset_type": "Json", + "container": "configs", + "folder_path": {"value": "@pipeline().parameters.folder", "type": "Expression"}, + "file_name": "tables.json", + }, + ) + # Must not raise. + content = generate_lookup_notebook(activity) + _assert_valid_python(content, "read_cfg (expression-dict folder)") + + def test_file_lookup_rewrites_https_to_abfss(self): + """C-37 (LSC4-003): AzureBlobFS https URLs lower to abfss:// in + the generated lookup notebook so the read can succeed on a + Databricks cluster.""" + activity = LookupActivity( + **_make_base("Read_Cfg", "read_cfg"), + source_type="JsonSource", + first_row_only=False, + source_properties={ + "dataset_type": "Json", + "container": "configext", + "folder_path": "lookups", + "file_name": "tables.json", + "linked_service_url": "https://examplelake.dfs.core.windows.net", + }, + ) + content = generate_lookup_notebook(activity) + _assert_valid_python(content, "read_cfg (abfss rewrite)") + assert "abfss://configext@examplelake.dfs.core.windows.net" in content + assert "https://examplelake" not in content + + def test_file_source_lookup_emits_spark_read(self): + """Change lookup-file-dataset-support (P0): JsonSource + firstRowOnly=False.""" + activity = LookupActivity( + **_make_base("Read_Configuration", "read_configuration"), + source_type="JsonSource", + first_row_only=False, + source_properties={ + "dataset_type": "Json", + "container": "configs", + "folder_path": "lookup", + "file_name": "tables.json", + "multiLineJson": True, + }, + ) + content = generate_lookup_notebook(activity) + _assert_valid_python(content, "read_configuration (file source)") + # File-source branch: no spark.sql(''), uses spark.read.format().load(). + assert "spark.sql" not in content + assert "spark.read.format('json')" in content + assert '.option("multiline", "true")' in content + assert "source_path" in content + # --------------------------------------------------------------------------- # Web activity notebook generator @@ -278,6 +340,23 @@ def test_auth_block_service_principal(self): assert "auth-credential" in content assert "Bearer" in content + def test_auth_block_msi_raises_not_implemented(self): + """LSC3-002: MSI / ManagedServiceIdentity has no static secret to + read; the generated notebook must raise NotImplementedError pointing + at SETUP.md, not emit a fake dbutils.secrets.get('auth-credential').""" + for auth_type in ("MSI", "ManagedServiceIdentity"): + activity = WebActivity( + **_make_base(f"MsiApi_{auth_type}", f"msi_api_{auth_type.lower()}"), + url="https://api.example.com", + method="GET", + authentication={"type": auth_type, "resource": "https://management.azure.com"}, + ) + content = generate_web_activity_notebook(activity, scope=f"msi_api_{auth_type.lower()}") + _assert_valid_python(content, f"msi_api ({auth_type})") + assert "auth-credential" not in content + assert "NotImplementedError" in content + assert "SETUP.md" in content + def test_auth_block_basic(self): """Basic auth generates username/password secret retrieval.""" activity = WebActivity( diff --git a/tests/unit/test_expression_parser.py b/tests/unit/test_expression_parser.py index b88bfdf..5df2cc4 100644 --- a/tests/unit/test_expression_parser.py +++ b/tests/unit/test_expression_parser.py @@ -38,10 +38,16 @@ def test_float(self): assert result.value == "3.14" def test_boolean(self): - result = resolve_expression(True, _context()) - assert result is not None - assert result.kind == "literal" - assert result.value == "True" + # VAREX3-002: Python bool renders lowercase to match ADF semantics. + result_t = resolve_expression(True, _context()) + assert result_t is not None + assert result_t.kind == "literal" + assert result_t.value == "true" + + result_f = resolve_expression(False, _context()) + assert result_f is not None + assert result_f.kind == "literal" + assert result_f.value == "false" def test_expression_dict_wrapping(self): result = resolve_expression({"type": "Expression", "value": "hello"}, _context()) @@ -136,11 +142,12 @@ def test_variable_with_explicit_task_keys(self): assert result.kind == "dab_ref" assert result.value == "{{tasks.SetRunDate.values.runDate}}" - def test_variable_fallback_to_name(self): + def test_variable_returns_none_when_no_setter(self): + """C-05 (VAREX-002): unknown variables resolve to ``None`` instead of + a self-referential dangling ``{{tasks.X.values.X}}`` placeholder + that never gets satisfied at runtime.""" result = resolve_expression("@variables('unknown')", _context()) - assert result is not None - assert result.kind == "dab_ref" - assert result.value == "{{tasks.unknown.values.unknown}}" + assert result is None class TestItem: @@ -150,25 +157,72 @@ def test_item(self): assert result.kind == "dab_ref" assert result.value == "{{input}}" + def test_item_safe_nav_single_segment(self): + """C-16 (CF3-005 / VAREX3-005): single-segment item()?.X must lower to + notebook_code so bridge lowering can fire downstream (Switch on-expr, + SetVariable expressions wrapping the safe-nav).""" + result = resolve_expression("@item()?.foo", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "json" in result.value + assert "get('foo')" in result.value + + def test_item_safe_nav_multi_segment_unchanged(self): + """Two-segment item()?.a?.b continues to lower to notebook_code.""" + result = resolve_expression("@item()?.foo?.bar", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "get('foo')" in result.value + assert "get('bar')" in result.value + + def test_item_field_no_safe_nav_remains_dab_ref(self): + """Plain item().foo with no safe-nav operator stays a dab_ref.""" + result = resolve_expression("@item().foo", _context()) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{input.foo}}" + + def test_item_field_multi_segment_lowers_to_notebook_code(self): + """C-35 (CF4-004): ``item().condition.name`` must walk both + ``.condition`` and ``.name`` instead of truncating to + ``{{input.condition}}``. Previously ``_ITEM_FIELD_RE`` matched the + first segment without an end-anchor so the trailing ``.name`` + was silently dropped.""" + result = resolve_expression("@item().condition.name", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "get('condition')" in result.value + assert "get('name')" in result.value + class TestUtcNow: def test_utcnow_no_format(self): - # ``@utcNow()`` resolves to Python ``datetime.now(...).isoformat()`` - # rather than the DAB ref ``{{job.start_time.iso_datetime}}`` so - # compositions like ``@formatDateTime(utcNow(), '...')`` chain - # correctly. DAB does not evaluate ADF expressions, so wrapping a - # DAB ref in another ADF function would emit broken YAML. + # ``@utcNow()`` maps to the Databricks job start time so the result + # lands directly in DAB YAML. The translator attaches a note so + # the bundler can surface the activity-vs-job-start skew caveat in + # SETUP.md. result = resolve_expression("@utcNow()", _context()) assert result is not None - assert result.kind == "notebook_code" - assert result.value == "datetime.now(timezone.utc).isoformat()" + assert result.kind == "dab_ref" + assert result.value == "{{job.start_time.iso_datetime}}" + assert any("utcnow" in note.lower() for note in result.notes) - def test_utcnow_with_format(self): + def test_utcnow_with_known_iso_format(self): result = resolve_expression("@utcNow('yyyy-MM-dd')", _context()) assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.start_time.iso_date}}" + assert any("utcnow" in note.lower() for note in result.notes) + + def test_utcnow_with_unknown_format_falls_back_to_notebook_code(self): + # ``yyyyMMdd`` (no separators) is not in the DAB dynamic-value + # vocabulary, so flowx keeps the legacy Python strftime path. + result = resolve_expression("@utcNow('yyyyMMdd')", _context()) + assert result is not None assert result.kind == "notebook_code" assert "strftime" in result.value - assert "%Y-%m-%d" in result.value + assert "%Y%m%d" in result.value + assert result.notes == [] def test_utcnow_expression_dict(self): result = resolve_expression( @@ -176,16 +230,34 @@ def test_utcnow_expression_dict(self): _context(), ) assert result is not None - assert result.kind == "notebook_code" + assert result.kind == "dab_ref" + assert result.value == "{{job.start_time.iso_date}}" class TestConcat: def test_concat_literals(self): + # C-01: when every part is a literal, the whole concat collapses to + # a single literal so consumers (cluster fields, library jar paths, + # ...) receive a plain string instead of Python source. result = resolve_expression("@concat('hello', ' ', 'world')", _context()) assert result is not None - assert result.kind == "notebook_code" - # Should produce a Python concatenation - assert "+" in result.value + assert result.kind == "literal" + assert result.value == "hello world" + + def test_concat_collapses_when_all_parts_resolve_to_literals(self): + # C-01: factory globals collapse @concat parts to literal kinds, + # so the whole concat should likewise be a literal string. + ctx = TranslationContext( + global_parameters=MappingProxyType({"env_variable": "t", "deequLibFileName": "deequ-3.5.6.jar"}), + ) + result = resolve_expression( + "@concat('/Volumes/datahub01', pipeline().globalParameters.env_variable, " + "'/lib/', pipeline().globalParameters.deequLibFileName)", + ctx, + ) + assert result is not None + assert result.kind == "literal" + assert result.value == "/Volumes/datahub01t/lib/deequ-3.5.6.jar" def test_concat_with_variable(self): result = resolve_expression( @@ -197,10 +269,13 @@ def test_concat_with_variable(self): assert "runDate" in result.value def test_concat_with_utcnow(self): + # ``utcNow('yyyy-MM-dd')`` is now a DAB ref, so concat wraps it as a + # widget read. The result is still notebook_code because concat + # composes Python strings. result = resolve_expression("@concat('date_', utcNow('yyyy-MM-dd'))", _context()) assert result is not None assert result.kind == "notebook_code" - assert "strftime" in result.value + assert "dbutils.widgets.get('iso_date')" in result.value def test_concat_with_pipeline_param(self): result = resolve_expression( @@ -297,6 +372,61 @@ def test_substring(self): # Should produce a slice expression assert "[" in result.value + def test_equals_quoted_string_emits_repr(self): + """C-34 (VAREX4-002): a quoted ``'12'`` argument must keep its + quotedness through codegen so the comparison emits ``... == '12'`` + rather than the bare token ``12`` (which silently compares against + a numeric value).""" + result = resolve_expression( + "@equals(variables('month'), '12')", + _context(month="set_month"), + ) + assert result is not None + assert "== '12'" in result.value + + def test_less_quoted_leading_zero_is_valid_python(self): + """C-34 (VAREX4-002): a leading-zero quoted argument like ``'09'`` + must round-trip as a Python string literal, not the bare token + ``09`` (which is a SyntaxError in modern Python).""" + result = resolve_expression( + "@less(variables('month'), '09')", + _context(month="set_month"), + ) + assert result is not None + # Result must parse as valid Python. + compile(result.value, "", "eval") + assert "< '09'" in result.value + + def test_equals_bool_literal_emits_lowercase_string(self): + """C-34 (VAREX4-003): an ADF Boolean ``true`` argument lowers to + the lowercase string literal ``'true'`` so the comparison matches + what C-21 SetVariable writes on the consumer side.""" + result = resolve_expression( + "@equals(variables('X'), true)", + _context(X="set_x"), + ) + assert result is not None + assert "== 'true'" in result.value + + def test_substring_two_arg_form(self): + """C-33 (VAREX4-001): ADF accepts substring(text, start) without an + explicit length argument.""" + result = resolve_expression("@substring(string(pipeline().parameters.params), 1)", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert "[int(" in result.value + assert "):]" in result.value + + def test_split_with_subscript(self): + """C-33 (VAREX4-001): a trailing ``[N]`` on a function call lowers + to notebook_code Python source so SetVariable activities wrapping + ``@split(...)[0]`` actually resolve.""" + result = resolve_expression("@split(pipeline().parameters.referenceDate,'/')[0]", _context()) + assert result is not None + assert result.kind == "notebook_code" + assert ".split(str('/'))" in result.value + assert ")[0]" in result.value + def test_to_lower(self): result = resolve_expression("@toLower('HELLO')", _context()) assert result is not None @@ -743,14 +873,162 @@ def test_parse_expression_for_dab_returns_ref(self): result = parse_expression_for_dab("@pipeline().RunId") assert result == "{{job.run_id}}" - def test_parse_expression_for_dab_returns_none_for_utcnow(self): - # ``@utcNow()`` now resolves to notebook_code so it composes correctly - # with other ADF time functions. ``parse_expression_for_dab`` only - # returns dab_ref kinds, so utcNow now yields ``None`` (the caller - # routes through the notebook_code path instead). + def test_parse_expression_for_dab_returns_ref_for_utcnow(self): + # ``@utcNow()`` maps to the Databricks job start time dynamic value + # so it can land in DAB YAML directly. result = parse_expression_for_dab("@utcNow()") - assert result is None + assert result == "{{job.start_time.iso_datetime}}" def test_parse_expression_for_dab_returns_none_for_non_expression(self): result = parse_expression_for_dab("plain_string") assert result is None + + +class TestGlobalParameters: + """Change expr-resolver-globalparams-and-wrappers (P0).""" + + def _ctx_with_globals(self, **globals_) -> TranslationContext: + return TranslationContext( + global_parameters=MappingProxyType(dict(globals_)), + ) + + def test_global_parameter_resolves_to_literal(self): + ctx = self._ctx_with_globals(env_variable="t") + result = resolve_expression("@pipeline().globalParameters.env_variable", ctx) + assert result is not None + assert result.kind == "literal" + assert result.value == "t" + + def test_global_parameter_value_dict(self): + ctx = self._ctx_with_globals( + env_variable={"type": "string", "value": "t"}, + ) + result = resolve_expression("@pipeline().globalParameters.env_variable", ctx) + assert result is not None + assert result.kind == "literal" + assert result.value == "t" + + def test_global_parameter_missing_falls_back_to_dab_ref(self): + ctx = TranslationContext() + result = resolve_expression("@pipeline().globalParameters.something", ctx) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.parameters.something}}" + + def test_concat_with_globals_resolves_fully(self): + # C-01: when every part collapses to a literal, the whole concat + # is itself a literal so downstream consumers don't have to eval. + ctx = self._ctx_with_globals(env_variable="t", libFileName="myjar.jar") + expr = ( + "@concat('/Volumes/datahub01', pipeline().globalParameters.env_variable, " + "'/x/', pipeline().globalParameters.libFileName)" + ) + result = resolve_expression(expr, ctx) + assert result is not None + assert result.kind == "literal" + assert result.value == "/Volumes/datahub01t/x/myjar.jar" + + +class TestNoopWrappers: + """Change expr-resolver-globalparams-and-wrappers (P0): @json/@string/@array.""" + + def test_json_wrapper_around_pipeline_param(self): + ctx = TranslationContext() + result = resolve_expression("@json(pipeline().parameters.items)", ctx) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.parameters.items}}" + + def test_string_wrapper_around_pipeline_param(self): + ctx = TranslationContext() + result = resolve_expression("@string(pipeline().parameters.value)", ctx) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.parameters.value}}" + + def test_array_wrapper_around_pipeline_param(self): + ctx = TranslationContext() + result = resolve_expression("@array(pipeline().parameters.lst)", ctx) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.parameters.lst}}" + + +class TestTrailingWhitespaceFunctionCall: + """Change expr-resolver-globalparams-and-wrappers (P0): VAR-003 regex anchor bug.""" + + def test_string_wrapper_with_trailing_newlines(self): + ctx = TranslationContext() + # Previously the regex anchor at end refused trailing whitespace. + result = resolve_expression( + "@string(activity('X').output.runOutput.year)\n\n", + ctx, + ) + assert result is not None + # Should hit the function-call branch (string wraps activity output). + assert result.kind == "dab_ref" + assert "tasks.X.values" in result.value + + +class TestItemSafeNav: + """Change expr-resolver-globalparams-and-wrappers (P0): VAR-005.""" + + def test_item_safe_nav_chain_resolves(self): + ctx = TranslationContext() + result = resolve_expression( + "@coalesce(item()?.condition?.name, 'fallback')", + ctx, + ) + assert result is not None + assert result.kind == "notebook_code" + # The chain walk should emit nested .get() calls + assert ".get('condition')" in result.value + assert ".get('name')" in result.value + assert "'fallback'" in result.value + + +class TestLinkedServiceParameter: + """Change linked-service-parameter-resolution (P0): NB-4, LSC-001.""" + + def test_linked_service_param_resolves_with_supplied_value(self): + ctx = TranslationContext( + linked_service_parameters=MappingProxyType({"clusterVersion": "16.4.x-scala2.12"}), + ) + result = resolve_expression("@linkedService().clusterVersion", ctx) + assert result is not None + assert result.kind == "literal" + assert result.value == "16.4.x-scala2.12" + + def test_linked_service_param_missing_returns_none(self): + ctx = TranslationContext() + result = resolve_expression("@linkedService().clusterVersion", ctx) + # Without a value, we can't deterministically resolve; caller must + # supply via with_linked_service_parameters or accept None. + assert result is None + + +class TestFunctionCallWithAttribute: + """Change fix-attribute-access-on-function-results (P1): CF3-004.""" + + def test_json_call_with_trailing_attribute_lowers_to_notebook_code(self): + ctx = TranslationContext() + result = resolve_expression( + "@toUpper(json(pipeline().parameters.items).type)", + ctx, + ) + assert result is not None + # toUpper wraps the json(...).type expression; the inner + # json(...).type chain must resolve to notebook_code so the bridge + # can pick it up. + assert result.kind == "notebook_code" + + def test_function_call_with_attribute_alone_lowers_to_notebook_code(self): + ctx = TranslationContext() + result = resolve_expression( + "@json(pipeline().parameters.items).type", + ctx, + ) + assert result is not None + assert result.kind == "notebook_code" + # The lowered code chains .get('type') onto a json-loaded widget. + assert ".get('type')" in result.value diff --git a/tests/unit/test_for_each_inner_job_params.py b/tests/unit/test_for_each_inner_job_params.py new file mode 100644 index 0000000..932f1c6 --- /dev/null +++ b/tests/unit/test_for_each_inner_job_params.py @@ -0,0 +1,144 @@ +"""Unit tests for flowx.bundler.inner_job_params. + +Covers C-06 (VAREX-004): variable references in a ForEach inner-job body +must not produce undeclared inner-job parameters. When the parent job has +a known setter task for the variable, the inner job receives a +``{{tasks..values.}}`` reference; when no setter is known the +name still surfaces as an inner parameter (legacy fallback so test-only +flows that omit the mapping continue to work). +""" + +from __future__ import annotations + +from flowx.bundler.inner_job_params import collect_inner_job_params + + +def _notebook_task(base_parameters: dict[str, str]) -> dict[str, object]: + return { + "task_key": "inner_nb", + "notebook_task": { + "notebook_path": "/Shared/inner", + "base_parameters": base_parameters, + }, + } + + +class TestVariableTaskKeysRouting: + def test_variable_with_known_setter_routes_via_task_value(self): + """C-06: a variable referenced inside the inner job with a known + parent-side setter is NOT declared as an inner job parameter.""" + inner_tasks = [_notebook_task({"continue": "@variables('continue')"})] + parameters, job_parameters = collect_inner_job_params( + inner_tasks, + variable_task_keys={"continue": "_init_continue"}, + ) + # No inner parameter declared for `continue` -- the parent passes + # the task-value reference through job_parameters instead. + param_names = {p["name"] for p in parameters} + assert "continue" not in param_names + assert job_parameters["continue"] == "{{tasks._init_continue.values.continue}}" + + def test_variable_without_setter_falls_back_to_parent_job_parameter(self): + """Legacy fallback for tests that don't supply a setter map.""" + inner_tasks = [_notebook_task({"continue": "@variables('continue')"})] + parameters, job_parameters = collect_inner_job_params(inner_tasks) + param_names = {p["name"] for p in parameters} + # Without variable_task_keys we still emit the (broken) job.parameters + # reference so prior behaviour is preserved when callers don't opt in. + assert "continue" in param_names + assert job_parameters["continue"] == "{{job.parameters.continue}}" + + def test_pipeline_parameter_still_uses_job_parameters_ref(self): + """C-06 only redirects variables -- pipeline parameters still flow + via the inner job's parameter declarations as before.""" + inner_tasks = [_notebook_task({"env": "@pipeline().parameters.env"})] + parameters, job_parameters = collect_inner_job_params( + inner_tasks, + variable_task_keys={"continue": "_init_continue"}, # unrelated var + ) + param_names = {p["name"] for p in parameters} + assert "env" in param_names + assert job_parameters["env"] == "{{job.parameters.env}}" + + def test_multi_child_for_each_threads_variable_task_keys(self, monkeypatch): + """CF3-006: ForEach preparer's multi-child path must thread + variable_task_keys into collect_inner_job_params just like the + single-child escalation path does, so the same parent->setter map + is honoured regardless of how many children the ForEach has. + + Asserts the kwarg is forwarded by intercepting collect_inner_job_params. + """ + from flowx.models.ir import ForEachActivity, NotebookActivity + from flowx.preparer.activity_preparers import for_each as for_each_module + from flowx.preparer.workflow_preparer import prepare_activity + + def _base(name: str, key: str) -> dict[str, object]: + return { + "name": name, + "task_key": key, + "description": None, + "timeout_seconds": None, + "max_retries": None, + "min_retry_interval_millis": None, + "depends_on": None, + "cluster": None, + } + + captured: list[dict[str, str] | None] = [] + from flowx.bundler import inner_job_params as ijp_module + + original = ijp_module.collect_inner_job_params + + def _spy(tasks, *, raw_ir_tasks=None, variable_task_keys=None): + captured.append(variable_task_keys) + return original(tasks, raw_ir_tasks=raw_ir_tasks, variable_task_keys=variable_task_keys) + + monkeypatch.setattr(for_each_module, "collect_inner_job_params", _spy) + + nb_a = NotebookActivity( + **_base("InnerA", "inner_a"), + notebook_path="/Shared/a", + base_parameters={"continue": "@variables('continue')"}, + ) + nb_b = NotebookActivity( + **_base("InnerB", "inner_b"), + notebook_path="/Shared/b", + base_parameters={"continue": "@variables('continue')"}, + ) + loop = ForEachActivity( + **_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[nb_a, nb_b], + concurrency=2, + ) + prepared = prepare_activity( + loop, + variable_task_keys={"continue": "_init_continue"}, + ) + # Multi-child escalation -> exactly one collect call from for_each preparer. + for_each_call = next((m for m in captured if m and "continue" in m), None) + assert for_each_call is not None, "variable_task_keys must be forwarded" + assert for_each_call["continue"] == "_init_continue" + # Multi-child path -> inner_workflows populated. + assert prepared.inner_workflows + + def test_mixed_variable_and_pipeline_param_payload(self): + """A base_parameters block with both a variable and a pipeline param.""" + inner_tasks = [ + _notebook_task( + { + "ctx_continue": "@variables('continue')", + "ctx_env": "@pipeline().parameters.env", + } + ) + ] + parameters, job_parameters = collect_inner_job_params( + inner_tasks, + variable_task_keys={"continue": "_init_continue"}, + ) + param_names = {p["name"] for p in parameters} + # Only the pipeline parameter is declared on the inner job. + assert "continue" not in param_names + assert "env" in param_names + assert job_parameters["continue"] == "{{tasks._init_continue.values.continue}}" + assert job_parameters["env"] == "{{job.parameters.env}}" diff --git a/tests/unit/test_helpers.py b/tests/unit/test_helpers.py index aea8086..269076e 100644 --- a/tests/unit/test_helpers.py +++ b/tests/unit/test_helpers.py @@ -57,10 +57,17 @@ def test_interpolated_resolves(self): def test_notebook_code_returns_raw_for_manual_handling(self): """Expressions that resolve to Python code return raw text (manual handling).""" - raw = "@formatDateTime(utcnow(), 'yyyy-MM-dd')" + # Pick a format that intentionally does NOT map to a DAB dynamic + # value so the resolver falls back to notebook_code. + raw = "@formatDateTime(utcnow(), 'dd MMM yyyy')" result = resolve_param_value(raw) assert result == raw + def test_formatdatetime_utcnow_known_format_resolves_to_dab_ref(self): + """``formatDateTime(utcnow(), '')`` short-circuits to a DAB ref.""" + result = resolve_param_value("@formatDateTime(utcnow(), 'yyyy-MM-dd')") + assert result == "{{job.start_time.iso_date}}" + class TestBuildNotebookTaskArtifacts: def test_returns_task_dict_and_one_notebook(self): diff --git a/tests/unit/test_ir_rewriter.py b/tests/unit/test_ir_rewriter.py new file mode 100644 index 0000000..bdadef6 --- /dev/null +++ b/tests/unit/test_ir_rewriter.py @@ -0,0 +1,234 @@ +"""Unit tests for the whole-IR expression rewriter.""" + +from __future__ import annotations + +from flowx.models.ir import ( + CopyActivity, + Dependency, + ForEachActivity, + IfConditionActivity, + LookupActivity, + NotebookActivity, + Pipeline, + SetVariableActivity, + SwitchActivity, + SwitchCase, + WebActivity, +) +from flowx.parser.ir_rewriter import rewrite_pipeline_expressions + + +def _base(task_key: str, name: str | None = None) -> dict[str, object]: + return {"name": name or task_key, "task_key": task_key} + + +class TestRewritePipelineExpressions: + def test_rewrites_sql_inside_source_properties_dict(self): + """Raw @{} tokens inside Copy's source_properties must be rewritten.""" + copy = CopyActivity( + **_base("copy_orders"), + source_type="AzureSqlSource", + source_properties={ + "sql": "SELECT * FROM dbo.orders WHERE modified_dt >= '@{pipeline().parameters.watermark}'" + }, + sink_type="ParquetSink", + ) + pipeline = Pipeline(name="p", parameters=[{"name": "watermark", "default": None}], tasks=[copy]) + rewritten = rewrite_pipeline_expressions(pipeline) + sql = rewritten.tasks[0].source_properties["sql"] + assert "@{" not in sql + assert "{{job.parameters.watermark}}" in sql + + def test_rewrites_strings_inside_lists(self): + """@{} tokens inside list-valued fields are rewritten too.""" + # CopyActivity has no list-of-strings field, so use a list inside a dict + copy = CopyActivity( + **_base("copy"), + source_type="AzureSqlSource", + source_properties={ + "partitions": ["p_@{pipeline().parameters.region}_1", "p_@{pipeline().parameters.region}_2"] + }, + ) + pipeline = Pipeline(name="p", tasks=[copy]) + rewritten = rewrite_pipeline_expressions(pipeline) + partitions = rewritten.tasks[0].source_properties["partitions"] + assert all("@{" not in p for p in partitions) + assert all("{{job.parameters.region}}" in p for p in partitions) + + def test_rewrites_web_activity_url_body_and_headers(self): + """Web activities embed @{} tokens in url/body/headers.""" + web = WebActivity( + **_base("call_api"), + url="https://api.example.com/jobs/@{pipeline().parameters.job_id}", + method="POST", + body='{"run":"@{pipeline().RunId}"}', + headers={"X-Tenant": "@{pipeline().parameters.tenant}"}, + ) + pipeline = Pipeline(name="p", tasks=[web]) + rewritten = rewrite_pipeline_expressions(pipeline) + task = rewritten.tasks[0] + assert "{{job.parameters.job_id}}" in task.url + assert "{{job.run_id}}" in task.body + assert "{{job.parameters.tenant}}" in task.headers["X-Tenant"] + + def test_recurses_into_foreach_inner_activities(self): + inner = CopyActivity( + **_base("inner_copy"), + source_type="AzureSqlSource", + source_properties={"sql": "SELECT * FROM @{item().table_name}"}, + ) + for_each = ForEachActivity( + **_base("loop"), + items_expression="@activity('lookup').output.value", + inner_activities=[inner], + ) + pipeline = Pipeline(name="p", tasks=[for_each]) + rewritten = rewrite_pipeline_expressions(pipeline) + inner_rewritten = rewritten.tasks[0].inner_activities[0] + assert "@{" not in inner_rewritten.source_properties["sql"] + assert "{{input.table_name}}" in inner_rewritten.source_properties["sql"] + + def test_recurses_into_if_condition_branches(self): + true_branch = NotebookActivity( + **_base("true_nb"), + notebook_path="/Shared/promote", + base_parameters={"score": "@{activity('lookup').output.firstRow.quality_score}"}, + ) + false_branch = NotebookActivity( + **_base("false_nb"), + notebook_path="/Shared/remediate", + base_parameters={"score": "@{activity('lookup').output.firstRow.quality_score}"}, + ) + ifc = IfConditionActivity( + **_base("gate"), + op="greaterOrEquals", + left="@activity('lookup').output.firstRow.quality_score", + right="0.95", + if_true_activities=[true_branch], + if_false_activities=[false_branch], + ) + pipeline = Pipeline(name="p", tasks=[ifc]) + rewritten = rewrite_pipeline_expressions(pipeline) + ifc_out = rewritten.tasks[0] + for nb in (ifc_out.if_true_activities[0], ifc_out.if_false_activities[0]): + assert "@{" not in nb.base_parameters["score"] + + def test_recurses_into_switch_cases(self): + case_nb = NotebookActivity( + **_base("case_nb"), + notebook_path="/Shared/h", + base_parameters={"d": "@{pipeline().parameters.dt}"}, + ) + default_nb = NotebookActivity( + **_base("default_nb"), + notebook_path="/Shared/d", + base_parameters={"d": "@{pipeline().parameters.dt}"}, + ) + switch = SwitchActivity( + **_base("sw"), + on_expression="@pipeline().parameters.mode", + cases=[SwitchCase(value="full", activities=[case_nb])], + default_activities=[default_nb], + ) + pipeline = Pipeline(name="p", tasks=[switch]) + rewritten = rewrite_pipeline_expressions(pipeline) + case_out = rewritten.tasks[0].cases[0].activities[0] + default_out = rewritten.tasks[0].default_activities[0] + assert "{{job.parameters.dt}}" in case_out.base_parameters["d"] + assert "{{job.parameters.dt}}" in default_out.base_parameters["d"] + + def test_skips_linked_service_definition_raw_field(self): + """linked_service_definition holds raw ADF input and must not be touched.""" + raw_ls = {"type": "AzureSqlDatabase", "connectionString": "@{pipeline().parameters.cs}"} + nb = NotebookActivity( + **_base("nb"), + notebook_path="/Shared/x", + linked_service_definition=raw_ls, + ) + pipeline = Pipeline(name="p", tasks=[nb]) + rewritten = rewrite_pipeline_expressions(pipeline) + # linked_service_definition stays verbatim — raw ADF passthrough + assert rewritten.tasks[0].linked_service_definition == raw_ls + + def test_unresolved_tokens_remain_and_surface_as_warning(self): + """An expression the parser can't resolve must (a) stay in the output and + (b) be recorded as a warning so the user sees the gap.""" + copy = CopyActivity( + **_base("copy"), + source_type="AzureSqlSource", + source_properties={"sql": "SELECT * FROM @{activity('NonexistentActivity').output.unknownField}"}, + ) + pipeline = Pipeline(name="p", tasks=[copy]) + warnings: list[str] = [] + rewritten = rewrite_pipeline_expressions(pipeline, warnings=warnings) + # Unresolved tokens get replaced by their best-effort dab_ref by the + # parser (activity-output → tasks.X.values...). When even the parser + # cannot produce *anything* it leaves the original @{} verbatim; in + # that case we must surface a warning. The activity-output regex + # *does* match unknownField, so this particular case resolves + # silently. Confirm the more pathological "completely unknown + # function" case raises a warning instead. + del rewritten + + weird_copy = CopyActivity( + **_base("weird"), + source_type="AzureSqlSource", + source_properties={"sql": "SELECT @{nonsense('foo')} FROM t"}, + ) + warnings = [] + rewritten = rewrite_pipeline_expressions(Pipeline(name="p", tasks=[weird_copy]), warnings=warnings) + assert "@{" in rewritten.tasks[0].source_properties["sql"] + assert any("Unresolved ADF expression" in w for w in warnings) + + def test_identifier_fields_are_left_alone(self): + """task_key and name must never be rewritten — they are reference identifiers.""" + nb = NotebookActivity( + name="@{pipeline().parameters.foo}", # intentionally bizarre + task_key="weird_name_with_@{x}_token", + notebook_path="/Shared/x", + ) + pipeline = Pipeline(name="p", tasks=[nb]) + rewritten = rewrite_pipeline_expressions(pipeline) + assert rewritten.tasks[0].name == "@{pipeline().parameters.foo}" + assert rewritten.tasks[0].task_key == "weird_name_with_@{x}_token" + + def test_pipeline_with_no_tokens_returns_equivalent_pipeline(self): + nb = NotebookActivity( + **_base("nb"), + notebook_path="/Shared/etl", + base_parameters={"date": "2026-05-30"}, + ) + pipeline = Pipeline(name="p", tasks=[nb]) + rewritten = rewrite_pipeline_expressions(pipeline) + # Same logical content + assert rewritten.tasks[0].base_parameters == {"date": "2026-05-30"} + + def test_variable_tokens_resolve_via_set_variable_setter(self): + """@{variables('x')} should pick up the SetVariableActivity's task_key.""" + setter = SetVariableActivity( + **_base("set_x"), + variable_name="x", + variable_value="42", + ) + consumer = NotebookActivity( + **_base("nb"), + notebook_path="/Shared/q", + base_parameters={"x_value": "@{variables('x')}"}, + depends_on=[Dependency(task_key="set_x")], + ) + pipeline = Pipeline(name="p", tasks=[setter, consumer]) + rewritten = rewrite_pipeline_expressions(pipeline) + consumer_out = rewritten.tasks[1] + assert "@{" not in consumer_out.base_parameters["x_value"] + # Resolves to task value reference for the setter + assert "tasks.set_x.values.x" in consumer_out.base_parameters["x_value"] + + def test_lookup_source_query_rewritten(self): + lookup = LookupActivity( + **_base("lookup_w"), + source_type="AzureSqlSource", + source_query="SELECT MAX(modified_dt) FROM dbo.@{pipeline().parameters.table_name}", + ) + pipeline = Pipeline(name="p", tasks=[lookup]) + rewritten = rewrite_pipeline_expressions(pipeline) + assert "{{job.parameters.table_name}}" in rewritten.tasks[0].source_query diff --git a/tests/unit/test_preparers.py b/tests/unit/test_preparers.py index 17b318a..0be332a 100644 --- a/tests/unit/test_preparers.py +++ b/tests/unit/test_preparers.py @@ -15,6 +15,7 @@ ExecutePipelineActivity, FilterActivity, ForEachActivity, + IfConditionActivity, LookupActivity, NotebookActivity, Pipeline, @@ -77,6 +78,55 @@ def test_prepare_notebook_task_structure(self): assert prepared.task["notebook_task"]["base_parameters"] == {"env": "dev"} assert prepared.notebooks == [] + def test_prepare_notebook_dispatch_stub_for_unresolved_path(self): + """C-28 (NB-ITER4-001): a NotebookActivity flagged + ``notebook_path_unresolved`` produces a dispatch-stub notebook + (not a NotImplementedError placeholder) plus a + ``dynamic_notebook_dispatch`` SetupTask for SETUP.md.""" + activity = NotebookActivity( + **_make_base("Dispatch", "dispatch"), + notebook_path="", + notebook_path_unresolved=True, + notebook_path_expression="@trim(json(activity('cfg').output.firstRow).notebook_path)", + base_parameters={"env": "dev"}, + ) + prepared = prepare_activity(activity) + assert len(prepared.notebooks) == 1 + content = prepared.notebooks[0].content + assert "dbutils.widgets.get('notebook_path')" in content + assert "dbutils.notebook.run" in content + assert "raise NotImplementedError" not in content + # SETUP.md SetupTask is emitted. + kinds = [st.type for st in prepared.setup_tasks] + assert "dynamic_notebook_dispatch" in kinds + config = next(st.config for st in prepared.setup_tasks if st.type == "dynamic_notebook_dispatch") + assert config["task_key"] == "dispatch" + assert "@trim" in config["expression"] + # The notebook_path widget is registered with an empty default. + assert prepared.task["notebook_task"]["base_parameters"]["notebook_path"] == "" + + def test_prepare_notebook_emits_unresolved_library_setup_task(self): + """C-30 (NB-ITER4-003): unresolved_libraries on the IR emerge as + ``unresolved_library`` SetupTasks the bundler renders in SETUP.md.""" + activity = NotebookActivity( + **_make_base("Run NB", "run_nb"), + notebook_path="/Shared/x", + unresolved_libraries=[ + { + "type": "jar", + "expression": "@concat('/Volumes/x/', pipeline().globalParameters.proj4jLibFileName)", + "missing": ["proj4jLibFileName"], + } + ], + ) + prepared = prepare_activity(activity) + kinds = [st.type for st in prepared.setup_tasks] + assert "unresolved_library" in kinds + config = next(st.config for st in prepared.setup_tasks if st.type == "unresolved_library") + assert config["task_key"] == "run_nb" + assert config["library_type"] == "jar" + assert "proj4jLibFileName" in config["missing"] + def test_prepare_notebook_no_params(self): activity = NotebookActivity( **_make_base("NB", "nb"), @@ -166,6 +216,66 @@ def test_prepare_notebook_resolves_expression_params(self): assert params["env"] == "dev" assert params["trigger_time"] == "{{job.start_time.iso_datetime}}" + def test_prepare_notebook_emits_libraries(self): + libraries = [ + {"whl": "dbfs:/libs/pkg.whl"}, + {"pypi": {"package": "requests"}}, + ] + activity = NotebookActivity( + **_make_base("NB", "nb"), + notebook_path="/Shared/nb", + libraries=libraries, + ) + prepared = prepare_activity(activity) + assert prepared.task["libraries"] == libraries + + def test_prepare_notebook_emits_existing_cluster_id(self): + activity = NotebookActivity( + **{**_make_base("NB", "nb"), "existing_cluster_id": "1234-567890-abcde123"}, + notebook_path="/Shared/nb", + ) + prepared = prepare_activity(activity) + assert prepared.task["existing_cluster_id"] == "1234-567890-abcde123" + assert "job_cluster_key" not in prepared.task + + def test_prepare_notebook_surfaces_parameter_approximations(self): + activity = NotebookActivity( + **_make_base("Score", "score"), + notebook_path="/Shared/score", + base_parameters={"scoring_date": "{{job.start_time.iso_date}}"}, + parameter_approximations=[ + { + "widget_name": "scoring_date", + "raw_expression": "@formatDateTime(utcnow(), 'yyyy-MM-dd')", + "replacement": "{{job.start_time.iso_date}}", + "note": "Mapped ADF `utcnow()` to the Databricks job start time.", + } + ], + ) + prepared = prepare_activity(activity) + assert len(prepared.parameter_approximations) == 1 + approximation = prepared.parameter_approximations[0] + assert approximation.task_key == "score" + assert approximation.widget_name == "scoring_date" + assert approximation.raw_expression == "@formatDateTime(utcnow(), 'yyyy-MM-dd')" + assert approximation.replacement == "{{job.start_time.iso_date}}" + + def test_prepare_notebook_existing_cluster_id_wins_over_default_cluster_bind(self, monkeypatch): + """When a downloaded workspace notebook would otherwise bind to default_cluster, + an explicit existing_cluster_id from the linked service takes precedence.""" + monkeypatch.setattr(workspace_downloader, "_downloads_enabled", True) + monkeypatch.setattr( + "flowx.preparer.activity_preparers.notebook.download_notebook", + lambda path: "# Databricks notebook source\nprint('hi')\n", + ) + activity = NotebookActivity( + **{**_make_base("NB", "nb"), "existing_cluster_id": "9876-543210-zyxwv987"}, + notebook_path="/Shared/ETL/transform", + ) + prepared = prepare_activity(activity) + assert prepared.task["existing_cluster_id"] == "9876-543210-zyxwv987" + assert "job_cluster_key" not in prepared.task + class TestCopyPreparer: def test_prepare_copy_generates_notebook(self): @@ -229,6 +339,19 @@ def test_prepare_spark_python_task(self): assert "scripts/etl.py" in prepared.notebooks[0].relative_path assert "dbfs:/scripts/etl.py" in prepared.notebooks[0].content + def test_prepare_spark_python_emits_libraries(self): + libraries = [ + {"pypi": {"package": "pandas"}}, + {"maven": {"coordinates": "org.example:lib:1.0"}}, + ] + activity = SparkPythonActivity( + **_make_base("Py Task", "py_task"), + python_file="dbfs:/scripts/etl.py", + libraries=libraries, + ) + prepared = prepare_activity(activity) + assert prepared.task["libraries"] == libraries + class TestLookupPreparer: def test_prepare_lookup_generates_notebook(self): @@ -268,6 +391,57 @@ def test_prepare_web_activity_with_auth_creates_secrets(self): assert len(prepared.secrets) >= 1 assert any(s.key == "auth-credential" for s in prepared.secrets) + def test_prepare_web_activity_key_vault_secret_uses_vault_scope_and_secret_name(self): + """C-11 (LSC2-005): an AzureKeyVaultSecret payload preserves the Key + Vault scope and secret name instead of collapsing to the generic + ``auth-credential`` placeholder.""" + activity = WebActivity( + **_make_base("Auth API", "auth_api"), + url="https://api.example.com", + method="POST", + authentication={ + "type": "ServicePrincipal", + "password": { + "type": "AzureKeyVaultSecret", + "store": {"referenceName": "lakeh_ls_keyvault"}, + "secretName": "adapp-auccommonutilssp-secret", + "typeProperties": {"baseUrl": "https://kv.example.net/"}, + }, + }, + ) + prepared = prepare_activity(activity) + assert any( + s.scope == "lakeh_ls_keyvault" and s.key == "adapp-auccommonutilssp-secret" for s in prepared.secrets + ) + # The generic auth-credential placeholder is suppressed when a real + # secret reference is available. + assert not any(s.key == "auth-credential" for s in prepared.secrets) + # C-38 (LSC4-002): the generated notebook must reference the + # resolved AKV scope and key, not the legacy + # ``(task_key, 'auth-credential')`` placeholder. + notebook_content = prepared.notebooks[0].content + assert 'scope="lakeh_ls_keyvault"' in notebook_content + assert 'key="adapp-auccommonutilssp-secret"' in notebook_content + assert 'key="auth-credential"' not in notebook_content + + def test_prepare_web_activity_credential_reference_emits_setup_note(self): + """C-11: CredentialReference (managed identity) routes to a SetupTask + instead of fabricating a static secret placeholder.""" + activity = WebActivity( + **_make_base("Auth API", "auth_api"), + url="https://api.example.com", + method="POST", + authentication={ + "type": "MSI", + "credential": {"referenceName": "msi_credential"}, + }, + ) + prepared = prepare_activity(activity) + manual = [t for t in prepared.setup_tasks if t.type == "manual_credential"] + assert manual, "credential reference must surface a manual_credential SetupTask" + # No static placeholder secret emitted for managed-identity auth. + assert not any(s.key == "auth-credential" for s in prepared.secrets) + class TestDeletePreparer: def test_prepare_delete_generates_notebook(self): @@ -396,6 +570,288 @@ def test_prepare_for_each_wraps_inner(self): assert prepared.task["for_each_task"]["concurrency"] == 10 assert prepared.task["for_each_task"]["inputs"] == "@output.value" + def test_prepare_for_each_uses_ir_bridge_for_variable_based_split(self): + """C-31 (CF4-001): when the items expression references a + ``@variables('X')`` setter, the translator captures the bridge + code on the IR while the variable_cache is populated. The + preparer must consume that IR-supplied bridge rather than + re-resolving against an empty TranslationContext (which used to + silently fail and ship the raw @split string as inputs).""" + inner = WaitActivity(**_make_base("Inner", "inner"), wait_time_seconds=1) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@split(variables('fecha'),',')", + inputs_bridge_notebook_code=( + "str(dbutils.jobs.taskValues.get(taskKey='_init_fecha', key='fecha')).split(str(','))" + ), + inputs_bridge_notebook_imports=[], + inputs_bridge_required_parameters={"fecha": "{{tasks._init_fecha.values.fecha}}"}, + inner_activities=[inner], + concurrency=10, + ) + prepared = prepare_activity(activity) + # The bridge fires off the IR fields even though resolve_expression + # against a bare context would fail to resolve @variables('fecha'). + bridge_keys = [ + t.get("task_key") for t in prepared.extra_tasks if t.get("task_key", "").endswith("_inputs_bridge") + ] + assert bridge_keys == ["loop_inputs_bridge"] + assert prepared.task["for_each_task"]["inputs"] == "{{tasks.loop_inputs_bridge.values.items}}" + bridge_task = next(t for t in prepared.extra_tasks if t["task_key"] == "loop_inputs_bridge") + assert bridge_task["notebook_task"]["base_parameters"]["fecha"] == "{{tasks._init_fecha.values.fecha}}" + + def test_prepare_for_each_bridges_split_items_via_seed_task(self): + """C-08 (CF-iter2-002): @split(, ',') as items_expression must + route through a seed bridge task so for_each_task.inputs is a real + DAB task-value reference rather than a raw ADF expression.""" + inner = WaitActivity(**_make_base("Inner", "inner"), wait_time_seconds=1) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@split(pipeline().parameters.ejecuciones, ';')", + inner_activities=[inner], + concurrency=10, + ) + prepared = prepare_activity(activity) + # Bridge task synthesised ahead of the ForEach. + bridge_keys = [ + t.get("task_key") for t in prepared.extra_tasks if t.get("task_key", "").endswith("_inputs_bridge") + ] + assert bridge_keys == ["loop_inputs_bridge"] + bridge_task = next(t for t in prepared.extra_tasks if t["task_key"] == "loop_inputs_bridge") + assert "notebook_task" in bridge_task + assert "ejecuciones" in bridge_task["notebook_task"]["base_parameters"] + # inputs now reference the bridge task value, not the @split string. + assert prepared.task["for_each_task"]["inputs"] == "{{tasks.loop_inputs_bridge.values.items}}" + # ForEach depends on the bridge so the value is materialised first. + assert any(dep.get("task_key") == "loop_inputs_bridge" for dep in prepared.task.get("depends_on") or []) + + def test_for_each_with_inner_if_condition_carries_branches(self): + """Change foreach-inner-extra-tasks (P0): CF-001. + + When the ForEach has multiple children and one of them is an + IfCondition / Switch, the branch bodies live in the child's + extra_tasks. The preparer must extend inner_tasks with those + so they land in the inner-job, not get dropped. + """ + from flowx.models.ir import IfConditionActivity + + # IfCondition with two branch tasks. + true_act = WaitActivity(**_make_base("TrueWait", "true_wait"), wait_time_seconds=1) + false_act = WaitActivity(**_make_base("FalseWait", "false_wait"), wait_time_seconds=2) + if_act = IfConditionActivity( + **_make_base("If_Condition1", "if_condition1"), + op="EQUAL_TO", + left="@item().x", + right="1", + if_true_activities=[true_act], + if_false_activities=[false_act], + ) + sibling = WaitActivity(**_make_base("Sibling", "sibling"), wait_time_seconds=3) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[if_act, sibling], + concurrency=5, + ) + prepared = prepare_activity(activity) + assert prepared.inner_workflows, "should escalate to sub-job" + inner_wf = prepared.inner_workflows[0] + task_keys = {t["task_key"] for t in inner_wf.tasks} + # Branch tasks survive alongside the condition task. + assert "true_wait" in task_keys + assert "false_wait" in task_keys + assert "sibling" in task_keys + + def test_for_each_inner_workflow_carries_cluster_hints_from_inner_activity(self): + """LSC3-001: ForEach inner-job PreparedWorkflow must carry cluster + hints lifted from nested NotebookActivity.cluster so the inner job's + default_cluster picks up LS-derived spark_env_vars / custom_tags / + driver_node_type_id. + """ + inner_nb = NotebookActivity( + **_make_base("InnerNB", "inner_nb"), + notebook_path="/Shared/ETL/inner", + base_parameters={}, + ) + inner_nb.cluster = { + "spark_version": "16.4.x-scala2.12", + "node_type_id": "Standard_D4s_v3", + "driver_node_type_id": "Standard_D8s_v3", + "spark_env_vars": {"PYSPARK_PYTHON": "/databricks/python3/bin/python3"}, + "custom_tags": {"DigitalCase": "X"}, + } + sibling = WaitActivity(**_make_base("Sibling", "sibling"), wait_time_seconds=1) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[inner_nb, sibling], + concurrency=5, + ) + prepared = prepare_activity(activity) + assert prepared.inner_workflows + inner_wf = prepared.inner_workflows[0] + # The cluster hint from inner_nb propagates through to the inner + # workflow so _infer_bundle_cluster_extras picks it up. + assert inner_wf.cluster_hints, "inner workflow must carry cluster hints" + hint = inner_wf.cluster_hints[0] + assert hint["driver_node_type_id"] == "Standard_D8s_v3" + assert hint["spark_env_vars"]["PYSPARK_PYTHON"] == "/databricks/python3/bin/python3" + assert hint["custom_tags"]["DigitalCase"] == "X" + + def test_for_each_single_child_inner_workflow_carries_cluster_hints(self): + """LSC3-001 single-child escalation path equally must propagate + cluster hints from the single nested IfCondition-wrapped activity. + """ + inner_nb = NotebookActivity( + **_make_base("InnerNB", "inner_nb"), + notebook_path="/Shared/ETL/inner", + base_parameters={}, + ) + inner_nb.cluster = {"custom_tags": {"DigitalCase": "Y"}} + if_act = IfConditionActivity( + **_make_base("If1", "if1"), + op="EQUAL_TO", + left="@item().x", + right="1", + if_true_activities=[inner_nb], + if_false_activities=[], + ) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[if_act], + concurrency=2, + ) + prepared = prepare_activity(activity) + assert prepared.inner_workflows + inner_wf = prepared.inner_workflows[0] + assert inner_wf.cluster_hints, "single-child escalation must propagate cluster hints" + assert inner_wf.cluster_hints[0]["custom_tags"]["DigitalCase"] == "Y" + + def test_for_each_with_single_child_if_condition_escalates_to_subjob(self): + """Single-child IfCondition forces the sub-job path so branches survive.""" + from flowx.models.ir import IfConditionActivity + + true_act = WaitActivity(**_make_base("Hot", "hot"), wait_time_seconds=1) + false_act = WaitActivity(**_make_base("Cold", "cold"), wait_time_seconds=2) + if_act = IfConditionActivity( + **_make_base("Maybe", "maybe"), + op="EQUAL_TO", + left="@item().x", + right="1", + if_true_activities=[true_act], + if_false_activities=[false_act], + ) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[if_act], + concurrency=2, + ) + prepared = prepare_activity(activity) + # Branch bodies require a sub-job (for_each_task.task is a single task). + assert prepared.inner_workflows + inner_wf = prepared.inner_workflows[0] + task_keys = {t["task_key"] for t in inner_wf.tasks} + assert "hot" in task_keys + assert "cold" in task_keys + + +class TestCrossForEachVariableReadDetection: + """Change fix-cross-foreach-variable-read-warning (P1): VAREX3-003.""" + + def test_set_var_in_foreach_read_by_sibling_emits_setup_task(self): + """When a SetVariable for `continue` lives only inside a ForEach and + a sibling IfCondition reads @variables('continue'), prepare_workflow + must emit a manual_variable_rollup SetupTask naming the variable + and the parent ForEach.""" + # Inside the ForEach: a SetVariable that mutates `continue`. + setter = SetVariableActivity( + **_make_base("Mark Continue", "mark_continue"), + variable_name="continue", + variable_value="false", + ) + loop = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[setter], + concurrency=2, + ) + sibling = IfConditionActivity( + **_make_base("CheckCont", "check_cont"), + op="EQUAL_TO", + left="@variables('continue')", + right="true", + if_true_activities=[], + if_false_activities=[], + ) + pipeline = Pipeline(name="cross_foreach_pipe", tasks=[loop, sibling]) + wf = prepare_workflow(pipeline) + rollups = [st for st in wf.setup_tasks if st.type == "manual_variable_rollup"] + assert len(rollups) == 1 + config = rollups[0].config + assert config["variable_name"] == "continue" + assert config["parent_foreach"] == "loop" + + def test_set_var_with_parent_scope_setter_emits_no_warning(self): + """When the variable is ALSO set at the parent scope, no warning.""" + parent_setter = SetVariableActivity( + **_make_base("Init", "init_cont"), + variable_name="continue", + variable_value="true", + ) + inner_setter = SetVariableActivity( + **_make_base("ResetInside", "reset_inside"), + variable_name="continue", + variable_value="false", + ) + loop = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@output.value", + inner_activities=[inner_setter], + concurrency=2, + ) + sibling = IfConditionActivity( + **_make_base("CheckCont", "check_cont"), + op="EQUAL_TO", + left="@variables('continue')", + right="true", + if_true_activities=[], + if_false_activities=[], + ) + pipeline = Pipeline( + name="parent_scope_pipe", + tasks=[parent_setter, loop, sibling], + ) + wf = prepare_workflow(pipeline) + rollups = [st for st in wf.setup_tasks if st.type == "manual_variable_rollup"] + assert rollups == [] + + +class TestManualScheduleTimeOfDaySetupTask: + """C-36 (SCHED4-001): a pipeline schedule whose recurrence carries + hours/minutes/weekDays that periodic can't encode emits a + manual_schedule_time_of_day SetupTask so SETUP.md can flag it.""" + + def test_periodic_schedule_with_time_of_day_emits_setup_task(self): + pipeline = Pipeline( + name="every_three_days", + tasks=[], + schedule={ + "kind": "periodic", + "interval": 3, + "unit": "DAYS", + "pause_status": "UNPAUSED", + "time_of_day_note": {"hours": [2]}, + }, + ) + wf = prepare_workflow(pipeline) + tasks = [st for st in wf.setup_tasks if st.type == "manual_schedule_time_of_day"] + assert len(tasks) == 1 + config = tasks[0].config + assert config["pipeline"] == "every_three_days" + assert config["time_of_day_note"] == {"hours": [2]} + class TestExecutePipelinePreparer: def test_prepare_execute_pipeline_task(self): @@ -560,8 +1016,28 @@ def test_prepare_switch_multi_case_chains_conditions(self): # Default hangs off the last case's outcome=false. assert extra_by_key["default_wait"]["depends_on"] == [{"task_key": "route_case_prod", "outcome": "false"}] - def test_prepare_switch_resolves_variables_expression(self): - """Switch on @variables('x') resolves to a DAB task value ref.""" + def test_resolve_switch_on_expression_is_idempotent_for_dab_refs(self): + """C-13 (CF-iter2-004): an already-resolved {{tasks.X.values.Y}} ref + passes through resolve_switch_on_expression unchanged rather than + being re-resolved with an empty context (which would strip globals + and variable_cache).""" + from flowx.preparer.activity_preparers.switch import ( + resolve_switch_on_expression, + ) + + assert resolve_switch_on_expression("{{tasks.x.values.x}}") == "{{tasks.x.values.x}}" + assert resolve_switch_on_expression("{{job.parameters.env}}") == "{{job.parameters.env}}" + # A bare literal passes through unchanged. + assert resolve_switch_on_expression("hello") == "hello" + # Translator-side bridge placeholder is preserved. + assert resolve_switch_on_expression("__BRIDGE__::result") == "__BRIDGE__::result" + + def test_prepare_switch_unresolved_variable_left_as_raw(self): + """C-05 (VAREX-002): when no setter for the variable is known the + Switch on-expression is left as the raw ``@variables(...)`` string + rather than producing a self-referential dangling task ref. C-07 + will eventually bridge this through a hidden task; until then the + raw string is preserved so a SETUP.md note can flag it.""" inner = WaitActivity(**_make_base("CaseWait", "case_wait"), wait_time_seconds=1) activity = SwitchActivity( **_make_base("Route", "route"), @@ -571,9 +1047,9 @@ def test_prepare_switch_resolves_variables_expression(self): ) prepared = prepare_activity(activity) cond = prepared.task["condition_task"] - # Should be resolved to a DAB ref (fallback: variable name used as task key) - assert "tasks." in cond["left"] - assert "sourceType" in cond["left"] + # Without a setter the raw ADF expression is preserved (no + # dangling {{tasks.sourceType.values.sourceType}} placeholder). + assert cond["left"] == "@variables('sourceType')" def test_prepare_switch_resolves_pipeline_param(self): """Switch on @pipeline().parameters.X resolves to a DAB job parameter ref.""" @@ -815,6 +1291,55 @@ def test_prepare_workflow_with_dependencies(self): assert "depends_on" in second_task assert second_task["depends_on"][0]["task_key"] == "first" + def test_prepare_workflow_collects_cluster_hints_from_nested_activities(self): + """C-04 (NB-ITER2-4 / LSC2-001): cluster hints from activities + nested inside IfCondition / Switch / ForEach must surface in the + workflow-level cluster_hints aggregation so the default-cluster + inference picks the LS-intended node type.""" + nested_nb = NotebookActivity( + **_make_base("Inner", "inner"), + notebook_path="/Shared/inner", + ) + # Override the cluster after construction since _make_base sets it None. + nested_nb.cluster = { + "spark_version": "16.4.x-scala2.12", + "num_workers": 0, + "node_type_id": "Standard_D8s_v3", + } + if_act = IfConditionActivity( + **_make_base("IfCond", "ifcond"), + op="EQUAL", + left="x", + right="y", + if_true_activities=[nested_nb], + ) + pipeline = Pipeline(name="nested_cluster", tasks=[if_act]) + wf = prepare_workflow(pipeline) + node_types = [hint.get("node_type_id") for hint in wf.cluster_hints] + assert "Standard_D8s_v3" in node_types + + def test_prepare_workflow_collects_cluster_hints_from_switch_default_branch(self): + """C-04: Switch default_activities cluster hints surface too.""" + nested_nb = NotebookActivity( + **_make_base("DefaultBranch", "default_branch"), + notebook_path="/Shared/default", + ) + nested_nb.cluster = { + "spark_version": "16.4.x-scala2.12", + "num_workers": 0, + "node_type_id": "Standard_D16s_v3", + } + switch_act = SwitchActivity( + **_make_base("Switch", "switch"), + on_expression="x", + cases=[], + default_activities=[nested_nb], + ) + pipeline = Pipeline(name="switch_default", tasks=[switch_act]) + wf = prepare_workflow(pipeline) + node_types = [hint.get("node_type_id") for hint in wf.cluster_hints] + assert "Standard_D16s_v3" in node_types + def test_prepare_workflow_with_retries(self): """Retry settings are carried through.""" pipeline = Pipeline( diff --git a/tests/unit/test_prereqs_writer.py b/tests/unit/test_prereqs_writer.py new file mode 100644 index 0000000..f053618 --- /dev/null +++ b/tests/unit/test_prereqs_writer.py @@ -0,0 +1,76 @@ +"""Unit tests for flowx.bundler.prereqs_writer.""" + +from __future__ import annotations + +from flowx.bundler.prereqs_writer import build_prereqs, render_setup_md +from flowx.models.dab import DabNotebook, SecretInstruction + + +class TestSecretsUnion: + """Change fix-setup-md-secrets-union-with-secret-instructions (P1): LSC3-006.""" + + def test_workflow_secrets_union_with_notebook_scanned_scopes(self): + """SETUP.md secrets section must list both notebook-scanned and + workflow.secrets sources without duplicating any (scope, key) pair.""" + notebooks = [ + DabNotebook( + relative_path="notebooks/x.py", + content=( + "# Databricks notebook source\n" + "auth_token = dbutils.secrets.get(" + 'scope="lakeh_a_pl_operational_sendMail", key="auth-credential")\n' + ), + ) + ] + secret_instructions = [ + SecretInstruction( + scope="lakeh_ls_keyvault", + key="adapp-clientSecret", + value_source="Azure Key Vault: lakeh-kv/adapp-clientSecret", + ), + # Same pair as the notebook scan -- must not be duplicated. + SecretInstruction( + scope="lakeh_a_pl_operational_sendMail", + key="auth-credential", + value_source="duplicate of notebook scan", + ), + ] + prereqs = build_prereqs( + notebooks=notebooks, + tasks=[], + known_bundle_jobs=set(), + secret_instructions=secret_instructions, + ) + # Both scopes present in the union, no duplicate keys. + assert "lakeh_ls_keyvault" in prereqs.secrets + assert "adapp-clientSecret" in prereqs.secrets["lakeh_ls_keyvault"] + assert "lakeh_a_pl_operational_sendMail" in prereqs.secrets + assert prereqs.secrets["lakeh_a_pl_operational_sendMail"] == {"auth-credential"} + + def test_setup_md_lists_unioned_secrets(self): + """SETUP.md Option A renders every (scope, key) from the union.""" + notebooks = [ + DabNotebook( + relative_path="notebooks/x.py", + content=('auth_token = dbutils.secrets.get(scope="scope_from_notebook", key="key_from_notebook")\n'), + ) + ] + secret_instructions = [ + SecretInstruction( + scope="scope_from_workflow", + key="key_from_workflow", + value_source="Azure Key Vault", + ), + ] + prereqs = build_prereqs( + notebooks=notebooks, + tasks=[], + known_bundle_jobs=set(), + secret_instructions=secret_instructions, + ) + md = render_setup_md(prereqs, bundle_name="test_bundle") + # Both (scope, key) pairs must appear in the rendered SETUP.md. + assert "scope_from_notebook" in md + assert "key_from_notebook" in md + assert "scope_from_workflow" in md + assert "key_from_workflow" in md diff --git a/tests/unit/test_resolve_field.py b/tests/unit/test_resolve_field.py index a025c81..c965066 100644 --- a/tests/unit/test_resolve_field.py +++ b/tests/unit/test_resolve_field.py @@ -68,8 +68,9 @@ def test_activity_output_ref(self): assert "tasks.Lookup.values.cnt" in result def test_boolean_value(self): - result = resolve_field(True, _ctx()) - assert result == "True" + # VAREX3-002: Python bool renders lowercase 'true'/'false' to match ADF. + assert resolve_field(True, _ctx()) == "true" + assert resolve_field(False, _ctx()) == "false" def test_variables_with_context(self): result = resolve_field("@variables('runDate')", _ctx(runDate="SetRunDate")) diff --git a/tests/unit/test_translators.py b/tests/unit/test_translators.py index d213604..92d52d4 100644 --- a/tests/unit/test_translators.py +++ b/tests/unit/test_translators.py @@ -50,6 +50,7 @@ def _base_kwargs(name: str = "test_activity") -> dict[str, Any]: "min_retry_interval_millis": None, "depends_on": None, "cluster": None, + "existing_cluster_id": None, } @@ -176,6 +177,495 @@ def test_translate_notebook_no_params(self): assert isinstance(result, NotebookActivity) assert result.base_parameters == {} + def test_translate_notebook_resolves_library_with_globals(self): + """C-01 (NB-ITER2-1, LSC2-004): @concat of literals collapses to a + literal jar path so the library install succeeds.""" + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Run NB", + "DatabricksNotebook", + { + "notebookPath": "/Shared/x", + "libraries": [{"jar": ("@concat('/Volumes/x/', pipeline().globalParameters.libFileName)")}], + }, + ) + # Context with global parameter so the @concat resolves. + ctx = TranslationContext( + global_parameters=MappingProxyType({"libFileName": "my-job.jar"}), + ) + result = translate(activity, _base_kwargs(), ctx, _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + # With C-01 the concat collapses to a literal string -- the bundle + # YAML carries the resolved jar path directly instead of a Python + # source string. + assert result.libraries == [{"jar": "/Volumes/x/my-job.jar"}] + + def test_translate_notebook_resolves_pipeline_param_in_library(self): + """Library entry referencing a single pipeline parameter resolves to a literal.""" + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Run NB", + "DatabricksNotebook", + { + "notebookPath": "/Shared/x", + "libraries": [ + {"jar": "@pipeline().globalParameters.libPath"}, + ], + }, + ) + ctx = TranslationContext( + global_parameters=MappingProxyType({"libPath": "/Volumes/my.jar"}), + ) + result = translate(activity, _base_kwargs(), ctx, _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.libraries == [{"jar": "/Volumes/my.jar"}] + + def test_translate_notebook_passes_libraries_through(self): + from flowx.translator.activity_translators.notebook import translate + + libraries = [ + {"jar": "dbfs:/libs/util.jar"}, + {"whl": "dbfs:/libs/pkg-1.0-py3-none-any.whl"}, + {"pypi": {"package": "requests==2.31.0"}}, + {"maven": {"coordinates": "org.jsoup:jsoup:1.7.2", "exclusions": ["slf4j:slf4j"]}}, + {"cran": {"package": "ada", "repo": "https://cran.us.r-project.org"}}, + ] + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb", "libraries": libraries}, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.libraries == libraries + + def test_translate_notebook_dynamic_path_marks_unresolved(self): + """C-28 (NB-ITER4-001): an expression notebookPath is captured as + ``notebook_path_unresolved`` so the preparer emits a dispatch stub.""" + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Dispatch", + "DatabricksNotebook", + { + "notebookPath": { + "value": "@trim(json(activity('cfg').output.firstRow).notebook_path)", + "type": "Expression", + }, + "baseParameters": {"env": "dev"}, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.notebook_path_unresolved is True + assert result.notebook_path == "" + assert "@trim" in (result.notebook_path_expression or "") + + def test_translate_notebook_unresolved_library_captured(self): + """C-30 (NB-ITER4-003): library jar/whl entries whose @concat + references a missing globalParameter surface as + ``unresolved_libraries`` so SETUP.md can flag them.""" + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Run NB", + "DatabricksNotebook", + { + "notebookPath": "/Shared/x", + "libraries": [{"jar": ("@concat('/Volumes/x/', pipeline().globalParameters.proj4jLibFileName)")}], + }, + ) + ctx = TranslationContext() + result = translate(activity, _base_kwargs(), ctx, _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + # The unresolved entry is captured with the missing identifier. + assert len(result.unresolved_libraries) == 1 + entry = result.unresolved_libraries[0] + assert entry["type"] == "jar" + assert "proj4jLibFileName" in entry["expression"] + assert "proj4jLibFileName" in entry["missing"] + + def test_translate_notebook_captures_utcnow_approximation(self): + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Score", + "DatabricksNotebook", + { + "notebookPath": "/Shared/score", + "baseParameters": { + "scoring_date": {"value": "@formatDateTime(utcnow(), 'yyyy-MM-dd')", "type": "Expression"}, + "env": "dev", + }, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.base_parameters["scoring_date"] == "{{job.start_time.iso_date}}" + assert result.base_parameters["env"] == "dev" + assert len(result.parameter_approximations) == 1 + approximation = result.parameter_approximations[0] + assert approximation["widget_name"] == "scoring_date" + assert approximation["raw_expression"] == "@formatDateTime(utcnow(), 'yyyy-MM-dd')" + assert approximation["replacement"] == "{{job.start_time.iso_date}}" + assert "utcnow" in approximation["note"].lower() + + +class TestCommonAttributes: + def test_existing_cluster_id_extracted_from_linked_service(self): + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="AzureDatabricks_LS", + type="AzureDatabricks", + properties={ + "typeProperties": { + "existingClusterId": "1234-567890-abcde123", + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"AzureDatabricks_LS": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference(reference_name="AzureDatabricks_LS"), + ) + kwargs = _build_base_kwargs(activity, definitions) + assert kwargs["existing_cluster_id"] == "1234-567890-abcde123" + assert kwargs["cluster"] == {"existing_cluster_id": "1234-567890-abcde123"} + + def test_existing_cluster_id_none_when_linked_service_uses_new_cluster(self): + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="AzureDatabricks_LS", + type="AzureDatabricks", + properties={ + "typeProperties": { + "newClusterSparkVersion": "15.4.x-scala2.12", + "newClusterNumOfWorker": 2, + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"AzureDatabricks_LS": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference(reference_name="AzureDatabricks_LS"), + ) + kwargs = _build_base_kwargs(activity, definitions) + assert kwargs["existing_cluster_id"] is None + + def test_linked_service_parameter_overrides_cluster_version(self): + """Change linked-service-parameter-resolution (P0): NB-4, LSC-001.""" + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="APP0001_ls_databricks", + type="AzureDatabricks", + properties={ + "parameters": { + "clusterVersion": { + "type": "string", + "defaultValue": "16.4.x-scala2.12", + }, + }, + "typeProperties": { + "newClusterVersion": "@linkedService().clusterVersion", + "newClusterNumOfWorker": "1", + "newClusterNodeType": "Standard_D4s_v3", + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"APP0001_ls_databricks": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference( + reference_name="APP0001_ls_databricks", + parameters={"clusterVersion": "16.4.x-scala2.12"}, + ), + ) + kwargs = _build_base_kwargs(activity, definitions) + assert kwargs["cluster"] is not None + # No literal ADF expression should leak into the cluster spec. + assert kwargs["cluster"]["spark_version"] == "16.4.x-scala2.12" + # num_workers='1' must coerce to int. + assert kwargs["cluster"]["num_workers"] == 1 + assert isinstance(kwargs["cluster"]["num_workers"], int) + + def test_parameter_default_coerces_bool_string_to_real_bool(self): + """Change expression-resolver-bool-and-numeric-coercion (P1): VAR-006.""" + from flowx.translator.engine import _coerce_parameter_default + + assert _coerce_parameter_default("false", "Bool") is False + assert _coerce_parameter_default("True", "Bool") is True + assert _coerce_parameter_default("FALSE", "boolean") is False + + def test_parameter_default_coerces_int_string_to_int(self): + from flowx.translator.engine import _coerce_parameter_default + + assert _coerce_parameter_default("42", "Int") == 42 + assert _coerce_parameter_default(42, "Int") == 42 + + def test_parameter_default_string_left_alone(self): + from flowx.translator.engine import _coerce_parameter_default + + assert _coerce_parameter_default("hello", "String") == "hello" + + def test_dependency_multi_condition_succeeded_and_failed_maps_to_completed(self): + """Change dependency-multi-condition-mapping (P1): CF-004.""" + from flowx.translator.engine import _map_dependency_conditions + + assert _map_dependency_conditions(["Succeeded"]) == "Succeeded" + assert _map_dependency_conditions(["Failed"]) == "Failed" + assert _map_dependency_conditions(["Completed"]) == "Completed" + assert _map_dependency_conditions(["Skipped"]) == "Skipped" + # [Succeeded, Failed] semantics = "run regardless" -> Completed + assert _map_dependency_conditions(["Succeeded", "Failed"]) == "Completed" + # [Succeeded, Skipped] -> Skipped wins (ALL_DONE downstream) + assert _map_dependency_conditions(["Succeeded", "Skipped"]) == "Skipped" + # [Failed] (multi-element with same) handled in single-item branch. + assert _map_dependency_conditions([]) is None + assert _map_dependency_conditions(None) is None + + def test_ls_param_expression_wrapper_unwrapped_in_custom_tags(self): + """C-02 (NB-ITER2-2 / LSC2-003): Expression-dict-wrapped LS params + must collapse to plain scalars in cluster fields like custom_tags.""" + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="LS", + type="AzureDatabricks", + properties={ + "parameters": { + "digitalCase": {"type": "string", "defaultValue": "APP0001"}, + }, + "typeProperties": { + "newClusterVersion": "16.4.x-scala2.12", + "newClusterNumOfWorker": 0, + "newClusterNodeType": "Standard_D4s_v3", + "newClusterCustomTags": { + "DigitalCase": "@linkedService().digitalCase", + }, + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"LS": linked_service}, + triggers=[], + ) + # Activity supplies the LS param as the {value, type:'Expression'} + # wrapper shape -- the same shape the ADF JSON corpus ships. + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference( + reference_name="LS", + parameters={"digitalCase": {"value": "APP0001", "type": "Expression"}}, + ), + ) + cluster = _build_base_kwargs(activity, definitions)["cluster"] + assert cluster is not None + # custom_tags must be Map[String, String] -- no dict wrapper survives. + assert cluster["custom_tags"] == {"DigitalCase": "APP0001"} + # spark_env_vars likewise stays scalar-valued. + assert "DigitalCase" in cluster["custom_tags"] + assert not isinstance(cluster["custom_tags"]["DigitalCase"], dict) + + def test_ls_param_resolved_against_factory_global_parameters(self): + """C-03 (NB-ITER2-3 / LSC2-002): activity-supplied LS param values + that reference @pipeline().globalParameters.X must collapse to the + factory-provided literal so cluster.spark_version is a real DBR.""" + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="LS", + type="AzureDatabricks", + properties={ + "parameters": { + "clusterVersion": {"type": "string", "defaultValue": "15.4.x-scala2.12"}, + }, + "typeProperties": { + "newClusterVersion": "@linkedService().clusterVersion", + "newClusterNumOfWorker": 0, + "newClusterNodeType": "Standard_D4s_v3", + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"LS": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference( + reference_name="LS", + parameters={ + "clusterVersion": { + "value": "@pipeline().globalParameters.clusterVersion", + "type": "Expression", + }, + }, + ), + ) + context = TranslationContext( + global_parameters=MappingProxyType({"clusterVersion": "16.4.x-scala2.12"}), + ) + cluster = _build_base_kwargs(activity, definitions, context=context)["cluster"] + assert cluster is not None + assert cluster["spark_version"] == "16.4.x-scala2.12" + # The raw @pipeline() expression must not leak into the cluster spec. + assert not cluster["spark_version"].startswith("@") + + def test_ls_param_resolved_against_pipeline_parameters_as_dab_ref(self): + """C-13 (NB-ITER3-002 / LSC3-003 / VAREX3-006): activity-supplied LS + param values that reference @pipeline().parameters.X must collapse to + {{job.parameters.X}} (a dab_ref), valid in custom_tags map values.""" + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="LS", + type="AzureDatabricks", + properties={ + "parameters": { + "digitalCase": {"type": "string", "defaultValue": "APP0001"}, + }, + "typeProperties": { + "newClusterVersion": "16.4.x-scala2.12", + "newClusterNumOfWorker": 0, + "newClusterNodeType": "Standard_D4s_v3", + "newClusterCustomTags": { + "DigitalCase": "@linkedService().digitalCase", + }, + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"LS": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference( + reference_name="LS", + parameters={ + "digitalCase": { + "value": "@pipeline().parameters.digitalCaseCode", + "type": "Expression", + }, + }, + ), + ) + # Pipeline parameters resolver routes via dab_ref kind, not literal. + context = TranslationContext() + cluster = _build_base_kwargs(activity, definitions, context=context)["cluster"] + assert cluster is not None + # The resolver should now substitute the dab_ref so the raw @pipeline + # expression does not leak. + assert cluster["custom_tags"]["DigitalCase"] == "{{job.parameters.digitalCaseCode}}" + assert not cluster["custom_tags"]["DigitalCase"].startswith("@") + + def test_notebook_library_resolves_pipeline_param_dab_ref(self): + """C-13 (NB-ITER3-004): a jar path referencing @pipeline().parameters.X + collapses to {{job.parameters.X}} in the emitted library entry.""" + from flowx.translator.activity_translators.notebook import translate + + activity = _make_activity( + "Run NB", + "DatabricksNotebook", + { + "notebookPath": "/Shared/x", + "libraries": [ + {"jar": "@pipeline().parameters.libName"}, + ], + }, + ) + ctx = TranslationContext() + result = translate(activity, _base_kwargs(), ctx, _EMPTY_DEFS) + assert isinstance(result, NotebookActivity) + assert result.libraries == [{"jar": "{{job.parameters.libName}}"}] + + def test_extended_cluster_fields_propagated(self): + """Change linked-service-cluster-field-coverage (P1): NB-3, LSC-003.""" + from flowx.models.adf_ast import AdfLinkedService + from flowx.translator.engine import _build_base_kwargs + + linked_service = AdfLinkedService( + name="LS", + type="AzureDatabricks", + properties={ + "typeProperties": { + "newClusterVersion": "16.4.x-scala2.12", + "newClusterNumOfWorker": 0, + "newClusterNodeType": "Standard_D4s_v3", + "newClusterDriverNodeType": "Standard_D8s_v3", + "newClusterSparkEnvVars": {"PYSPARK_PYTHON": "/databricks/python3/bin/python3"}, + "newClusterCustomTags": {"DigitalCase": "MyCase"}, + "newClusterInitScripts": [{"workspace": {"destination": "/init.sh"}}], + "dataSecurityMode": "SINGLE_USER", + "clusterLogConf": {"dbfs": {"destination": "dbfs:/cluster-logs"}}, + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={}, + linked_services={"LS": linked_service}, + triggers=[], + ) + activity = _make_activity( + "Run Notebook", + "DatabricksNotebook", + {"notebookPath": "/Shared/nb"}, + linked_service_name=AdfLinkedServiceReference(reference_name="LS"), + ) + cluster = _build_base_kwargs(activity, definitions)["cluster"] + assert cluster is not None + assert cluster["driver_node_type_id"] == "Standard_D8s_v3" + assert cluster["spark_env_vars"]["PYSPARK_PYTHON"] == "/databricks/python3/bin/python3" + assert cluster["custom_tags"]["DigitalCase"] == "MyCase" + assert cluster["init_scripts"] == [{"workspace": {"destination": "/init.sh"}}] + assert cluster["data_security_mode"] == "SINGLE_USER" + assert cluster["cluster_log_conf"] == {"dbfs": {"destination": "dbfs:/cluster-logs"}} + class TestSparkJarTranslator: def test_translate_spark_jar(self): @@ -211,6 +701,22 @@ def test_translate_spark_python(self): assert result.python_file == "dbfs:/scripts/etl.py" assert result.parameters == ["--mode", "batch"] + def test_translate_spark_python_passes_libraries_through(self): + from flowx.translator.activity_translators.spark_python import translate + + libraries = [ + {"egg": "dbfs:/libs/util.egg"}, + {"pypi": {"package": "pandas", "repo": "https://pypi.example.com"}}, + ] + activity = _make_activity( + "Run Python", + "DatabricksSparkPython", + {"pythonFile": "dbfs:/scripts/etl.py", "libraries": libraries}, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, SparkPythonActivity) + assert result.libraries == libraries + class TestLookupTranslator: def test_translate_lookup_first_row(self): @@ -245,6 +751,199 @@ def test_translate_lookup_all_rows(self): assert isinstance(result, LookupActivity) assert result.first_row_only is False + def test_translate_lookup_resolves_json_file_dataset(self): + """Change lookup-file-dataset-support (P0).""" + from flowx.models.adf_ast import AdfDataset + from flowx.translator.activity_translators.lookup import translate + + json_dataset = AdfDataset( + name="ConfigDataset", + type="Json", + properties={ + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileSystem": "configs", + "folderPath": "lookup", + "fileName": "tables.json", + }, + "formatSettings": {"multiLineJson": True}, + }, + }, + linked_service_name="LS", + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={"ConfigDataset": json_dataset}, + linked_services={}, + triggers=[], + ) + activity = _make_activity( + "Read_Configuration", + "Lookup", + { + "source": {"type": "JsonSource"}, + "dataset": { + "referenceName": "ConfigDataset", + "type": "DatasetReference", + }, + "firstRowOnly": False, + }, + ) + result = translate(activity, _base_kwargs(), _context(), definitions) + assert isinstance(result, LookupActivity) + assert result.source_type == "JsonSource" + assert result.first_row_only is False + # Source properties carry the dataset type plus location bits. + assert result.source_properties is not None + assert result.source_properties["dataset_type"] == "Json" + assert result.source_properties["container"] == "configs" + assert result.source_properties["file_name"] == "tables.json" + assert result.source_properties.get("multiLineJson") is True + + def test_translate_lookup_substitutes_dataset_parameter_refs(self): + """C-47 (LSC5-001): a file Lookup whose dataset folderPath references + ``dataset().X`` substitutes the dataset reference's parameter bindings + so the baked path carries no literal ``dataset(`` expression.""" + from flowx.models.adf_ast import AdfDataset + from flowx.translator.activity_translators.lookup import translate + + ds = AdfDataset( + name="arq_ds", + type="Json", + properties={ + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileSystem": "configs", + "folderPath": { + "value": "@toLower(dataset().digitalCase)", + "type": "Expression", + }, + "fileName": {"value": "@dataset().fileName", "type": "Expression"}, + }, + }, + }, + linked_service_name="LS", + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={"arq_ds": ds}, + linked_services={}, + triggers=[], + ) + activity = _make_activity( + "Read_Arq", + "Lookup", + { + "source": {"type": "JsonSource"}, + "dataset": { + "referenceName": "arq_ds", + "type": "DatasetReference", + "parameters": { + "digitalCase": "@pipeline().parameters.digitalCaseCode", + "fileName": "@pipeline().parameters.fileName", + }, + }, + "firstRowOnly": True, + }, + ) + result = translate(activity, _base_kwargs(), _context(), definitions) + assert isinstance(result, LookupActivity) + assert result.source_properties is not None + folder = result.source_properties.get("folder_path", "") + filename = result.source_properties.get("file_name", "") + # The dataset() reference is gone; the pipeline-param binding takes over. + assert "dataset(" not in folder + assert "dataset(" not in filename + assert "{{job.parameters.digitalCaseCode}}" in folder + + +class TestLookupCaseInsensitiveAndLinkedService: + """Change fix-dataset-and-linked-service-case-insensitive-lookup (P1): LSC3-005.""" + + def test_lookup_resolves_dataset_case_insensitively(self): + """ADF identifiers are case-insensitive; a pipeline referencing + 'app0001_a_ds_conf_json' must resolve dataset 'APP0001_a_ds_conf_json'.""" + from flowx.models.adf_ast import AdfDataset, AdfLinkedService + from flowx.translator.activity_translators.lookup import translate + + ds = AdfDataset( + name="APP0001_a_ds_conf_json", + type="Json", + properties={ + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileSystem": "configext", + "folderPath": "settings", + "fileName": "tables.json", + }, + }, + }, + linked_service_name="LS_ABFSS", + ) + ls = AdfLinkedService( + name="LS_ABFSS", + type="AzureBlobFS", + properties={ + "typeProperties": { + "url": "abfss://configext@myacct.dfs.core.windows.net", + }, + }, + ) + definitions = AdfDefinitions( + pipelines=[], + datasets={"APP0001_a_ds_conf_json": ds}, + linked_services={"LS_ABFSS": ls}, + triggers=[], + ) + # NOTE: lowercase reference name in the activity. + activity = _make_activity( + "Read_Conf", + "Lookup", + { + "source": {"type": "JsonSource"}, + "dataset": { + "referenceName": "app0001_a_ds_conf_json", + "type": "DatasetReference", + }, + "firstRowOnly": True, + }, + ) + result = translate(activity, _base_kwargs(), _context(), definitions) + assert isinstance(result, LookupActivity) + assert result.source_properties is not None + assert result.source_properties["dataset_type"] == "Json" + assert result.source_properties["container"] == "configext" + # Linked-service URL surfaces so the code generator can build abfss://... + assert result.source_properties["linked_service_url"] == ("abfss://configext@myacct.dfs.core.windows.net") + + def test_generated_file_lookup_notebook_uses_abfss_path(self): + """LSC3-005 end-to-end: generated file-lookup notebook ships a real + abfss:// default path instead of an empty widget fallback.""" + from flowx.preparer.code_generator import generate_lookup_notebook + + base = _base_kwargs("Read_Conf") + base.pop("existing_cluster_id", None) + activity = LookupActivity( + **base, + source_type="JsonSource", + source_properties={ + "dataset_type": "Json", + "container": "configext", + "folder_path": "settings", + "file_name": "tables.json", + "linked_service_url": "abfss://configext@myacct.dfs.core.windows.net", + }, + first_row_only=True, + ) + content = generate_lookup_notebook(activity) + assert "abfss://configext@myacct.dfs.core.windows.net" in content + assert "tables.json" in content + # spark.sql('') sentinel must not appear for file-source lookups. + assert "spark.sql('')" not in content + class TestWebActivityTranslator: def test_translate_web_activity_get(self): @@ -317,6 +1016,37 @@ def test_translate_execute_pipeline(self): assert result.parameters == {"date": "2024-01-01"} assert result.wait_on_completion is True + def test_translate_execute_pipeline_drops_notebook_code_parameters(self): + """C-09 (VAREX-001): an ExecutePipeline parameter value that resolves + to notebook_code (e.g. @concat('x', pipeline().parameters.Y)) must NOT + ride through as a literal Python source string -- it's dropped from + the parameters dict and surfaced via parameter_approximations.""" + from flowx.translator.activity_translators.execute_pipeline import translate + + activity = _make_activity( + "Run Child", + "ExecutePipeline", + { + "pipeline": {"referenceName": "child_pipeline", "type": "PipelineReference"}, + "parameters": { + "ok_value": "@pipeline().parameters.env", # dab_ref -- kept + "bad_value": { + "value": "@concat('json: ', pipeline().parameters.configFile)", + "type": "Expression", + }, + }, + "waitOnCompletion": True, + }, + ) + result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) + assert isinstance(result, ExecutePipelineActivity) + assert result.parameters == {"ok_value": "{{job.parameters.env}}"} + # bad_value surfaced as a parameter_approximation for SETUP.md. + approximations = result.parameter_approximations + assert any(a.get("widget_name") == "bad_value" for a in approximations) + # Literal Python source must NOT leak into the parameters dict. + assert "dbutils.widgets.get" not in str(result.parameters) + class TestDatabricksJobTranslator: def test_translate_databricks_job(self): @@ -442,6 +1172,56 @@ def test_translate_foreach_sequential(self): assert isinstance(result, ForEachActivity) assert result.concurrency == 1 + def test_translate_foreach_propagates_globals_to_child_context(self): + """C-13 (NB-ITER3-001 / CF3-002 / LSC3-004): ForEach child context + must carry global_parameters and linked_service_parameters so inner + notebooks resolve @pipeline().globalParameters.X to literals.""" + from flowx.translator.activity_translators.for_each import translate + + # Inner notebook whose library jar references a global parameter. + inner_activity = _make_activity( + "InnerNB", + "DatabricksNotebook", + { + "notebookPath": "/Shared/nb", + "libraries": [{"jar": "@pipeline().globalParameters.libPath"}], + }, + ) + activity = _make_activity( + "Loop", + "ForEach", + {"items": "@activity('GetList').output.value"}, + activities=[inner_activity], + ) + + # The parent context carries the global parameter the inner notebook + # needs. We use the real notebook translator inside our mock callback + # so the inner activity is processed exactly as the engine would. + from flowx.translator.activity_translators.notebook import translate as translate_nb + + def _mock_translate(activities, ctx, defs): + results: list[Any] = [] + for child in activities: + results.append(translate_nb(child, _base_kwargs(child.name), ctx, defs)) + return results, ctx + + ctx = TranslationContext( + global_parameters=MappingProxyType({"libPath": "/Volumes/my.jar"}), + ) + result, _ = translate( + activity, + _base_kwargs("Loop"), + ctx, + _EMPTY_DEFS, + translate_activities_fn=_mock_translate, + ) + assert isinstance(result, ForEachActivity) + inner = result.inner_activities[0] + assert isinstance(inner, NotebookActivity) + # The jar should resolve to the literal from global_parameters, not the + # raw @pipeline() expression. + assert inner.libraries == [{"jar": "/Volumes/my.jar"}] + class TestIfConditionTranslator: def test_translate_if_condition_equals(self): @@ -490,6 +1270,174 @@ def test_translate_if_condition_greater(self): assert "tasks.Copy.values.rowsCopied" in result.left assert result.right == "0" + def test_translate_if_condition_empty_bridges_via_notebook(self): + """C-07 (CF-iter2-001 / VAREX-003): @empty(...) operand routes through + a bridge SetVariable task rather than shipping as a raw ADF expression.""" + from flowx.translator.activity_translators.if_condition import translate + + activity = _make_activity( + "Branch", + "IfCondition", + { + "expression": { + "type": "Expression", + "value": "@empty(pipeline().parameters.X)", + } + }, + ) + result, _ = translate(activity, _base_kwargs("Branch"), _context(), _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + # Bridge populated and right operand is False (no longer the legacy '0'). + assert result.bridge_notebook_code is not None + assert "len(" in result.bridge_notebook_code # @empty -> (len(X) == 0) + assert result.op == "NOT_EQUAL" + assert result.right == "False" + # Left operand is a translator placeholder the preparer rewrites. + assert result.left.startswith("__BRIDGE__::") + + def test_translate_if_condition_boolean_variable_uses_lowercase_false(self): + """C-32 (CF4-002): the truthy fallback path emits ``right='false'`` (not + ``'0'``) when the operand is a known-Boolean variable, since C-21 + SetVariable now writes lowercase ``'true'/'false'`` strings.""" + from flowx.translator.activity_translators.if_condition import translate + + # Seed the context with a Boolean default-valued variable so the + # truthy fallback knows the operand renders as 'true'/'false'. + ctx = _context().with_variable("continue", "_init_continue", dab_ref_value="true") + activity = _make_activity( + "Branch", + "IfCondition", + {"expression": {"type": "Expression", "value": "@variables('continue')"}}, + ) + result, _ = translate(activity, _base_kwargs("Branch"), ctx, _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + assert result.op == "NOT_EQUAL" + # C-32: lowercase 'false' (compatible with C-21 SetVariable output); + # was '0' before this change. + assert result.right == "false" + + def test_translate_if_condition_boolean_variable_by_declared_type(self): + """C-41 (CF5-001): a Boolean variable seeded only by a literal default + init task never populates variable_value_cache as a dab_ref, so the + IfCondition fallback must fall back to the declared type and still emit + ``right='false'`` (not the always-true ``'0'``).""" + from flowx.translator.activity_translators.if_condition import translate + + # No dab_ref value cached — only the declared Boolean type is known. + ctx = _context().with_variable_types({"continue": "Boolean"}) + activity = _make_activity( + "Branch", + "IfCondition", + {"expression": {"type": "Expression", "value": "@variables('continue')"}}, + ) + result, _ = translate(activity, _base_kwargs("Branch"), ctx, _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + assert result.op == "NOT_EQUAL" + assert result.right == "false" + + def test_translate_if_condition_boolean_variable_bridges_when_default_literal_known(self): + """C-43 (CF5-001): when a Boolean variable carries a seeded literal + default, the IfCondition operand is recomputed locally via a + BridgeRequest (``left='__BRIDGE__::...'`` + ``bridge_notebook_code``) + rather than left as a parent-job task-value ref the bundler would + blank. This keeps the operand local so an inner-ForEach condition + survives the dangling-ref safety net.""" + from flowx.translator.activity_translators.if_condition import translate + + # Declared Boolean type AND a seeded literal default -> bridge. + ctx = _context().with_variable_types({"continue": "Boolean"}, default_literals={"continue": "true"}) + activity = _make_activity( + "Branch", + "IfCondition", + {"expression": {"type": "Expression", "value": "@variables('continue')"}}, + ) + result, _ = translate(activity, _base_kwargs("Branch"), ctx, _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + assert result.op == "NOT_EQUAL" + assert result.left.startswith("__BRIDGE__::") + assert result.right == "False" + assert result.bridge_notebook_code == "True" + + def test_translate_if_condition_not_of_function_uses_false_right(self): + """C-15 (CF3-003 / VAREX3-004): @not() produces a bridge + task value compared against 'False', not '' or '0', so the condition + can actually evaluate to FALSE against the Python bool the bridge writes.""" + from flowx.translator.activity_translators.if_condition import translate + + activity = _make_activity( + "Branch", + "IfCondition", + { + "expression": { + "type": "Expression", + # @not(empty(...)) — the bridge writes a Python bool for + # the comparison; right operand must be 'False'. + "value": "@not(empty(pipeline().parameters.X))", + } + }, + ) + result, _ = translate(activity, _base_kwargs("Branch"), _context(), _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + assert result.op == "NOT_EQUAL" + # When bridged, right must be 'False' (was '' under the legacy code path). + assert result.right == "False" + assert result.left.startswith("__BRIDGE__::") or result.bridge_notebook_code is not None + + def test_translate_if_condition_truthy_fallback_bridges_with_false_right(self): + """C-15 (CF3-003 / VAREX3-004): the legacy truthy fallback path emits + right='False' when the resolved operand is a bridge placeholder. + Previously emitted right='0', which the bridge's Python bool output + can never satisfy.""" + from flowx.translator.activity_translators.if_condition import translate + + # An expression with a function call that bridges (e.g. @toUpper). + activity = _make_activity( + "Branch", + "IfCondition", + { + "expression": { + "type": "Expression", + "value": "@toUpper(pipeline().parameters.X)", + } + }, + ) + result, _ = translate(activity, _base_kwargs("Branch"), _context(), _EMPTY_DEFS) + assert isinstance(result, IfConditionActivity) + assert result.op == "NOT_EQUAL" + # Either bridge=task value with right='False', or the truthy path - + # both legitimate; ensure right is not '0'. + assert result.right != "0" + + +class TestIfConditionPreparer: + """C-07: preparer rewrites bridge placeholder to the real task value.""" + + def test_prepare_if_condition_emits_bridge_task(self): + from flowx.preparer.activity_preparers.if_condition import prepare + + if_act = IfConditionActivity( + name="Branch", + task_key="branch", + op="NOT_EQUAL", + left="__BRIDGE__::result", + right="False", + bridge_notebook_code="(len(dbutils.widgets.get('X')) == 0)", + bridge_required_parameters={"X": "{{job.parameters.X}}"}, + ) + prepared = prepare(if_act) + # A bridge task is prepended ahead of the condition. + bridge_tasks = [t for t in prepared.extra_tasks if t.get("task_key", "").endswith("_bridge")] + assert len(bridge_tasks) == 1 + bridge_task = bridge_tasks[0] + assert "notebook_task" in bridge_task + assert bridge_task["notebook_task"]["base_parameters"] == {"X": "{{job.parameters.X}}"} + # Condition operand now references the bridge task value. + cond = prepared.task["condition_task"] + assert cond["left"] == "{{tasks.branch_bridge.values.result}}" + assert cond["right"] == "False" + # Condition depends on the bridge task. + assert any(dep.get("task_key") == "branch_bridge" for dep in prepared.task.get("depends_on") or []) + class TestSetVariableTranslator: def test_translate_set_variable_literal(self): @@ -509,9 +1457,41 @@ def test_translate_set_variable_literal(self): # Context should have the variable mapped assert context.get_variable_task_key("status") == "Set_Status" + def test_translate_set_variable_return_value_pairs_resolves_inner(self): + """C-42 (VAREX5-001): a Set Pipeline Return Value list-of-pairs value + whose inner expression references a resolvable variable lowers to a + dab_ref task-value reference instead of being stringified and blanked.""" + from flowx.translator.activity_translators.set_variable import translate + + # Seed the referenced variable so @variables('executionOutputs') + # resolves to its setter task value. + ctx = _context().with_variable("executionOutputs", "set_outputs") + activity = _make_activity( + "Set Return", + "SetVariable", + { + "variableName": "result", + "value": [ + { + "key": "result", + "value": { + "type": "Expression", + "content": "@variables('executionOutputs')", + }, + } + ], + }, + ) + result, _ = translate(activity, _base_kwargs("Set_Return"), ctx, _EMPTY_DEFS) + assert isinstance(result, SetVariableActivity) + assert result.value_kind == "dab_ref" + assert "{{tasks." in result.variable_value + def test_translate_set_variable_utcnow(self): from flowx.translator.activity_translators.set_variable import translate + # ``utcNow('yyyy-MM-dd')`` now maps to a DAB dynamic value, so the + # SetVariable result is dab_ref rather than notebook_code. activity = _make_activity( "SetRunDate", "SetVariable", @@ -519,6 +1499,68 @@ def test_translate_set_variable_utcnow(self): ) result, context = translate(activity, _base_kwargs("SetRunDate"), _context(), _EMPTY_DEFS) assert isinstance(result, SetVariableActivity) + assert result.value_kind == "dab_ref" + assert result.variable_value == "{{job.start_time.iso_date}}" + + def test_translate_set_variable_split_subscript_lowers_to_notebook_code(self): + """C-33 (VAREX4-001): ``split(...)[N]`` previously left value_kind + stamped as 'literal' with the raw @concat text; now it lowers to + notebook_code so the SetVariable notebook computes the value.""" + from flowx.translator.activity_translators.set_variable import translate + + activity = _make_activity( + "SetPart", + "SetVariable", + { + "variableName": "year", + "value": { + "type": "Expression", + "value": "@split(pipeline().parameters.referenceDate,'/')[0]", + }, + }, + ) + result, _ = translate(activity, _base_kwargs("SetPart"), _context(), _EMPTY_DEFS) + assert isinstance(result, SetVariableActivity) + assert result.value_kind == "notebook_code" + assert result.notebook_code is not None + assert ".split(str('/'))" in result.notebook_code + + def test_translate_set_variable_unresolved_expression_blanks_value(self): + """C-33 (VAREX4-001 / CF4-003): an ADF expression the resolver + cannot lower no longer ships as value_kind='literal' with the raw + @-expression. The value is blanked, value_kind='unresolved', and + raw_expression captures the original text for SETUP.md.""" + from flowx.translator.activity_translators.set_variable import translate + + activity = _make_activity( + "SetX", + "SetVariable", + { + "variableName": "x", + "value": { + "type": "Expression", + # No handler exists for foo(...) so the resolver returns None. + "value": "@foo(pipeline().parameters.bar)", + }, + }, + ) + result, _ = translate(activity, _base_kwargs("SetX"), _context(), _EMPTY_DEFS) + assert isinstance(result, SetVariableActivity) + assert result.value_kind == "unresolved" + assert result.variable_value == "" + assert result.raw_expression == "@foo(pipeline().parameters.bar)" + + def test_translate_set_variable_utcnow_unknown_format(self): + from flowx.translator.activity_translators.set_variable import translate + + # Unrecognised format falls back to the legacy notebook_code path. + activity = _make_activity( + "SetRunDate", + "SetVariable", + {"variableName": "runDate", "value": {"type": "Expression", "value": "@utcNow('yyyyMMdd')"}}, + ) + result, context = translate(activity, _base_kwargs("SetRunDate"), _context(), _EMPTY_DEFS) + assert isinstance(result, SetVariableActivity) assert result.value_kind == "notebook_code" assert result.notebook_code is not None assert "strftime" in result.notebook_code @@ -595,6 +1637,416 @@ def _mock_translate(activities, context, definitions): assert result.cases[1].value == "incremental" assert len(result.default_activities) == 1 + def test_translate_switch_function_call_routes_through_bridge(self): + """C-07 (CF-iter2-001 / CF-iter2-003): @toUpper(coalesce(...)) on the + Switch on-expression lowers to a bridge SetVariable task rather than + shipping as a raw ADF expression.""" + from flowx.translator.activity_translators.switch import translate + + activity = _make_activity( + "Route", + "Switch", + { + "on": { + "type": "Expression", + "value": "@toUpper(coalesce(item().type, 'default'))", + }, + "cases": [], + "defaultActivities": [], + }, + ) + result, _ = translate(activity, _base_kwargs("Route"), _context(), _EMPTY_DEFS) + assert isinstance(result, SwitchActivity) + assert result.bridge_notebook_code is not None + assert ".upper()" in result.bridge_notebook_code + # The on-expression carries the translator placeholder so the + # preparer can rewrite it to the bridge task value. + assert result.on_expression.startswith("__BRIDGE__::") + + +class TestVariableInitTasks: + """C-05 (VAREX-002): init SetVariable tasks for default-valued variables.""" + + def test_default_valued_variable_yields_init_task(self): + from flowx.models.adf_ast import AdfPipeline, AdfVariable + + pipeline = AdfPipeline( + name="pl_with_var_default", + activities=[ + _make_activity( + "Echo", + "DatabricksNotebook", + { + "notebookPath": "/Shared/nb", + "baseParameters": { + "uuid": {"value": "@variables('uuid')", "type": "Expression"}, + }, + }, + ), + ], + variables={"uuid": AdfVariable(type="String", default_value="seed-value")}, + ) + definitions = AdfDefinitions(pipelines=[pipeline], datasets={}, linked_services={}, triggers=[]) + report = translate_pipeline(pipeline, definitions) + # An init task is prepended before the regular activities. + task_keys = [t.task_key for t in report.pipeline.tasks] + assert "_init_uuid" in task_keys + init_task = next(t for t in report.pipeline.tasks if t.task_key == "_init_uuid") + assert isinstance(init_task, SetVariableActivity) + assert init_task.variable_name == "uuid" + # Downstream @variables('uuid') routes through the init task value. + notebook_task = next(t for t in report.pipeline.tasks if t.name == "Echo") + assert isinstance(notebook_task, NotebookActivity) + assert notebook_task.base_parameters["uuid"] == "{{tasks._init_uuid.values.uuid}}" + + def test_default_valued_boolean_variable_renders_lowercase(self): + """VAREX3-002: Boolean variable default ``True`` must serialise as + the lowercase string 'true' so downstream ADF + ``@equals(variables('continue'), true)`` evaluates consistently. + Python's title-case ``'True'`` silently inverted comparisons.""" + from flowx.models.adf_ast import AdfPipeline, AdfVariable + + pipeline = AdfPipeline( + name="pl_bool_var", + activities=[], + variables={ + "continue_t": AdfVariable(type="Boolean", default_value=True), + "continue_f": AdfVariable(type="Boolean", default_value=False), + }, + ) + definitions = AdfDefinitions(pipelines=[pipeline], datasets={}, linked_services={}, triggers=[]) + report = translate_pipeline(pipeline, definitions) + init_true = next(t for t in report.pipeline.tasks if t.task_key == "_init_continue_t") + init_false = next(t for t in report.pipeline.tasks if t.task_key == "_init_continue_f") + assert isinstance(init_true, SetVariableActivity) + assert isinstance(init_false, SetVariableActivity) + assert init_true.variable_value == "true" + assert init_false.variable_value == "false" + + def test_set_variable_with_raw_bool_value_renders_lowercase(self): + """VAREX3-002: a SetVariable activity carrying a raw Python ``False`` + as its typeProperties.value must serialise as 'false' (lowercase), + not 'False' (title-case).""" + from flowx.models.adf_ast import AdfPipeline + + pipeline = AdfPipeline( + name="pl_set_var_bool", + activities=[ + _make_activity( + "Reset", + "SetVariable", + {"variableName": "flag", "value": False}, + ), + ], + ) + definitions = AdfDefinitions(pipelines=[pipeline], datasets={}, linked_services={}, triggers=[]) + report = translate_pipeline(pipeline, definitions) + set_var = next(t for t in report.pipeline.tasks if t.name == "Reset") + assert isinstance(set_var, SetVariableActivity) + assert set_var.variable_value == "false" + + def test_default_valued_variable_with_concat_expression(self): + """An @concat defaultValue resolves like a SetVariable value would.""" + from flowx.models.adf_ast import AdfPipeline, AdfVariable + + pipeline = AdfPipeline( + name="pl_var_default_concat", + activities=[], + variables={ + "fullPath": AdfVariable( + type="String", + default_value="@concat('/Volumes/', pipeline().globalParameters.env)", + ) + }, + ) + definitions = AdfDefinitions( + pipelines=[pipeline], + datasets={}, + linked_services={}, + triggers=[], + global_parameters={"env": "prod"}, + ) + report = translate_pipeline(pipeline, definitions) + init_task = next(t for t in report.pipeline.tasks if t.task_key == "_init_fullPath") + assert isinstance(init_task, SetVariableActivity) + # All-literal concat collapses to a literal (C-01 interplay). + assert init_task.value_kind == "literal" + assert init_task.variable_value == "/Volumes/prod" + + +class TestScheduleCompilation: + """C-10 (SCHED-001): map AdfTrigger objects onto Pipeline.schedule.""" + + def _build_definitions(self, trigger_props, *, runtime_state="Started", trigger_type="ScheduleTrigger"): + from flowx.models.adf_ast import AdfPipeline, AdfTrigger + + pipeline = AdfPipeline(name="pl_with_trigger", activities=[]) + props = dict(trigger_props) + props["runtimeState"] = runtime_state + trigger = AdfTrigger( + name="trg", + type=trigger_type, + properties=props, + pipelines=[{"pipelineReference": {"referenceName": "pl_with_trigger"}}], + ) + definitions = AdfDefinitions(pipelines=[pipeline], datasets={}, linked_services={}, triggers=[trigger]) + return pipeline, definitions + + def test_schedule_trigger_daily_at_specific_time(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "schedule": {"hours": [4], "minutes": [30]}, + "timeZone": "UTC", + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule["kind"] == "schedule" + assert report.pipeline.schedule["quartz_cron_expression"] == "0 30 4 * * ?" + assert report.pipeline.schedule["timezone_id"] == "UTC" + assert report.pipeline.schedule["pause_status"] == "UNPAUSED" + + def test_schedule_trigger_derives_time_of_day_from_start_time(self): + """C-44 (SCHED5-001): a Day recurrence with no schedule block derives + the cron hour/minute from ``startTime`` instead of silently defaulting + to midnight.""" + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "startTime": "2023-03-15T21:00:00Z", + "timeZone": "UTC", + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule["quartz_cron_expression"] == "0 0 21 * * ?" + + def test_schedule_trigger_interval_3_days_emits_periodic(self): + """SCHED3-002 + C-36 (SCHED4-001): Day/Week/Month with interval > 1 + emits periodic, AND the time-of-day from the schedule block is + captured as ``time_of_day_note`` so SETUP.md can surface it.""" + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 3, + "schedule": {"hours": [2]}, + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule["kind"] == "periodic" + assert report.pipeline.schedule["interval"] == 3 + assert report.pipeline.schedule["unit"] == "DAYS" + # C-36: the schedule block (``hours: [2]``) is captured as + # ``time_of_day_note`` rather than silently dropped. + assert report.pipeline.schedule["time_of_day_note"] == {"hours": [2]} + + def test_schedule_trigger_interval_2_months_does_not_emit_months_unit(self): + """C-45 (SCHED5-002): a Month recurrence with interval > 1 must never + emit a periodic spec with the invalid DAB unit 'MONTHS' (the + PeriodicTriggerConfigurationTimeUnit enum only has DAYS/HOURS/WEEKS). + Instead it surfaces a manual setup note.""" + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Month", + "interval": 2, + "schedule": {"monthDays": [1]}, + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule.get("unit") != "MONTHS" + # Either a manual setup note or a cron expr, never a MONTHS periodic. + assert report.pipeline.schedule["kind"] in ("manual_setup", "schedule") + + def test_schedule_trigger_interval_2_weeks_emits_periodic(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Week", + "interval": 2, + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule["kind"] == "periodic" + assert report.pipeline.schedule["unit"] == "WEEKS" + assert report.pipeline.schedule["interval"] == 2 + + def test_trigger_carries_per_pipeline_parameter_overrides(self): + """SCHED3-003: parameters on the trigger's pipelineReference entry + must surface on the schedule spec so the bundler can mutate the + matching job.parameter defaults for scheduled runs.""" + from flowx.models.adf_ast import AdfParameter, AdfPipeline, AdfTrigger + + pipeline = AdfPipeline( + name="pl_with_overrides", + activities=[], + parameters={ + "negocio": AdfParameter(type="String", default_value="DEFAULT"), + "applicationName": AdfParameter(type="String", default_value="DEFAULT"), + }, + ) + trigger = AdfTrigger( + name="nightly", + type="ScheduleTrigger", + properties={ + "runtimeState": "Started", + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "schedule": {"hours": [2]}, + } + }, + }, + pipelines=[ + { + "pipelineReference": {"referenceName": "pl_with_overrides"}, + "parameters": { + "negocio": "GLP", + "applicationName": "app0001", + }, + } + ], + ) + definitions = AdfDefinitions(pipelines=[pipeline], datasets={}, linked_services={}, triggers=[trigger]) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + overrides = report.pipeline.schedule.get("parameter_overrides") or {} + assert overrides["negocio"] == "GLP" + assert overrides["applicationName"] == "app0001" + + def test_schedule_trigger_interval_1_day_still_cron(self): + """Interval == 1 stays on the cron path so we keep timezone/hour spec.""" + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "schedule": {"hours": [4], "minutes": [30]}, + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule["kind"] == "schedule" + assert "quartz_cron_expression" in report.pipeline.schedule + + def test_schedule_trigger_runtime_state_stopped_pauses(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "schedule": {"hours": [0], "minutes": [0]}, + "timeZone": "UTC", + } + } + }, + runtime_state="Stopped", + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule["pause_status"] == "PAUSED" + + def test_schedule_trigger_normalises_romance_standard_time(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Day", + "interval": 1, + "schedule": {"hours": [8], "minutes": [0]}, + "timeZone": "Romance Standard Time", + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + # Romance Standard Time -> Europe/Madrid per the IANA map. + assert report.pipeline.schedule["timezone_id"] == "Europe/Madrid" + + def test_schedule_trigger_weekly_with_week_days(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "recurrence": { + "frequency": "Week", + "interval": 1, + "schedule": { + "hours": [9], + "minutes": [0], + "weekDays": ["Monday", "Wednesday", "Friday"], + }, + "timeZone": "UTC", + } + } + } + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule["quartz_cron_expression"] == "0 0 9 ? * MON,WED,FRI" + + def test_tumbling_window_trigger_surfaces_setup_note(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "frequency": "Hour", + "interval": 1, + } + }, + trigger_type="TumblingWindowTrigger", + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule["kind"] == "schedule" + assert report.pipeline.schedule["tumbling"] is True + + def test_blob_events_trigger_maps_to_file_arrival(self): + pipeline, definitions = self._build_definitions( + { + "typeProperties": { + "scope": "/subscriptions/x/y", + "events": ["Microsoft.Storage.BlobCreated"], + } + }, + trigger_type="BlobEventsTrigger", + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule is not None + assert report.pipeline.schedule["kind"] == "file_arrival" + assert report.pipeline.schedule["url"] == "/subscriptions/x/y" + + def test_custom_events_trigger_routed_to_manual_setup(self): + pipeline, definitions = self._build_definitions( + {"typeProperties": {}}, + trigger_type="CustomEventsTrigger", + ) + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.schedule["kind"] == "manual_setup" + class TestTranslateEngine: def test_translate_pipeline_produces_report(self, adf_definitions): From abc8347c4575f0c42e65b02b9e96ad578127cb28 Mon Sep 17 00:00:00 2001 From: service-jira-pub-repo-auto Date: Tue, 16 Jun 2026 01:01:46 +0530 Subject: [PATCH 13/77] Initial commit --- .gitignore | 5 +++++ CODEOWNERS.txt | 0 LICENSE.md | 24 ++++++++++++++++++++++++ NOTICE.md | 3 +++ README.md | 28 ++++++++++++++++++++++++++++ SECURITY.md | 6 ++++++ 6 files changed, 66 insertions(+) create mode 100644 .gitignore create mode 100644 CODEOWNERS.txt create mode 100644 LICENSE.md create mode 100644 NOTICE.md create mode 100644 README.md create mode 100644 SECURITY.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4f25aa2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*conf*.json +.DS_Store +__pycache__ +.idea/ +.env diff --git a/CODEOWNERS.txt b/CODEOWNERS.txt new file mode 100644 index 0000000..e69de29 diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..7e2ee1e --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,24 @@ +## DB license + +**Definitions**. + +Agreement: The agreement between Databricks, Inc., and you governing the use of the Databricks Services, as that term is defined in the Master Cloud Services Agreement (MCSA) located at www.databricks.com/legal/mcsa. + +Licensed Materials: The source code, object code, data, and/or other works to which this license applies. + +**Scope of Use**. You may not use the Licensed Materials except in connection with your use of the Databricks Services pursuant to the Agreement. Your use of the Licensed Materials must comply at all times with any restrictions applicable to the Databricks Services, generally, and must be used in accordance with any applicable documentation. You may view, use, copy, modify, publish, and/or distribute the Licensed Materials solely for the purposes of using the Licensed Materials within or connecting to the Databricks Services. If you do not agree to these terms, you may not view, use, copy, modify, publish, and/or distribute the Licensed Materials. + +**Redistribution**. You may redistribute and sublicense the Licensed Materials so long as all use is in compliance with these terms. In addition: + +- You must give any other recipients a copy of this License; +- You must cause any modified files to carry prominent notices stating that you changed the files; +- You must retain, in any derivative works that you distribute, all copyright, patent, trademark, and attribution notices, excluding those notices that do not pertain to any part of the derivative works; and +- If a "NOTICE" text file is provided as part of its distribution, then any derivative works that you distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the derivative works. + +You may add your own copyright statement to your modifications and may provide additional license terms and conditions for use, reproduction, or distribution of your modifications, or for any such derivative works as a whole, provided your use, reproduction, and distribution of the Licensed Materials otherwise complies with the conditions stated in this License. + +**Termination**. This license terminates automatically upon your breach of these terms or upon the termination of your Agreement. Additionally, Databricks may terminate this license at any time on notice. Upon termination, you must permanently delete the Licensed Materials and all copies thereof. + +**DISCLAIMER; LIMITATION OF LIABILITY.** + +THE LICENSED MATERIALS ARE PROVIDED “AS-IS” AND WITH ALL FAULTS. DATABRICKS, ON BEHALF OF ITSELF AND ITS LICENSORS, SPECIFICALLY DISCLAIMS ALL WARRANTIES RELATING TO THE LICENSED MATERIALS, EXPRESS AND IMPLIED, INCLUDING, WITHOUT LIMITATION, IMPLIED WARRANTIES, CONDITIONS AND OTHER TERMS OF MERCHANTABILITY, SATISFACTORY QUALITY OR FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. DATABRICKS AND ITS LICENSORS TOTAL AGGREGATE LIABILITY RELATING TO OR ARISING OUT OF YOUR USE OF OR DATABRICKS’ PROVISIONING OF THE LICENSED MATERIALS SHALL BE LIMITED TO ONE THOUSAND ($1,000) DOLLARS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE LICENSED MATERIALS OR THE USE OR OTHER DEALINGS IN THE LICENSED MATERIALS. diff --git a/NOTICE.md b/NOTICE.md new file mode 100644 index 0000000..3ab6a0d --- /dev/null +++ b/NOTICE.md @@ -0,0 +1,3 @@ +## Support +Databricks does not offer official support for Databricks Solutions and its repository. +For any issue with this assets or the demos installed, please open an issue using github and the team will have a look on a best effort basis. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7e1abb8 --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +# REPO NAME + +``` +Placeholder + +Fill here a description at a functional level - what is this content doing +``` + +## Video Overview + +Include a GIF overview of what your project does. Use a service like Quicktime, Zoom or Loom to create the video, then convert to a GIF. + + +## Installation + +Include details on how to use and install this content. + +## How to get help + +Databricks support doesn't cover this content. For questions or bugs, please open a GitHub issue and the team will help on a best effort basis. + + +## License + +© 2025 Databricks, Inc. All rights reserved. The source in this notebook is provided subject to the Databricks License [https://databricks.com/db-license-source]. All included or referenced third party libraries are subject to the licenses set forth below. + +| library | description | license | source | +|----------------------------------------|-------------------------|------------|-----------------------------------------------------| diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..75e9821 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,6 @@ +# Security Policy + +## Reporting a Vulnerability + +Please email bugbounty@databricks.com to report any security vulnerabilities. We will acknowledge receipt of your vulnerability and strive to send you regular updates about our progress. If you're curious about the status of your disclosure please feel free to email us again. + From 0595e12d874212fb4bb5a442b17c94dfac4e3100 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Tue, 16 Jun 2026 13:19:21 -0400 Subject: [PATCH 14/77] Initial release --- .build-constraints.txt | 12 +- .claude-plugin/plugin.json | 12 +- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/ISSUE_TEMPLATE/feature_request.yml | 2 +- .github/workflows/push.yml | 14 +- .github/workflows/skill-eval.yml | 30 - .gitignore | 4 +- AGENTS.md | 68 +- CHANGELOG.md | 8 +- Makefile | 2 +- README.md | 39 +- app/README.md | 192 +++ app/app.py | 21 + app/app.yaml | 5 + app/deploy.sh | 96 ++ app/requirements.txt | 10 + docs/README.md | 2 +- docs/app/(home)/page.tsx | 2 +- docs/app/layout.tsx | 4 +- docs/content/docs/architecture.mdx | 90 ++ docs/content/docs/guide.mdx | 41 +- docs/content/docs/index.mdx | 17 +- docs/content/docs/installation.mdx | 135 +- docs/content/docs/meta.json | 3 +- docs/content/docs/options.mdx | 49 +- pyproject.toml | 19 +- requirements.txt | 6 +- scripts/bootstrap.sh | 106 +- skills/flowx-convert/SKILL.md | 434 +++++ .../references/activity-mapping.md | 2 +- .../references/expression-functions.md | 0 skills/flowx-discover/SKILL.md | 273 ++++ skills/flowx-migrate/SKILL.md | 422 +++++ .../references/workflow.md | 24 +- skills/flowx-package/SKILL.md | 344 ++++ skills/flowx-setup/SKILL.md | 172 ++ skills/ingest/SKILL.md | 212 --- skills/migrate/SKILL.md | 324 ---- skills/prepare/SKILL.md | 254 --- skills/setup/SKILL.md | 94 -- skills/translate/SKILL.md | 329 ---- src/AGENTS.md | 16 +- src/flowx/__init__.py | 3 + src/flowx/adapter/__init__.py | 75 + src/flowx/adapter/__main__.py | 792 +++++++++ src/{orchestra => flowx}/adapter/constants.py | 48 +- src/{orchestra => flowx}/adapter/models.py | 115 +- .../adapter/operations.py | 872 +++++++--- .../adapter/predicates.py | 2 +- src/flowx/adapter/session.py | 486 ++++++ src/{orchestra => flowx}/bundler/__init__.py | 0 src/{orchestra => flowx}/bundler/constants.py | 0 .../bundler/dab_writer.py | 329 ++-- .../bundler/inner_job_params.py | 28 +- .../bundler/notebook_writer.py | 0 .../bundler/prereqs_writer.py | 51 +- .../bundler/setup_generator.py | 32 +- src/flowx/mcp/__init__.py | 18 + src/flowx/mcp/__main__.py | 6 + src/flowx/mcp/runner.py | 365 +++++ src/flowx/mcp/server.py | 554 +++++++ src/{orchestra => flowx}/models/__init__.py | 0 src/{orchestra => flowx}/models/adf_ast.py | 6 + src/{orchestra => flowx}/models/dab.py | 0 src/{orchestra => flowx}/models/ir.py | 53 +- src/{orchestra => flowx}/models/motifs.py | 20 +- .../models/source_types.py | 15 +- src/{orchestra => flowx}/motifs/__init__.py | 0 src/{orchestra => flowx}/motifs/collapser.py | 12 +- src/{orchestra => flowx}/motifs/detector.py | 41 +- src/{orchestra => flowx}/parser/__init__.py | 0 src/{orchestra => flowx}/parser/adf_loader.py | 310 +++- .../parser/expression_parser.py | 105 +- .../parser/ir_rewriter.py | 12 +- src/{orchestra => flowx}/preparer/__init__.py | 0 .../preparer/activity_preparers/__init__.py | 0 .../activity_preparers/append_variable.py | 0 .../preparer/activity_preparers/copy.py | 12 +- .../activity_preparers/databricks_job.py | 0 .../preparer/activity_preparers/delete.py | 0 .../activity_preparers/execute_pipeline.py | 0 .../preparer/activity_preparers/filter.py | 0 .../preparer/activity_preparers/for_each.py | 16 +- .../preparer/activity_preparers/helpers.py | 0 .../activity_preparers/if_condition.py | 0 .../preparer/activity_preparers/lookup.py | 0 .../preparer/activity_preparers/motif.py | 96 +- .../preparer/activity_preparers/naming.py | 0 .../preparer/activity_preparers/notebook.py | 0 .../activity_preparers/set_variable.py | 0 .../preparer/activity_preparers/spark_jar.py | 0 .../activity_preparers/spark_python.py | 0 .../preparer/activity_preparers/switch.py | 9 +- .../preparer/activity_preparers/wait.py | 0 .../activity_preparers/web_activity.py | 10 +- .../preparer/code_generator.py | 177 +- src/flowx/preparer/notifications.py | 158 ++ .../preparer/workflow_preparer.py | 84 +- .../preparer/workspace_downloader.py | 187 ++- src/flowx/reporting/__init__.py | 1 + src/flowx/reporting/coverage.py | 117 ++ src/flowx/reporting/dashboard.py | 97 ++ src/flowx/reporting/dashboard_template.json | 538 +++++++ src/flowx/reporting/results.py | 171 ++ .../translator/__init__.py | 0 .../activity_translators/__init__.py | 0 .../activity_translators/append_variable.py | 0 .../translator/activity_translators/copy.py | 25 +- .../activity_translators/databricks_job.py | 0 .../translator/activity_translators/delete.py | 0 .../activity_translators/execute_pipeline.py | 0 .../translator/activity_translators/filter.py | 12 +- .../activity_translators/for_each.py | 8 +- .../activity_translators/if_condition.py | 61 +- .../translator/activity_translators/lookup.py | 30 +- .../activity_translators/notebook.py | 32 +- .../activity_translators/resolve.py | 12 +- .../activity_translators/set_variable.py | 32 +- .../activity_translators/spark_jar.py | 0 .../activity_translators/spark_python.py | 2 +- .../translator/activity_translators/switch.py | 0 .../translator/activity_translators/wait.py | 0 .../activity_translators/web_activity.py | 162 ++ src/{orchestra => flowx}/translator/engine.py | 432 +++-- .../translator/query_analysis.py | 0 src/{orchestra => flowx}/utils.py | 4 +- src/flowx/validate/__init__.py | 27 + src/flowx/validate/bundle_invariants.py | 181 +++ src/flowx/validate/dag_equivalence.py | 408 +++++ src/orchestra/__init__.py | 3 - src/orchestra/adapter/__init__.py | 96 -- src/orchestra/adapter/__main__.py | 597 ------- src/orchestra/adapter/session.py | 450 ------ .../activity_translators/web_activity.py | 104 -- tests/integration/test_end_to_end.py | 2 +- tests/integration/test_path_equivalence.py | 59 + tests/unit/test_adapter.py | 625 ++++--- tests/unit/test_adf_loader.py | 50 + tests/unit/test_bundle_invariants.py | 61 + tests/unit/test_dag_equivalence.py | 220 +++ tests/unit/test_mcp_migrate.py | 117 ++ tests/unit/test_merge_agentic.py | 100 ++ tests/unit/test_motifs.py | 2 +- tests/unit/test_notify.py | 397 +++++ tests/unit/test_param_dedup.py | 36 + tests/unit/test_preparers.py | 56 +- tests/unit/test_profile_report.py | 107 ++ tests/unit/test_reporting_coverage.py | 90 ++ tests/unit/test_reporting_dashboard.py | 101 ++ tests/unit/test_reporting_results.py | 180 +++ tests/unit/test_until_agentic_handler.py | 71 + .../unit/test_web_body_and_param_defaults.py | 71 + tests/unit/test_workspace_downloader.py | 28 + uv.lock | 1432 +++++++++++------ 154 files changed, 11938 insertions(+), 4459 deletions(-) delete mode 100644 .github/workflows/skill-eval.yml create mode 100644 app/README.md create mode 100644 app/app.py create mode 100644 app/app.yaml create mode 100755 app/deploy.sh create mode 100644 app/requirements.txt create mode 100644 docs/content/docs/architecture.mdx mode change 100755 => 100644 scripts/bootstrap.sh create mode 100644 skills/flowx-convert/SKILL.md rename skills/{translate => flowx-convert}/references/activity-mapping.md (99%) rename skills/{translate => flowx-convert}/references/expression-functions.md (100%) create mode 100644 skills/flowx-discover/SKILL.md create mode 100644 skills/flowx-migrate/SKILL.md rename skills/{migrate => flowx-migrate}/references/workflow.md (93%) create mode 100644 skills/flowx-package/SKILL.md create mode 100644 skills/flowx-setup/SKILL.md delete mode 100644 skills/ingest/SKILL.md delete mode 100644 skills/migrate/SKILL.md delete mode 100644 skills/prepare/SKILL.md delete mode 100644 skills/setup/SKILL.md delete mode 100644 skills/translate/SKILL.md create mode 100644 src/flowx/__init__.py create mode 100644 src/flowx/adapter/__init__.py create mode 100644 src/flowx/adapter/__main__.py rename src/{orchestra => flowx}/adapter/constants.py (51%) rename src/{orchestra => flowx}/adapter/models.py (72%) rename src/{orchestra => flowx}/adapter/operations.py (51%) rename src/{orchestra => flowx}/adapter/predicates.py (99%) create mode 100644 src/flowx/adapter/session.py rename src/{orchestra => flowx}/bundler/__init__.py (100%) rename src/{orchestra => flowx}/bundler/constants.py (100%) rename src/{orchestra => flowx}/bundler/dab_writer.py (83%) rename src/{orchestra => flowx}/bundler/inner_job_params.py (95%) rename src/{orchestra => flowx}/bundler/notebook_writer.py (100%) rename src/{orchestra => flowx}/bundler/prereqs_writer.py (94%) rename src/{orchestra => flowx}/bundler/setup_generator.py (91%) create mode 100644 src/flowx/mcp/__init__.py create mode 100644 src/flowx/mcp/__main__.py create mode 100644 src/flowx/mcp/runner.py create mode 100644 src/flowx/mcp/server.py rename src/{orchestra => flowx}/models/__init__.py (100%) rename src/{orchestra => flowx}/models/adf_ast.py (96%) rename src/{orchestra => flowx}/models/dab.py (100%) rename src/{orchestra => flowx}/models/ir.py (93%) rename src/{orchestra => flowx}/models/motifs.py (93%) rename src/{orchestra => flowx}/models/source_types.py (56%) rename src/{orchestra => flowx}/motifs/__init__.py (100%) rename src/{orchestra => flowx}/motifs/collapser.py (91%) rename src/{orchestra => flowx}/motifs/detector.py (95%) rename src/{orchestra => flowx}/parser/__init__.py (100%) rename src/{orchestra => flowx}/parser/adf_loader.py (66%) rename src/{orchestra => flowx}/parser/expression_parser.py (93%) rename src/{orchestra => flowx}/parser/ir_rewriter.py (96%) rename src/{orchestra => flowx}/preparer/__init__.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/__init__.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/append_variable.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/copy.py (97%) rename src/{orchestra => flowx}/preparer/activity_preparers/databricks_job.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/delete.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/execute_pipeline.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/filter.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/for_each.py (95%) rename src/{orchestra => flowx}/preparer/activity_preparers/helpers.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/if_condition.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/lookup.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/motif.py (56%) rename src/{orchestra => flowx}/preparer/activity_preparers/naming.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/notebook.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/set_variable.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/spark_jar.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/spark_python.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/switch.py (96%) rename src/{orchestra => flowx}/preparer/activity_preparers/wait.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/web_activity.py (93%) rename src/{orchestra => flowx}/preparer/code_generator.py (90%) create mode 100644 src/flowx/preparer/notifications.py rename src/{orchestra => flowx}/preparer/workflow_preparer.py (86%) rename src/{orchestra => flowx}/preparer/workspace_downloader.py (55%) create mode 100644 src/flowx/reporting/__init__.py create mode 100644 src/flowx/reporting/coverage.py create mode 100644 src/flowx/reporting/dashboard.py create mode 100644 src/flowx/reporting/dashboard_template.json create mode 100644 src/flowx/reporting/results.py rename src/{orchestra => flowx}/translator/__init__.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/__init__.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/append_variable.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/copy.py (96%) rename src/{orchestra => flowx}/translator/activity_translators/databricks_job.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/delete.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/execute_pipeline.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/filter.py (82%) rename src/{orchestra => flowx}/translator/activity_translators/for_each.py (90%) rename src/{orchestra => flowx}/translator/activity_translators/if_condition.py (84%) rename src/{orchestra => flowx}/translator/activity_translators/lookup.py (86%) rename src/{orchestra => flowx}/translator/activity_translators/notebook.py (86%) rename src/{orchestra => flowx}/translator/activity_translators/resolve.py (92%) rename src/{orchestra => flowx}/translator/activity_translators/set_variable.py (79%) rename src/{orchestra => flowx}/translator/activity_translators/spark_jar.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/spark_python.py (95%) rename src/{orchestra => flowx}/translator/activity_translators/switch.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/wait.py (100%) create mode 100644 src/flowx/translator/activity_translators/web_activity.py rename src/{orchestra => flowx}/translator/engine.py (77%) rename src/{orchestra => flowx}/translator/query_analysis.py (100%) rename src/{orchestra => flowx}/utils.py (94%) create mode 100644 src/flowx/validate/__init__.py create mode 100644 src/flowx/validate/bundle_invariants.py create mode 100644 src/flowx/validate/dag_equivalence.py delete mode 100644 src/orchestra/__init__.py delete mode 100644 src/orchestra/adapter/__init__.py delete mode 100644 src/orchestra/adapter/__main__.py delete mode 100644 src/orchestra/adapter/session.py delete mode 100644 src/orchestra/translator/activity_translators/web_activity.py create mode 100644 tests/integration/test_path_equivalence.py create mode 100644 tests/unit/test_bundle_invariants.py create mode 100644 tests/unit/test_dag_equivalence.py create mode 100644 tests/unit/test_mcp_migrate.py create mode 100644 tests/unit/test_merge_agentic.py create mode 100644 tests/unit/test_notify.py create mode 100644 tests/unit/test_param_dedup.py create mode 100644 tests/unit/test_profile_report.py create mode 100644 tests/unit/test_reporting_coverage.py create mode 100644 tests/unit/test_reporting_dashboard.py create mode 100644 tests/unit/test_reporting_results.py create mode 100644 tests/unit/test_until_agentic_handler.py create mode 100644 tests/unit/test_web_body_and_param_defaults.py diff --git a/.build-constraints.txt b/.build-constraints.txt index c6be8d7..ee7a59e 100644 --- a/.build-constraints.txt +++ b/.build-constraints.txt @@ -1,6 +1,6 @@ -hatchling==1.29.0 \ - --hash=sha256:50af9343281f34785fab12da82e445ed987a6efb34fd8c2fc0f6e6630dbcc1b0 \ - --hash=sha256:793c31816d952cee405b83488ce001c719f325d9cda69f1fc4cd750527640ea6 +hatchling==1.30.1 \ + --hash=sha256:161eacafb3c6f91526e92116d21426369f2c36e98c36a864f11a96345ad4ee31 \ + --hash=sha256:eee4fd45357f72ebb3d7a42e5d72cfb5e29ed426d79e8836288926c4258d5f2e packaging==26.2 \ --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 @@ -13,7 +13,7 @@ pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 # via hatchling -trove-classifiers==2026.5.22.10 \ - --hash=sha256:01fe864225726e03efb843827ecabfe319fc4dee8dd66d65b8996cb09be46e2c \ - --hash=sha256:5477e9974e91904fb2cfa4a7581ab6e2f30c2c38d847fd00ed866080748101d5 +trove-classifiers==2026.6.1.19 \ + --hash=sha256:ab4c4ec93cc4a4e7815fa759906e05e6bb3f2fbd92ea0f897288c6a43efd15b3 \ + --hash=sha256:c5132b4b61a829d11cfbd2d72e97f20a45ed6edb95e45c5efdeb5e00836b2745 # via hatchling diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index cf9403a..e41c463 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "flowx", - "version": "0.2.0", + "version": "0.1.0", "description": "Translate Azure Data Factory pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. Deterministic translation for known activity types with agentic fallback.", "author": { "name": "Greg Hansen", @@ -9,10 +9,10 @@ "license": "MIT", "keywords": ["adf", "databricks", "migration", "dabs", "lakeflow", "orchestration", "azure-data-factory"], "skills": [ - "./skills/setup", - "./skills/ingest", - "./skills/translate", - "./skills/prepare", - "./skills/migrate" + "./skills/flowx-setup", + "./skills/flowx-discover", + "./skills/flowx-convert", + "./skills/flowx-package", + "./skills/flowx-migrate" ] } diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 78d82a7..67fd130 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,7 +1,7 @@ # See https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms # and https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-githubs-form-schema name: Bug Report -description: Something is not working with Flowx +description: Something is not working with flowx title: "[BUG]: " labels: ["bug"] body: diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index b8f2257..056de77 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,7 +1,7 @@ # See https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms # and https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-githubs-form-schema name: Feature Request -description: Something new needs to happen with Flowx +description: Something new needs to happen with flowx title: "[FEATURE]: " labels: ["enhancement"] body: diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 71081c5..5063847 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -9,10 +9,11 @@ jobs: ci: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 with: + version: "0.11.2" + checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" python-version: "3.12" - name: Scrub internal proxy URLs from uv.lock run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock @@ -27,10 +28,11 @@ jobs: fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 with: + version: "0.11.2" + checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" python-version: "3.12" - name: Scrub internal proxy URLs from uv.lock run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock diff --git a/.github/workflows/skill-eval.yml b/.github/workflows/skill-eval.yml deleted file mode 100644 index c1a93d6..0000000 --- a/.github/workflows/skill-eval.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: skill-eval - -on: - pull_request: - workflow_dispatch: - -jobs: - integration: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Scrub internal proxy URLs from uv.lock - run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock - - run: uv sync --frozen - - # Azure login for live ADF integration tests. - # Requires AZURE_CREDENTIALS secret configured with a service principal - # that has Reader access to the flowx-rg resource group. - # Tests skip gracefully when credentials are not available. - - name: Azure Login - if: ${{ secrets.AZURE_CREDENTIALS != '' }} - uses: azure/login@v2 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - - - run: make integration diff --git a/.gitignore b/.gitignore index 5bbf192..01bef33 100644 --- a/.gitignore +++ b/.gitignore @@ -33,8 +33,8 @@ fixlog/ .DS_Store Thumbs.db -# Flowx output directories -orchestra_output/ +# flowx output directories +flowx_output/ dab_output/ # Temporary ingest downloads diff --git a/AGENTS.md b/AGENTS.md index 6d5d79e..3bfe895 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ -# AI Agent Guidelines for Flowx +# AI Agent Guidelines for flowx ## Quick Command Reference @@ -10,31 +10,55 @@ make fmt # Format + lint (ruff + mypy) make clean # Remove build artifacts ``` -To run the **plugin skills** (ingest/translate/prepare/migrate) without a uv-based dev setup, +To run the **plugin skills** (discover/convert/package/migrate) without a uv-based dev setup, bootstrap a self-contained virtual environment with pip via the `setup` skill or directly: ```bash -bash scripts/bootstrap.sh # creates .venv and pip-installs requirements.txt -# then run plugin code with src/ on PYTHONPATH: -PYTHONPATH=src .venv/bin/python -m flowx.adapter inputs ingest +bash scripts/bootstrap.sh # creates the venv, pip-installs requirements.txt, writes .migration-venv +# then run plugin code with src/ on PYTHONPATH, using the interpreter from the marker file: +PY="$(cat .migration-venv)" +PYTHONPATH=src "$PY" -m flowx.adapter inputs discover ``` +`bootstrap.sh` creates the venv at `/Workspace/Users//.migration-skills` when running +under Databricks (Genie Code / notebooks; detected via `DATABRICKS_RUNTIME_VERSION`) and at +`/.venv` everywhere else. It writes the resolved interpreter path to +`/.migration-venv`; read that marker file rather than hardcoding an interpreter path. + +### Databricks Serverless / Genie Code Compatibility + +All skills and the bootstrap script are designed to run on **Databricks serverless compute** +(Genie Code, notebook serverless) as well as local machines. Key adaptations: + +- **venv:** `bootstrap.sh` creates the venv at `/Workspace/Users//.migration-skills` + on Databricks and `/.venv` locally, writing the interpreter path to + `/.migration-venv`. It falls back to `--without-pip` + `get-pip.py` when `ensurepip` + is unavailable (standard on serverless images). +- **Auth:** `workspace_downloader.py` auto-detects `DATABRICKS_RUNTIME_VERSION` and writes + `~/.databrickscfg` from `dbruntime.databricks_repl_context` so the SDK can authenticate. +- **CLI:** `databricks bundle validate/deploy` is NOT available on serverless — use the web + terminal or a local CLI session for those steps. + ## Project Overview -Flowx is an agent plugin that translates Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs via Declarative Automation Bundles (DABs). +flowx is an agent plugin that translates Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs via Declarative Automation Bundles (DABs). ## Data Flow ``` -ADF JSON -> Parse (AST) -> Classify (Inventory) -> Translate (IR) -> Prepare (Tasks + Notebooks) -> Bundle (DABs) +ADF JSON -> Parse (AST) -> Classify (Inventory) -> Convert (IR) -> Package (Tasks + Notebooks) -> Bundle (DABs) ``` ## Architecture ### Three-Phase Pipeline -1. **Ingest** -- Parse ADF JSON from UC volumes -> typed AST -> inventory.json -2. **Translate** -- Registry dispatch + topological sort -> Pipeline IR (deterministic + agentic gaps) -3. **Prepare** -- IR -> DAB YAML + generated notebooks + setup scripts +All three phases write into one shared `` (default `./flowx_output`): +the DAB bundle at the top level, kept artifacts under `metadata/`, and transient +intermediates under `.work/` (pruned by `package`). + +1. **Discover** -- Parse ADF JSON from UC volumes -> typed AST -> `metadata/inventory.json` + `metadata/profile_report.csv` + verbatim `metadata/.arm.json` +2. **Convert** -- Registry dispatch + topological sort -> Pipeline IR (deterministic + agentic gaps); transient report at `.work/translation_report.json` +3. **Package** -- IR -> DAB YAML + generated notebooks + setup scripts; prunes `.work/` ### Key Patterns - `@dataclass(slots=True, kw_only=True)` for all models @@ -49,7 +73,7 @@ ADF JSON -> Parse (AST) -> Classify (Inventory) -> Translate (IR) -> Prepare (Ta | `models/adf_ast.py` | Typed AST nodes for ADF definitions | | `models/ir.py` | Databricks intermediate representation | | `models/dab.py` | DAB output schema types | -| `parser/adf_loader.py` | Parses ADF exports, produces inventory.json | +| `parser/adf_loader.py` | Parses ADF exports, produces `metadata/inventory.json` + `metadata/profile_report.csv` | | `parser/expression_parser.py` | Translates ADF expressions (@activity, @pipeline, @variables) | | `translator/engine.py` | Registry dispatch, topological sort, context threading | | `translator/activity_translators/` | One module per deterministic activity type (16 total) | @@ -59,6 +83,9 @@ ADF JSON -> Parse (AST) -> Classify (Inventory) -> Translate (IR) -> Prepare (Ta | `bundler/dab_writer.py` | Generates databricks.yml, job YAML, resources | | `bundler/notebook_writer.py` | Writes generated notebooks to bundle | | `bundler/setup_generator.py` | Setup scripts for UC volumes, secrets, connections | +| `reporting/coverage.py` | Builds per-pipeline coverage rows from `metadata/` | +| `reporting/results.py` | Writes per-run coverage to a UC table (run_id/run_date/run_by) via the SDK | +| `reporting/dashboard.py` | Installs + publishes an AI/BI coverage dashboard over the results table | ## Activity Types @@ -96,3 +123,22 @@ ExecuteDataFlow, SqlServerStoredProcedure, AzureFunction, WebHook, Custom, Execu 6. Move from AGENTIC_TYPES to DETERMINISTIC_TYPES in adf_loader.py 7. Update activity-mapping.md reference 8. Add test fixtures and unit tests + +## MCP server design notes + +Rationale behind non-obvious choices in `src/flowx/mcp/server.py` (kept here so the code carries +only one-line pointers): + +- **`@mcp.tool(structured_output=False)`** — FastMCP derives an `outputSchema` from a tool's + `-> dict` return annotation, but Databricks Genie Code's MCP client rejects tools that declare an + `outputSchema` (a 2025-06-18 spec feature): `tools/list` fails and Genie reports "can't fetch + tools" even though `initialize` succeeded. Suppressing it returns the dict as JSON text instead, + which every client accepts. +- **`build_http_app` returns FastMCP's own app** — it is not mounted inside another Starlette app. + Starlette does not run the lifespan of a *mounted* sub-app, so mounting leaves FastMCP's + StreamableHTTP session manager uninitialized and every `/mcp` request 500s with "Task group is not + initialized". +- **DNS-rebinding protection disabled** (`_transport_security`) — behind the Databricks Apps OAuth + proxy the SDK sees the workspace `Origin` and a proxied `Host: localhost:`, so its Host/Origin + allowlist misfires (403/421) while adding nothing on top of the proxy's authentication. Browser + CORS is a separate concern configured via `FLOWX_ALLOWED_ORIGINS`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d612ff..79d76a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,10 @@ -# Flowx Changelog +# flowx Changelog -All notable changes to Flowx will be documented in this file. +All notable changes to flowx will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). -## [0.0.1] +## [0.1.0] ### Added -- Initial release of the Flowx library +- Initial release of the flowx library diff --git a/Makefile b/Makefile index 419faa1..a99a435 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ lock-dependencies: requirements: uv export --frozen --no-dev --no-emit-project --no-hashes --format requirements-txt -o requirements.txt -precommit: fmt requirements +precommit: fmt lock-dependencies requirements help: @echo "Available targets:" diff --git a/README.md b/README.md index f265657..0ba13d5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ -# Flowx +# flowx ADF to Databricks Lakeflow Jobs translator via Declarative Automation Bundles. -Flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). It deterministically translates known activity types and falls back to agentic LLM-assisted translation for complex or rare types. +flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). It deterministically translates known activity types and falls back to agentic LLM-assisted translation for complex or rare types. ## Architecture ``` - Flowx Pipeline + flowx Pipeline ================== ADF JSON (UC Volumes) | v +------------------+ - | 1. INGEST | Parse ADF ARM/JSON exports - | adf_loader.py | -> Typed AST -> inventory.json + | 1. PROFILE | Parse ADF ARM/JSON exports + | adf_loader.py | -> Typed AST -> metadata/inventory.json +------------------+ | v @@ -43,14 +43,14 @@ Flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline de 2. Run the end-to-end migration: ``` - /flowx:migrate + /flowx:flowx-migrate ``` Or run individual phases: ``` - /flowx:ingest # Parse ADF JSON, produce inventory - /flowx:translate # Deterministic + agentic translation - /flowx:prepare # Generate DABs project + /flowx:flowx-discover # Parse ADF JSON, produce inventory + complexity report + /flowx:flowx-convert # Deterministic + agentic translation + /flowx:flowx-package # Generate DABs project ``` ## Supported ADF Activity Types @@ -95,20 +95,22 @@ Flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline de ## How It Works -### Phase 1: Ingest -Reads ADF JSON definitions from Unity Catalog volumes, normalizes ARM template format, parses into typed AST nodes, and classifies each activity as deterministic, agentic, or unsupported. Produces `inventory.json`. +### Phase 1: Discover +Reads ADF JSON definitions from Unity Catalog volumes, normalizes ARM template format, parses into typed AST nodes, and classifies each activity as deterministic, agentic, or unsupported. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`. -### Phase 2: Translate +### Phase 2: Convert Applies deterministic translators via registry dispatch, resolves dependencies through topological sort, and threads immutable `TranslationContext` through control-flow visitors. Agentic gaps are flagged for LLM-assisted translation. Produces Pipeline IR. -### Phase 3: Prepare +### Phase 3: Package Converts Pipeline IR into a deployable DABs project: `databricks.yml`, per-job YAML resource files, generated Python notebooks, and setup scripts for UC volumes, secrets, and connections. ## Output Format +All three phases write into one shared output directory (default `./flowx_output`): + ``` -dab_output/ - databricks.yml # Bundle configuration +flowx_output/ + databricks.yml # Bundle configuration (package) resources/ jobs/ .yml # One job per ADF pipeline @@ -120,6 +122,13 @@ dab_output/ create_volumes.py # UC volume setup create_secrets.py # Secret scope setup create_connections.py # Connection setup + SETUP.md # Setup instructions (package) + metadata/ + inventory.json # discover: activity inventory + profile_report.csv # profile: per-pipeline complexity report + .arm.json # discover: verbatim original ADF/ARM source + configuration.json # modify: collected configuration answers + .work/ # transient intermediates (translation report, IR, gaps.json); pruned by prepare ``` ## Development diff --git a/app/README.md b/app/README.md new file mode 100644 index 0000000..6f52aa3 --- /dev/null +++ b/app/README.md @@ -0,0 +1,192 @@ +# flowx MCP server (Databricks App) + +Hosts flowx's migration phases and helper operations as [Model Context +Protocol](https://modelcontextprotocol.io) tools so agentic clients (Claude +Code, Claude Desktop, Databricks Genie Code) can drive an ADF → Databricks +Lakeflow migration by calling tools instead of shelling out to a CLI. + +The MCP server itself lives in the flowx package at +[`src/flowx/mcp/`](../src/flowx/mcp); this directory is just the +Databricks App wrapper and deployment tooling. + +## Tool + +The server exposes a **single** MCP tool, `flowx(command, parameters)`, to stay well under +host tool-count limits (e.g. Genie Code's 20-tools-across-all-servers cap). `command` selects the +operation; `parameters` is its keyword-argument dict. + +| `command` | Wraps | Purpose | +|-----------|-------|---------| +| `inputs` | `adapter inputs` | List a phase's input prompts/defaults | +| `discover` | `adapter discover` | Parse ADF JSON, classify activities | +| `convert` | `adapter convert` | ADF activities → Databricks IR | +| `merge_agentic` | `adapter convert --merge-agentic` | Merge agent-produced results into the report | +| `inspect` | `adapter inspect` | Surface pending translation options | +| `apply_answers` | `adapter modify` | Apply answers → stamped IR | +| `materialize_lookup` | `adapter materialize-lookup` | CSV → lookup-values JSON | +| `workspace_paths` | `adapter workspace-paths` | Detect workspace paths / hosts | +| `package` | `adapter package` | Emit the deployable DAB bundle | +| `migrate` | discover→convert→package | Full non-interactive migration | +| `record_results` | `adapter record-results` | Write coverage to a UC table | +| `install_dashboard` | `adapter install-dashboard` | Publish the coverage dashboard | + +Example: `flowx(command="discover", parameters={"adf_source_path": "/Volumes/main/default/adf_export", "output_dir": "./out"})`. + +Each command is a thin bridge over `python -m flowx.adapter` (the same entry point the agent +skills use), then reads back the JSON/CSV artifacts each phase writes — so the MCP surface stays in +lockstep with the tested CLI contract. + +## Run locally + +```bash +pip install -e ".[mcp]" # from the repo root + +# stdio transport (Claude Code / Claude Desktop): +python -m flowx.mcp + +# streamable-HTTP transport (same server Databricks Apps runs): +python -m flowx.mcp --http --port 8000 +# MCP endpoint: http://localhost:8000/mcp health: http://localhost:8000/ +``` + +Register the stdio server with a local MCP client, e.g.: + +```json +{ "mcpServers": { "flowx": { "command": "python", "args": ["-m", "flowx.mcp"] } } } +``` + +## Deploy as a Databricks App (for Genie Code) + +```bash +# Authenticated Databricks CLI (v0.230+) required. +./app/deploy.sh +``` + +`deploy.sh` stages a self-contained bundle in a temporary directory outside the repo (the app entrypoint +plus a vendored copy of the pure-Python `flowx` package), syncs it to your +workspace, and creates/deploys the app (default name **`mcp-flowx`**). + +> **Clone into `/Workspace/Shared`.** `deploy.sh` deploys the app source from +> `/Workspace/Shared/` because the app's service principal **cannot read +> private `/Workspace/Users/` folders** by default. Clone flowx into a Git +> folder under `/Workspace/Shared` (e.g. `/Workspace/Shared/flowx`); if that +> folder is restricted in your workspace, use another all-users location and pass it +> via `APP_SOURCE_PATH`. + +End-to-end, to use it from Genie Code: + +1. **Deploy** with `./app/deploy.sh`. The app is named `mcp-flowx` and deploys the + source from `/Workspace/Shared/mcp-flowx` (override with `APP_SOURCE_PATH`). The + script prints the app URL; the MCP endpoint is `/mcp`. +2. **Grant app access:** give **Can use** on `mcp-flowx` to the users / service + principals that will call it (Apps UI → *Permissions*, or + `databricks apps set-permissions mcp-flowx ...`). +3. **Grant data access:** the app authenticates as its own service principal, so + grant that principal access to the catalogs / schemas / Unity Catalog volumes + the migration reads/writes (and any SQL warehouse used by `record-results` / + `install-dashboard`). +4. **Add it in Genie Code (Agent mode):** open Genie Code **Settings → MCP Servers → + Add Server**, choose **Custom MCP server**, select the `mcp-flowx` app, and + **Save**. The `flowx` tool becomes available immediately. Verify via the + health endpoint `/`. + +> The `mcp-` name prefix also makes the app **auto-listed in the AI Playground**. Genie Code's +> **Custom MCP server** picker selects any Databricks App by name regardless of prefix. + +Genie Code requires a custom MCP app to be (1) in the **same workspace**, (2) reachable +at `https:///mcp`, and (3) **stateless** — this server sets +`stateless_http=True` and adds CORS, so it qualifies. If a browser CORS error appears, +set the app env var `FLOWX_ALLOWED_ORIGINS` to your workspace URL and redeploy. +MCP access is capped at **20 tools** across all servers; flowx exposes just **one** tool +(`flowx`, with 12 commands), so it uses a single slot. + +> Run `./app/deploy.sh` from a Databricks CLI session (workspace web terminal or a +> local machine) — `databricks apps` deploy is not available from serverless +> notebook Python. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp) +> and [host a custom MCP server](https://docs.databricks.com/aws/en/generative-ai/mcp/custom-mcp). + +## Troubleshooting + +**Genie Code can't connect / add the server (is it CORS or the server?)** — Tell them apart +from the app's logs (`databricks apps logs ` or the app UI): + +- **Server-side error (not CORS):** the logs show a `500` / `RuntimeError: Task group is not + initialized` on `POST /mcp`. That means the StreamableHTTP session manager never started — + it happens if the MCP app is *mounted inside another Starlette app* (whose lifespan doesn't + run the sub-app). The server now avoids this by serving FastMCP's own app directly; make sure + you redeployed the current `app/`. Sanity-check the app is up with `curl /` + (expect `{"status":"ok"}`). +- **CORS:** the request reaches the server fine but the **browser console** shows a CORS error + (blocked by `Access-Control-Allow-Origin`), with no corresponding 500 in the app logs. Set the + app env var `FLOWX_ALLOWED_ORIGINS` to your workspace URL and redeploy. + +**Server connects but Genie Code "fails to fetch tools"** — the connection (`initialize`) succeeds, +but `tools/list` comes back empty / errors. This is a **schema-compatibility** problem, not a +transport one: Genie Code's MCP client rejects tools that declare an `outputSchema` (the structured- +output feature from the 2025-06-18 spec). FastMCP derives one automatically from a tool's return-type +annotation, so the `flowx` tool registers with `@mcp.tool(structured_output=False)` to suppress +it (the result is still returned as JSON text). If you see this after customizing the server, make +sure no tool emits an `outputSchema` — check with `curl`-ing a `tools/list` request or inspecting +`mcp.list_tools()`. + +**`403 Forbidden` with `Invalid Origin header`, or `421 Misdirected Request` with `Invalid Host +header: localhost:8000`** (from `transport_security.py` in the logs) — this is the MCP SDK's +DNS-rebinding protection, **not** CORS. Behind the Databricks Apps OAuth proxy the app sees the +workspace `Origin` and a proxied `Host: localhost:8000`, so that allowlist check misfires. The +server now **disables** DNS-rebinding protection (the OAuth proxy already authenticates every +request); just redeploy the current `app/`. Note: `FLOWX_ALLOWED_ORIGINS` controls **only** +browser CORS now — it does not enable or affect this Host/Origin check. + + + +**`mkdir: cannot create directory ...: Permission denied`** — On Databricks (Genie web +terminal / serverless), `/tmp` and `$TMPDIR` are often not writable, while the `/Workspace` +filesystem where the repo lives is. `deploy.sh` stages the bundle in the **repo's parent +directory** first (e.g. `/Workspace/Shared`, which is writable and outside the git repo), +falling back to `$TMPDIR` / `/tmp` / `$HOME` for local runs. If it still can't find a writable +base, set `TMPDIR` to a writable path and re-run. (Staging is never placed inside the repo, +so `databricks sync` won't drop files via the repo's `.gitignore`.) + +**`Error: please specify target`** — The Databricks CLI (v0.298+) makes `sync` and +`apps deploy` bundle-aware: if a `databricks.yml` is discoverable in the working +directory or any parent (for example a generated `flowx_output/databricks.yml`, +or one in your workspace home), the CLI loads that bundle and—when it has multiple +targets with no default—aborts with this error before deploying. `deploy.sh` already +runs every CLI call from a throwaway directory to avoid this; if you invoke the CLI +manually, do the same (or pass `--target `), and don't run it from inside a +generated bundle directory. + +## Inputs and outputs on a hosted app + +A Databricks App can't read the user's workspace / UC Volume files (only the container's +ephemeral disk is local; `/Volumes/...` is **not** auto-mounted). The MCP surface handles this +by passing data **inline** through the tool, since the calling agent *can* read those files: + +- **Input — small jobs:** pass the ADF JSON via `adf_definitions` — a mapping of relative path → JSON + content mirroring the ADF Git-export layout (`pipeline/…`, `dataset/…`, `linkedService/…`, + `trigger/…`); a single ARM-template object is also accepted. The server materializes it to a temp + dir. Capped at ~5 MB (`FLOWX_MAX_INLINE_BYTES`) since it flows through the agent's context. +- **Input — large factories (recommended):** point the server at the source by reference, so the + bytes bypass the agent and it scales to thousands of pipelines: + - `adf_volume_path` — a UC Volume directory, downloaded via the SDK **Files API**. + - `adf_workspace_path` — a `/Workspace` directory (e.g. an ADF Git folder), listed and downloaded + via the SDK **Workspace API** (workspace files use the Workspace API, *not* the Files API). + + Grant the app's service principal read on whichever path you use. (`adf_source_path` / `source_dir` + remain for paths the server itself can read — local hosting or a mounted volume.) +- **Output — large bundles (recommended):** `package`/`migrate` write the DAB to the target via the + SDK so the contents bypass the agent. Pass `output_volume_path` (uploaded to a UC Volume via the SDK + Files API) **or** `output_workspace_path` (uploaded to a `/Workspace` directory via the SDK Workspace + API, with `ImportFormat.RAW` so files land verbatim rather than as notebooks). Either returns + `bundle_uploaded` (location + file list). Grant the service principal write on the target. +- **Output — small bundles:** without an output path, `package`/`migrate` return the DAB inline as + `bundle = {"files": {relpath: text, …}, "truncated": [...]}` (capped ~2 MB) for the caller to persist. + +## Known constraints / follow-ups + +- **`databricks bundle validate/deploy`** of the *generated* DAB is a separate, + user-driven step (run from a CLI session); it is intentionally not invoked by + these tools. +- **Long-running phases.** Large factories can exceed default client timeouts; + the server-side subprocess timeout is configurable via `FLOWX_MCP_TIMEOUT` + (seconds). diff --git a/app/app.py b/app/app.py new file mode 100644 index 0000000..b12226d --- /dev/null +++ b/app/app.py @@ -0,0 +1,21 @@ +"""Databricks App entry point for the flowx MCP server. + +Databricks Apps run this module's ``command`` from ``app.yaml``. The flowx +package is vendored alongside this file by ``deploy.sh`` (into ``flowx/``), +so it imports directly without a separate install step. + +The module also exposes ``app`` so it can be served with ``uvicorn app:app``. +""" + +import os + +from flowx.mcp.server import build_http_app + +app = build_http_app() + +if __name__ == "__main__": + import uvicorn + + # Databricks Apps inject the port to bind via DATABRICKS_APP_PORT. + port = int(os.environ.get("DATABRICKS_APP_PORT", "8000")) + uvicorn.run(app, host="0.0.0.0", port=port) diff --git a/app/app.yaml b/app/app.yaml new file mode 100644 index 0000000..748afd0 --- /dev/null +++ b/app/app.yaml @@ -0,0 +1,5 @@ +# Databricks App manifest for the flowx MCP server. +# The app serves the MCP streamable-HTTP transport at /mcp and a health check at /. +command: + - python + - app.py diff --git a/app/deploy.sh b/app/deploy.sh new file mode 100755 index 0000000..68a1ca2 --- /dev/null +++ b/app/deploy.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Deploy the flowx MCP server as a Databricks App. +# +# Builds a self-contained source bundle (app entrypoint + vendored flowx +# package), syncs it to the workspace, and creates/deploys the app. +# +# Requirements: Databricks CLI v0.230+ authenticated to the target workspace. +# +# Env overrides: +# APP_NAME App name (default: mcp-flowx). The `mcp-` prefix makes the app +# auto-listed in the AI Playground; Genie Code's "Add Server > Custom MCP +# server" picker can also select any Databricks App by name. +# APP_SOURCE_PATH Workspace source path the app deploys from (default: +# /Workspace/Shared/). It MUST be readable by the app's +# service principal, so it defaults to /Workspace/Shared — NOT a user's +# private /Workspace/Users/ home, which the app SP cannot read. +# DATABRICKS_PROFILE CLI profile to use (default: env/DEFAULT auth) +set -euo pipefail + +APP_NAME="${APP_NAME:-mcp-flowx}" +APP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$APP_DIR/.." && pwd)" + +PROFILE_FLAG=() +if [ -n "${DATABRICKS_PROFILE:-}" ]; then + PROFILE_FLAG=(--profile "$DATABRICKS_PROFILE") +fi + +# Stage the bundle OUTSIDE the repo. `databricks sync` is git-aware and applies the +# enclosing repo's .gitignore: a staging dir inside the repo (e.g. app/.build, which +# .gitignore lists) gets its files excluded, so the deployed source is missing app.py +# and flowx/. A temp dir has no enclosing git repo / .gitignore, so every file syncs. +# +# The CLI is also bundle-aware: if a databricks.yml is discoverable in the working +# directory or any parent (e.g. a generated flowx_output/databricks.yml), `sync` / +# `apps deploy` load it and fail with "Error: please specify target". Running from the +# (databricks.yml-free) staging dir avoids that too. All CLI paths are absolute. +# +# Create the staging dir in a writable location, trying the repo's PARENT first. On +# Databricks (Genie web terminal / serverless) the repo lives on the writable /Workspace +# filesystem while /tmp and $TMPDIR are often unwritable — a bare `mktemp -d` there fails +# with "mkdir: cannot create directory ...: Permission denied". The repo's parent +# (e.g. /Workspace/Shared) is writable AND outside the git repo, so staging there both +# succeeds and avoids the repo's .gitignore (which would otherwise make `databricks sync` +# drop staged files). $TMPDIR/tmp/$HOME are fallbacks for local (non-Databricks) runs. +STAGE_DIR="" +for _base in "$(dirname "$REPO_ROOT")" "${TMPDIR:-}" /tmp /local_disk0/tmp "$HOME"; do + [ -n "$_base" ] && [ -d "$_base" ] && [ -w "$_base" ] || continue + STAGE_DIR="$(mktemp -d "${_base%/}/mcp-flowx-build.XXXXXX" 2>/dev/null)" && break +done +if [ -z "$STAGE_DIR" ]; then + echo "ERROR: could not create a writable staging directory (tried the repo parent, \$TMPDIR, /tmp, \$HOME)." >&2 + echo " Set TMPDIR to a writable local path and re-run." >&2 + exit 1 +fi +trap 'rm -rf "$STAGE_DIR"' EXIT +dbx() { (cd "$STAGE_DIR" && databricks "${PROFILE_FLAG[@]}" "$@"); } + +echo "==> Staging self-contained app bundle in $STAGE_DIR (outside the repo so sync includes every file)" +cp "$APP_DIR/app.py" "$APP_DIR/app.yaml" "$APP_DIR/requirements.txt" "$STAGE_DIR/" +cp -R "$REPO_ROOT/src/flowx" "$STAGE_DIR/flowx" +find "$STAGE_DIR/flowx" -type d -name '__pycache__' -prune -exec rm -rf {} + 2>/dev/null || true + +# Deploy the source from a location the app's service principal can read. A user's +# private /Workspace/Users/ home is NOT readable by the app SP, so default to +# the shared workspace folder. Override with APP_SOURCE_PATH if you use a different +# all-users location. +SOURCE_PATH="${APP_SOURCE_PATH:-/Workspace/Shared/$APP_NAME}" + +echo "==> Ensuring app '$APP_NAME' exists" +if ! dbx apps get "$APP_NAME" >/dev/null 2>&1; then + dbx apps create "$APP_NAME" +fi + +echo "==> Syncing bundle to $SOURCE_PATH" +dbx sync --full "$STAGE_DIR" "$SOURCE_PATH" + +echo "==> Deploying app" +dbx apps deploy "$APP_NAME" --source-code-path "$SOURCE_PATH" + +echo "==> Deployed. App details:" +APP_URL="$(dbx apps get "$APP_NAME" --output json \ + | python3 -c 'import sys, json; print(json.load(sys.stdin).get("url", ""))')" +echo " name: $APP_NAME" +echo " url: ${APP_URL:-(pending — re-run 'databricks apps get $APP_NAME')}" +echo +echo "==> Next steps to use it in Genie Code:" +echo " 1. MCP endpoint: ${APP_URL:-}/mcp" +echo " 2. Grant 'Can use' on the app to the users / service principals that will call it" +echo " (Apps UI > Permissions, or: databricks apps set-permissions $APP_NAME ...)." +echo " 3. Grant that app's service principal access to the catalogs/schemas/volumes the" +echo " migration touches (and any SQL warehouse used by the reporting tools)." +echo " 4. Add it in Genie Code (Agent mode): Settings > MCP Servers > Add Server >" +echo " Custom MCP server > select '$APP_NAME' > Save. Tools appear immediately." +echo " 5. If a browser CORS error appears, set the app env var FLOWX_ALLOWED_ORIGINS" +echo " to your workspace URL and redeploy." diff --git a/app/requirements.txt b/app/requirements.txt new file mode 100644 index 0000000..fd66841 --- /dev/null +++ b/app/requirements.txt @@ -0,0 +1,10 @@ +# Databricks App dependencies for the flowx MCP server. +# The flowx package itself is vendored next to app.py by deploy.sh, so only +# third-party runtime dependencies are installed here (flowx's own deps plus +# the MCP/HTTP server stack). +mcp>=1.12 +uvicorn>=0.30 +starlette>=0.40 +databricks-sdk>=0.40 +pyyaml>=6.0 +sqlglot>=25.0 diff --git a/docs/README.md b/docs/README.md index 6e9f65b..f151cad 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,4 +1,4 @@ -# Flowx docs +# flowx docs Documentation site for [flowx](https://github.com/ghanse/flowx), built with [fumadocs](https://fumadocs.dev) and deployed to GitHub Pages. diff --git a/docs/app/(home)/page.tsx b/docs/app/(home)/page.tsx index 266cfcf..021cb6b 100644 --- a/docs/app/(home)/page.tsx +++ b/docs/app/(home)/page.tsx @@ -3,7 +3,7 @@ import Link from 'next/link'; export default function HomePage() { return (
-

Flowx

+

flowx

Programmatically translate your data pipelines to Databricks Lakeflow jobs.

diff --git a/docs/app/layout.tsx b/docs/app/layout.tsx index 975d91b..da935fb 100644 --- a/docs/app/layout.tsx +++ b/docs/app/layout.tsx @@ -9,8 +9,8 @@ const inter = Inter({ export const metadata = { title: { - default: 'Flowx', - template: '%s | Flowx', + default: 'flowx', + template: '%s | flowx', }, description: 'Translate Azure Data Factory pipelines to Databricks Lakeflow Jobs.', diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx new file mode 100644 index 0000000..bed4d9e --- /dev/null +++ b/docs/content/docs/architecture.mdx @@ -0,0 +1,90 @@ +--- +title: Architecture +description: How flowx is structured as a set of agent skills and MCP tools. +--- + +import { Callout } from 'fumadocs-ui/components/callout'; + +flowx translates data pipelines into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). A set of Python functions is exposed through two surfaces: **agent skills** invoked through a CLI and **MCP tools** called by an agent session. + +## Three-phase pipeline + +```text +ADF JSON ──▶ discover ──▶ convert ──▶ package ──▶ databricks.yml (DAB) + (parse + (activities (IR → jobs, + classify) → IR) notebooks, setup) +``` + +All three phases share one `output_dir` (default `./flowx_output`): + +| Phase | Module | Reads | Writes | +|---------------|-------------------------|------------------------|------------------------------------------------------------------------------------------| +| **profile** | `parser/adf_loader.py` | ADF JSON exports | `metadata/inventory.json`, `metadata/profile_report.csv`, `metadata/.arm.json` | +| **translate** | `translator/engine.py` | inventory + ADF source | `.work/translation_report.json` (transient IR) | +| **prepare** | `bundler/dab_writer.py` | translation report | `databricks.yml`, `resources/`, `src/`, `setup/` (and prunes `.work/`) | + +Each activity is classified with a `TranslationStrategy`: +* `DETERMINISTIC` (in-process translators) +* `AGENTIC` (LLM-assisted gaps) +* `UNSUPPORTED` + +The reporting layer (`reporting/`) can write per-run coverage to a Unity Catalog table and publish an AI/BI dashboard. + +## Two surfaces over one core + +```text + ┌─────────────────────────────┐ + Agent skills ───▶│ python -m flowx.adapter │───▶ phase modules (discover/ + (discover/..., │ (unified CLI entry point) │ convert/package) + + setup, migrate) └─────────────────────────────┘ adapter operations + ▲ + │ subprocess (same contract) + ┌─────────────────────────────┐ + MCP tool ──────▶│ flowx.mcp (FastMCP) │ + (Claude / Genie) │ 1 tool, 12 commands → │ + │ adapter bridge │ + └─────────────────────────────┘ +``` + +The unified `flowx.adapter` CLI is the single contract. Both surfaces go through it: + +- **Agent skills** (`skills/`) shell out to `python -m flowx.adapter …` directly. +- **MCP tool** (`src/flowx/mcp/`) is a thin bridge: the single `flowx(command, parameters)` tool builds the same adapter arguments for the chosen `command`, runs them as a subprocess via `flowx.mcp.runner`, then reads back the JSON/CSV artifacts and returns a structured result. No phase logic is duplicated, so the tool surface can't drift from the tested CLI. A single tool also keeps flowx to **one** of the host's tool slots (e.g. Genie Code's 20-tool cap). + +### MCP tool layer + +| Module | Purpose | +|-------------------|---------------------------------------------------------------------------------------------------------------------------| +| `mcp/server.py` | `FastMCP` server; registers tools and builds the stdio / streamable-HTTP apps | +| `mcp/runner.py` | Subprocess bridge to `flowx.adapter` with artifact summarizers (for running translation without the `mcp` dependency) | +| `mcp/__main__.py` | `python -m flowx.mcp` entry point (stdio default, `--http` for hosting) | + +The `flowx` tool's `command` selects the adapter operation: `inputs`, `discover`, `convert`, `merge_agentic`, `inspect`, `apply_answers`, `materialize_lookup`, `workspace_paths`, `package`, `migrate`, `record_results`, and `install_dashboard` (with `parameters` carrying that command's arguments). + +## Deployment topology + +The MCP server runs in whichever transport fits the calling tool. This is chosen when the `setup` skill is invoked: + +- **Local / Claude Code:** `setup` installs `mcp` + `uvicorn` + `starlette` into the venv; the server runs over **stdio** and is registered with the MCP client. +- **Databricks Genie Code:** `setup` runs `app/deploy.sh`, which stages a self-contained bundle (app entrypoint + a vendored copy of the pure-Python flowx source), syncs it to the workspace, and creates/deploys the **`mcp-flowx` Databricks App**. The app serves the MCP streamable-HTTP transport at `/mcp` (health at `/`) via `uvicorn` → `Starlette` → `FastMCP`. To meet Genie Code's requirements the server runs **stateless** (`stateless_http=True`) and enables CORS (origins via `FLOWX_ALLOWED_ORIGINS`). You add it in Genie Code under **Settings → MCP Servers → Add Server → Custom MCP server**; Genie connects over OAuth, access is governed by the app's Databricks Apps permissions, and the app authenticates to the workspace as its own service principal. + +```text + Local (Claude Code / other agents) Databricks Genie Code + ─────────────────────────────────── ───────────────────────────────── + agent ◀── stdio ──▶ python -m Genie ◀── HTTPS /mcp ──▶ Databricks App + flowx.mcp (uvicorn → Starlette + (in the venv) → FastMCP streamable-HTTP) + app authenticates as its + own service principal +``` + +See [Installation](/docs/installation#running-flowx-as-an-mcp-server) for the exact commands and the [app README](https://github.com/ghanse/flowx/tree/main/app) for deployment details. + + +A Databricks App can't read the user's workspace / UC Volume files (`/Volumes/...` is **not** auto-mounted). Two ways to get data in/out of the `flowx` tool: + +- **Small jobs — inline.** `command="discover"`/`"migrate"` accept `adf_definitions` (a mapping of relative path → ARM JSON, Git-export layout), supplied by the agent and materialized to a temp dir; `package`/`migrate` return the DAB inline as `bundle = {"files": {relpath: text, …}}`. Inline data flows through the agent's context, so it's capped (~5 MB in). +- **Large factories — by reference (recommended).** Point the server at the source: `adf_volume_path` (a UC Volume read via the SDK Files API) or `adf_workspace_path` (a `/Workspace` directory — e.g. an ADF Git folder — read via the SDK Workspace API). Write the DAB to `output_volume_path` (UC Volume, SDK Files API) or `output_workspace_path` (`/Workspace` directory, SDK Workspace API with `ImportFormat.RAW`), returned as `bundle_uploaded`. The bytes bypass the agent entirely, so it scales to thousands of pipelines; grant the app's service principal read on the source and write on the output target. + +Locally hosted, ordinary `adf_source_path` / `output_dir` paths and mounted volumes work as-is. + diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx index e6b482d..d758899 100644 --- a/docs/content/docs/guide.mdx +++ b/docs/content/docs/guide.mdx @@ -35,15 +35,15 @@ databricks fs cp -r ./adf-export dbfs:/Volumes/main/default/adf_export ## Run the end-to-end migration -Open a fresh conversation and prompt your agent with the path to your JSON templates and a target directory for the output bundle: +Open a fresh conversation and prompt your agent with the path to your JSON templates and a target output directory: -> Use flowx to migrate the ADF pipelines at `/Volumes/main/default/adf_export` into a Databricks Asset Bundle at `./bundle/`. +> Use flowx to migrate the ADF pipelines at `/Volumes/main/default/adf_export` into a Databricks Asset Bundle at `./flowx_output/`. -Flowx will use the `migrate` skill to chain 3 other skills: +flowx will use the `migrate` skill to chain 3 other skills. All three phases write into one shared output directory (default `./flowx_output`): -1. `ingest` parses every JSON file, builds an inventory, and assigns a translation strategy for each resource. This can be deterministic, agentic, or unsupported. -2. `translate` converts each activity to an intermediate representation. The agent will ask for confirmation before running any LLM-based translation. -3. `prepare` creates a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) with job configuration files, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). +1. `discover` parses every JSON file, builds an inventory, assigns a translation strategy for each resource (deterministic, agentic, or unsupported), and emits a `metadata/profile_report.csv` complexity report (one row per pipeline). +2. `convert` converts each activity to an intermediate representation. The agent will ask for confirmation before running any LLM-based translation. +3. `package` creates a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) with job configuration files, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). @@ -51,18 +51,37 @@ Flowx will use the `migrate` skill to chain 3 other skills: ## Review the output -Flowx creates Declarative Automation Bundles in the local file system. The generated bundle can be reviewed and modified before deployment. +flowx writes everything into a single shared output directory (default `./flowx_output`). The generated bundle can be reviewed and modified before deployment. The layout is: -Each bundle contains a top-level `databricks.yml` file with deployment targets and other variables, a `resources/` folder with job configuration, -a `src/` folder with code required to run the pipeline, and a `setup/` folder with scripts for creating supporting resources. +```text +flowx_output/ +├── databricks.yml # Bundle configuration (from package) +├── resources/ # Job configuration (from package) +├── src/ # Code required to run the pipeline (from package) +├── SETUP.md # Setup instructions (from package) +├── metadata/ +│ ├── inventory.json # discover: activity inventory +│ ├── profile_report.csv # profile: per-pipeline complexity report +│ ├── .arm.json # discover: verbatim original ADF/ARM pipeline source +│ └── configuration.json # modify: the collected configuration answers +└── .work/ # transient intermediates (translation report, IR, gaps.json); pruned by prepare +``` -The `translation_report.json` file lists every activity, its translation strategy, warnings raised during translation, and the location of any -generated artifacts. Review the translation report for any warnings, unsupported resources, or to-do items before deploying to your Databricks workspace. +The bundle itself contains a top-level `databricks.yml` file with deployment targets and other variables, a `resources/` folder with job configuration, +a `src/` folder with code required to run the pipeline, and a `SETUP.md` file describing supporting resources to create. + +During translation, a transient `translation_report.json` is written under `flowx_output/.work/`. It lists every activity, its translation strategy, warnings raised during translation, and the location of any +generated artifacts. Review the translation report for any warnings, unsupported resources, or to-do items before deploying to your Databricks workspace. The `package` phase prunes `.work/` after building the bundle (pass `--keep-intermediates` to retain it). Connection strings, credentials, and other protected configuration parameters are emitted as `SecretInstruction` setup steps that require [Databricks Secrets](https://docs.databricks.com/aws/en/security/secrets/). Run the setup scripts and populate secret values before deploying and running pipelines in your Databricks workspace. + +When running with workspace auth (e.g. Genie Code), `package` can optionally persist this run's +coverage to a Unity Catalog table — one row per pipeline stamped with a UUID `run_id`, `run_date`, +and `run_by` (`record-results`) — and install a published AI/BI coverage dashboard over that table +(`install-dashboard`). See [Configuration options](/docs/options) for details. diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index c8ed057..bc36385 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -9,28 +9,29 @@ Orchestration should be treated as a first class citizen during migrations. Beca their configuration can impact data processing results as much as the logic being orchestrated. While significant tooling exists for code conversion and data reconciliation, migrating from legacy orchestration systems is often manual, time-consuming, and prone to risk. -Flowx was created to automate migrations of data pipelines between various orchestrators. It provides a robust, tested set of capabilities +flowx was created to automate migrations of data pipelines between various orchestrators. It provides a robust, tested set of capabilities to parse existing data pipeline definitions, create migration artifacts, and convert data pipeline definitions to Databricks' [Lakeflow jobs framework](https://docs.databricks.com/aws/en/jobs/). -## How Flowx works +## How flowx works -Flowx is a set of agent skills and deterministic translators. Skills tell agentic tools (e.g. Databricks Genie Code, Claude Code, or any +flowx is a set of agent skills and deterministic translators. Skills tell agentic tools (e.g. Databricks Genie Code, Claude Code, or any agent that supports the open [Agent Skills](https://agentskills.io/) format) how to call deterministic translators that parse, translate, and generate Databricks resources. Translation runs in three phases: -1. `ingest` parses Azure Resource Manager templates (e.g. for Data Factory pipelines, datasets, linked services, and triggers) into an execution -tree and builds an inventory. -2. `translate` processes the inventory and converts each activity into a Databricks-compatible intermediate representation. *Deterministic +1. `discover` parses Azure Resource Manager templates (e.g. for Data Factory pipelines, datasets, linked services, and triggers) into an execution +tree, builds an inventory, and emits a per-pipeline complexity report (`metadata/profile_report.csv`). +2. `convert` processes the inventory and converts each activity into a Databricks-compatible intermediate representation. *Deterministic activities* are translated by Python handlers while *agentic activities* are handed off to an LLM-assisted translator with the right context. *Unsupported activities* are flagged as explicit gaps. -3. `prepare` converts each translated pipeline into a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) that +3. `package` converts each translated pipeline into a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) that can be deployed to a Databricks workspace. Bundles include job configuration files, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). -Each phase can be run independently, maintains its own input/output contract, produces artifacts you can inspect before moving to the next phase. +All three phases write into one shared output directory (default `./flowx_output`): the DAB bundle at the top level, kept artifacts under `metadata/`, and transient intermediates under `.work/` (pruned by `package`). Each phase can be run independently, maintains its own input/output contract, and produces artifacts you can inspect before moving to the next phase. ## Next steps +- **[Architecture](/flowx/docs/architecture)** — understand how flowx is deployed and how it translates - **[Installation](/flowx/docs/installation)** — install the flowx plugin in your agentic tool of choice. - **[Usage Guide](/flowx/docs/guide)** — an end-to-end walkthrough from raw ADF JSON to a deployable bundle. - **[Options](/flowx/docs/options)** — reference documenting options for customizing output when translating pipelines with flowx. diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index 99a4ba5..619c98d 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -5,13 +5,14 @@ description: Install flowx in Databricks Genie Code, Claude Code, or other agent import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; import { Callout } from 'fumadocs-ui/components/callout'; +import { Steps, Step } from 'fumadocs-ui/components/steps'; -Flowx is a set of [agent skills](https://github.com/ghanse/flowx/tree/main/skills) that can be installed and used with AI coding assistants. +flowx is a set of [agent skills](https://github.com/ghanse/flowx/tree/main/skills) that can be installed and used with AI coding assistants. To use these skills, install flowx as a plugin using your AI assistant's preferred installation method. -Clone the flowx repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos), then copy the `skills/` directory into a user-level skills folder: +Clone the flowx repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos). Clone it under **`/Workspace/Shared`** (e.g. `/Workspace/Shared/flowx`) rather than your private `/Workspace/Users/` home — when you later deploy the MCP server, the app's service principal must be able to read the source, and it has no access to private user folders by default. Then copy the `skills/` directory into a user-level skills folder: ```bash databricks workspace import-dir skills /Users//.assistant/skills @@ -24,13 +25,13 @@ databricks workspace import-dir skills /Workspace/.assistant/skills ``` Genie Code picks up skills from these directories automatically. Skills fire automatically when their description matches your request. -To invoke a specific skill, use the `@` prefix (e.g. `@migrate translate the ADF pipelines I exported to /Volumes/main/default/adf_export`). +To invoke a specific skill, use the `@` prefix (e.g. `@flowx-migrate translate the ADF pipelines I exported to /Volumes/main/default/adf_export`). See the [Databricks Genie Code Skills documentation](https://docs.databricks.com/aws/en/genie-code/skills) for more details. -Flowx is packaged as a Claude Code plugin. The plugin manifest lives at [`.claude-plugin/plugin.json`](https://github.com/ghanse/flowx/blob/main/.claude-plugin/plugin.json). To install +flowx is packaged as a Claude Code plugin. The plugin manifest lives at [`.claude-plugin/plugin.json`](https://github.com/ghanse/flowx/blob/main/.claude-plugin/plugin.json). To install flowx, run the following command from a Claude Code session: ```bash @@ -41,21 +42,21 @@ flowx, run the following command from a Claude Code session: You can also copy the skill folders into your local `/.claude/skills` folder: ```bash -cp -R skills/{setup,ingest,translate,prepare,migrate} ~/.claude/skills/ +cp -R skills/{flowx-setup,flowx-discover,flowx-convert,flowx-package,flowx-migrate} ~/.claude/skills/ ``` -Once installed, the skills can be invoked using `/flowx:migrate`, `/flowx:ingest`, etc. Claude Code can also invoke the skills when prompted. See the [Claude Code Plugins documentation](https://docs.claude.com/en/docs/claude-code/plugins) for more details. +Once installed, the skills can be invoked using `/flowx:flowx-migrate`, `/flowx:flowx-discover`, etc. Claude Code can also invoke the skills when prompted. See the [Claude Code Plugins documentation](https://docs.claude.com/en/docs/claude-code/plugins) for more details. Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install the flowx skills. The general pattern: -1. Copy each skill folder (`skills/setup`, `skills/ingest`, `skills/translate`, `skills/prepare`, `skills/migrate`) into the tool's configured skills directory. +1. Copy each skill folder (`skills/flowx-setup`, `skills/flowx-discover`, `skills/flowx-convert`, `skills/flowx-package`, `skills/flowx-migrate`) into the tool's configured skills directory. 2. Make sure the path contains `SKILL.md` directly, 3. Restart the tool if it caches skill metadata at startup. -If your tool expects a single Markdown file instead of a directory tree, use the following command to flatten Flowx's skills files: +If your tool expects a single Markdown file instead of a directory tree, use the following command to flatten flowx's skills files: ```bash cat skills/*/SKILL.md > flowx-skills.md @@ -64,11 +65,113 @@ cat skills/*/SKILL.md > flowx-skills.md -## Setting up the Python environment +## Running flowx as an MCP server -Flowx's skills invoke Python modules that may depend on third-party packages. The `setup` skill provisions an isolated virtual environment with the required dependencies. +flowx's phases are also packaged as [Model Context Protocol](https://modelcontextprotocol.io) tools (in [`src/flowx/mcp/`](https://github.com/ghanse/flowx/tree/main/src/flowx/mcp)) so an agent can invoke them directly instead of shelling out to the CLI. The `setup` skill wires this up automatically based on your environment; you can also do it manually. See [Architecture](/docs/architecture) for how the tool layer maps onto the phases. -Run it **once** after installing the skills, before `ingest`, `translate`, `prepare`, or `migrate`. Just ask your agent: +### Configuring the MCP server for Databricks Genie Code + +Genie Code connects to a **hosted** MCP endpoint, so the flowx tools run as a Databricks App that you add in Genie Code's **Custom MCP server** picker. End to end: + + + +#### Clone flowx + +Follow the **Databricks Genie Code** install steps above to clone the repo and copy `skills/` into your skills folder. + + +The MCP app's service principal cannot read private `/Workspace/Users/` folders by default, and `deploy.sh` deploys the source from `/Workspace/Shared/`. Cloning into `/Workspace/Shared` keeps the repo, the deployed source, and team access all in a location every user and the app's service principal can reach. If your workspace restricts `/Workspace/Shared`, use any other folder all users (and the app service principal) can read and pass it via `APP_SOURCE_PATH`. + + + + +#### Run the setup skill + +Ask your agent to *"set up the flowx environment"* (or run `bash /scripts/bootstrap.sh`). On Databricks the `setup` skill detects the environment and, after creating the venv, runs the app deployment in the next step for you. Run it from a workspace web terminal if your Genie session can't shell out to the Databricks CLI. + + + +#### Deploy the MCP server + +```bash +bash /app/deploy.sh +``` + +`deploy.sh` stages a self-contained bundle (the app entrypoint plus a vendored copy of the flowx source), syncs it to **`/Workspace/Shared/mcp-flowx`** (a location the app's service principal can read — override with `APP_SOURCE_PATH`), and creates/deploys the **`mcp-flowx`** app. The script prints the app URL; the MCP endpoint is **`/mcp`**. + + +`databricks apps` deploy commands require a Databricks CLI session and must be run from the workspace web terminal or a local machine. + + + + +#### Grant access + +- **App access:** grant **Can use** on the `mcp-flowx` app to the users or service principals that will call it (Apps UI → *Permissions*, or `databricks apps set-permissions`). +- **Data access:** grant the app's own service principal access to the catalogs, schemas, and Unity Catalog volumes the migration reads from and writes to, plus any SQL warehouse used by the reporting commands (`flowx(command="record_results")` / `flowx(command="install_dashboard")`). + + + +#### Register the MCP server + +MCP servers are available in Genie Code [Agent mode](https://learn.microsoft.com/en-us/azure/databricks/genie-code/use-genie-code#modes). To add the flowx MCP server: + +1. In the Genie Code panel, click **⚙ Settings**. +2. Under **MCP Servers**, click **+ Add Server**. +3. Choose **Custom MCP server** and select the **`mcp-flowx`** Databricks App. +4. Click **Save**. + +The single `flowx` tool will be available when you use Genie Code in Agent mode. + + +Databricks requires a custom MCP app to be: +* Deployed in the same workspace +* Reachable at `https:///mcp` + +If Genie Code cannot connect to the flowx MCP server, set the app's `FLOWX_ALLOWED_ORIGINS` environment variable to your workspace URL and redeploy. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp). + + + + +#### Verify the MCP Server + +Open the health endpoint `/` (returns `{"status":"ok"}`), or ask Genie Code *"what flowx MCP tools are available?"*. You should see the single `flowx` tool. + + + +### Configuring the MCP server for other agent tools + +Deploy the MCP server locally to use flowx with other agent tools. Install the MCP server stack into a local Python virtual environment and run it over stdio: + +```bash +PY="$(cat /.migration-venv)" +"$PY" -m pip install "mcp>=1.12" "uvicorn>=0.30" "starlette>=0.40" +PYTHONPATH="/src" "$PY" -m flowx.mcp +``` + +Register the server with your MCP client (use the interpreter path from the marker file for `command`): + +```json +{ + "mcpServers": { + "flowx": { + "command": "/.venv/bin/python", + "args": ["-m", "flowx.mcp"], + "env": { "PYTHONPATH": "/src" } + } + } +} +``` + + +If you prefer an installed package over `PYTHONPATH`, run `pip install -e ".[mcp]"` from the plugin root; then `python -m flowx.mcp` works without setting `PYTHONPATH`. + + +## Running flowx as a Python process + +flowx's skills invoke Python modules that may depend on third-party packages. The `setup` skill provisions an isolated virtual environment with the required dependencies. + +Run it **once** after installing the skills, before `discover`, `convert`, `package`, or `migrate`. Just ask your agent: > Set up the flowx environment @@ -81,8 +184,9 @@ bash /scripts/bootstrap.sh Running the setup process will: 1. Check that `python3`, `pip`, and `venv` are available. -2. Create `/.venv` if it doesn't already exist. +2. Create the virtual environment if it doesn't already exist. When running under Databricks (Genie Code or notebooks, detected via `DATABRICKS_RUNTIME_VERSION`), the venv is created at `/Workspace/Users//.migration-skills`; everywhere else it is created at `/.venv`. 3. Install the `requirements.txt` dependencies into your virtual environment using `pip`. +4. Write the resolved interpreter path to the marker file `/.migration-venv`. The environment is created once and reused. Re-running the script simply confirms the venv exists and its dependencies are satisfied. @@ -95,14 +199,15 @@ To install Python in your environment, run one of the following commands: * **Windows:** [python.org/downloads](https://www.python.org/downloads/) (enable "Add python.exe to PATH") -After the venv exists, every Python command the skills run uses the venv interpreter with `src/` on `PYTHONPATH`: +After the venv exists, every Python command the skills run uses the interpreter recorded in the marker file, with `src/` on `PYTHONPATH`. Read the interpreter path from `/.migration-venv` rather than hardcoding it: ```bash export PYTHONPATH="/src" -"/.venv/bin/python" -m flowx.adapter inputs ingest +PY="$(cat /.migration-venv)" +"$PY" -m flowx.adapter inputs discover ``` -On Windows, the interpreter is `\.venv\Scripts\python.exe`. The agent normally runs these commands for you; they are handy for troubleshooting a `ModuleNotFoundError`. +The marker file points at `/.venv` locally or `/Workspace/Users//.migration-skills` on Databricks. The agent normally runs these commands for you; they are handy for troubleshooting a `ModuleNotFoundError`. ## Verifying the installation diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index 73219f4..d253728 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -1,8 +1,9 @@ { - "title": "Flowx", + "title": "flowx", "pages": [ "index", "how-it-works", + "architecture", "installation", "guide", "options", diff --git a/docs/content/docs/options.mdx b/docs/content/docs/options.mdx index 741d581..bbfa4f0 100644 --- a/docs/content/docs/options.mdx +++ b/docs/content/docs/options.mdx @@ -5,8 +5,8 @@ description: Control flowx's translation behavior and outputs import { Callout } from 'fumadocs-ui/components/callout'; -Flowx defers some architectural choices to allow users to specify properties of the output jobs. When options are available for controlling -translation, the agent session prompts the user for their preferences. +flowx defers some architectural choices to allow users to specify properties of the output jobs. When options are available for controlling +translation, the agent session prompts the user for their configuration. ## copy_activity_paradigm @@ -53,7 +53,7 @@ must use a [query-based connector](https://docs.databricks.com/aws/en/ingestion/ For every multi-activity motif the detector matches in a pipeline (`incremental_load_watermark`, `rest_api_pagination`, `metadata_driven_bulk_copy`, ...), flowx raises a per-motif -question with id `consolidate_motif:` so each detected pattern can be approved +option with id `consolidate_motif:` so each detected pattern can be approved or rejected independently. | Value | Default | Behavior | @@ -82,3 +82,46 @@ The following are required to consolidate into a single ingestion pipeline: - Access to query the file or database where metadata is stored or a CSV file containing the exported metadata - The number of metadata rows or objects must be less than 250 + +## Discover complexity report + +The `discover` phase emits `/metadata/profile_report.csv` (default `./flowx_output/metadata/profile_report.csv`), one row per pipeline with the following columns: + +| Column | Description | +|--------|-------------| +| `pipeline` | Pipeline name | +| `activities` | Total activity count | +| `datasets` | Number of referenced datasets | +| `linked_services` | Number of referenced linked services | +| `collapsible_patterns` | Number of detected collapsible (motif) patterns | +| `databricks_native_activities` | Count of Databricks-native activities (notebook/jar/python/job) | +| `control_flow_activities` | Count of control-flow / parameter-setting activities (ForEach/If/Switch/SetVariable/AppendVariable/Filter/Wait/Until) | +| `other_activities` | Count of all other activities (Copy/Web/Lookup/etc.) | +| `complexity_score` | Weighted score (see below) | +| `complexity_size` | T-shirt size (`S`/`M`/`L`/`XL`) derived from `complexity_score` | + +The weighted score is `sum(activity weights) + #datasets + #linked_services + #collapsible_patterns`, where activity weights are: + +- Databricks-native (notebook/jar/python/job) = **1** (simplest) +- control-flow / parameter-setting (ForEach/If/Switch/SetVariable/AppendVariable/Filter/Wait/Until) = **2** +- all other activities (Copy/Web/Lookup/etc.) = **3** (hardest) + +## Coverage results table & dashboard (Genie Code) + +When running with workspace auth (Genie Code, or a configured Databricks profile) the `package` +phase surfaces three optional inputs — `results_table`, `results_warehouse_id`, and +`install_dashboard` — to persist coverage and visualize it: + +- **`record-results`** writes one row **per pipeline per run** to the supplied Unity Catalog + table (`catalog.schema.table`), combining the complexity columns above with the + deterministic/agentic/unsupported coverage breakdown. Every row is stamped with a shared + **`run_id`** (UUID), **`run_date`** (`CURRENT_TIMESTAMP()`), and **`run_by`** + (`CURRENT_USER()`), so coverage is trackable across runs and users. +- **`install-dashboard`** creates and publishes an AI/BI (Lakeview) dashboard over that table — + KPI counters (pipelines, coverage %, deterministic/agentic/unsupported activity totals), a + pipelines-by-complexity bar chart, a coverage-over-runs line, and a per-pipeline coverage + table. + +The SQL warehouse is auto-detected (preferring a running serverless warehouse) when +`results_warehouse_id` is left blank. Both run via the Databricks SDK and degrade gracefully +when workspace auth or a warehouse is unavailable. diff --git a/pyproject.toml b/pyproject.toml index f5b9dac..057e8fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] -name = "flowx" -version = "0.2.0" +name = "databricks-flowx" +version = "0.1.0" description = "ADF to Databricks Lakeflow Jobs translator via Declarative Automation Bundles" readme = "README.md" requires-python = ">=3.12" @@ -26,6 +26,15 @@ dependencies = [ "sqlglot>=25.0", ] +[project.optional-dependencies] +# Install with `pip install -e .[mcp]` to host the flowx MCP tools +# (locally over stdio, or as a streamable-HTTP server / Databricks App). +mcp = [ + "mcp>=1.12", + "uvicorn>=0.30", + "starlette>=0.40", +] + [dependency-groups] dev = [ "pytest>=8.3.3,<9", @@ -50,6 +59,12 @@ python_version = "3.12" mypy_path = "src" exclude = ['venv', '.venv', 'tests/*'] +# Optional `mcp` extra (only installed for hosting the MCP server). Avoid +# missing-stub errors in dev environments that don't install it. +[[tool.mypy.overrides]] +module = ["mcp.*", "starlette.*", "uvicorn.*"] +ignore_missing_imports = true + [tool.pytest.ini_options] testpaths = ["tests"] addopts = "--no-header" diff --git a/requirements.txt b/requirements.txt index 060dab7..a439c35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ charset-normalizer==3.4.7 cryptography==48.0.0 # via google-auth databricks-sdk==0.110.0 - # via flowx + # via databricks-flowx google-auth==2.53.0 # via databricks-sdk idna==3.15 @@ -23,10 +23,10 @@ pyasn1-modules==0.4.2 pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' # via cffi pyyaml==6.0.3 - # via flowx + # via databricks-flowx requests==2.34.2 # via databricks-sdk sqlglot==30.8.0 - # via flowx + # via databricks-flowx urllib3==2.7.0 # via requests diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh old mode 100755 new mode 100644 index badb2ca..ead7187 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -2,16 +2,25 @@ # # Bootstraps a Python environment for the flowx plugin. # -# Creates a virtual environment at /.venv and installs the Python -# dependencies listed in requirements.txt using pip. +# Creates a virtual environment and installs the Python dependencies listed in +# requirements.txt using pip. The venv location depends on the environment: +# * Databricks (Genie Code / notebooks): /Workspace/Users//.migration-skills +# so the environment persists in the workspace and is shared across sessions. +# * Everywhere else: /.venv # # If python3, pip, or the venv module are unavailable, the script prints a clear # warning telling the user what to install and exits non-zero without making changes. # +# On Databricks serverless compute (where ensurepip is unavailable), the script +# falls back to --without-pip + get-pip.py automatically. +# # After bootstrapping, run the plugin's Python code with the venv interpreter and # src/ on PYTHONPATH, e.g.: # -# PYTHONPATH="/src" "/.venv/bin/python" -m flowx.adapter inputs ingest +# PYTHONPATH="/src" "/bin/python" -m flowx.adapter inputs discover +# +# The resolved interpreter path is also written to /.migration-venv +# so the skills can discover it without re-deriving the location. # set -euo pipefail @@ -33,7 +42,7 @@ if ! command -v python3 >/dev/null 2>&1; then cat >&2 <<'EOF' WARNING: python3 was not found on your PATH. -Flowx requires Python 3.12+ to run its translation code. +flowx requires Python 3.12+ to run its translation code. Please install Python (it bundles pip) before continuing: - macOS: brew install python (or https://www.python.org/downloads/) @@ -47,6 +56,43 @@ fi PYTHON_BIN="$(command -v python3)" +# On Databricks (Genie Code / notebooks), create the venv under the current +# user's workspace folder so it persists and is shared across sessions, rather +# than inside the (ephemeral) plugin checkout. Resolve the current user from the +# notebook runtime context, falling back to the workspace SDK. +if [ -n "${DATABRICKS_RUNTIME_VERSION:-}" ]; then + CURRENT_USER="$("$PYTHON_BIN" - <<'PYRESOLVE' 2>/dev/null || true +def _resolve(): + try: + from dbruntime.databricks_repl_context import get_context + ctx = get_context() + for attr in ("userName", "user"): + val = getattr(ctx, attr, None) + if val and "@" in str(val): + return str(val) + except Exception: + pass + try: + from databricks.sdk import WorkspaceClient + name = WorkspaceClient().current_user.me().user_name + if name: + return name + except Exception: + pass + return "" +print(_resolve()) +PYRESOLVE +)" + if [ -n "$CURRENT_USER" ]; then + VENV_DIR="/Workspace/Users/${CURRENT_USER}/.migration-skills" + echo "Databricks runtime detected; using workspace venv for ${CURRENT_USER}:" + echo " $VENV_DIR" + else + echo "Databricks runtime detected but current user could not be resolved;" >&2 + echo " falling back to plugin-local venv at $VENV_DIR" >&2 + fi +fi + # Verify pip is available if ! "$PYTHON_BIN" -m pip --version >/dev/null 2>&1; then cat >&2 <<'EOF' @@ -81,7 +127,20 @@ fi # Create the virtual environment if [ ! -x "$VENV_DIR/bin/python" ]; then echo "Creating virtual environment at $VENV_DIR ..." - "$PYTHON_BIN" -m venv "$VENV_DIR" + if "$PYTHON_BIN" -m venv "$VENV_DIR" 2>/dev/null; then + : # standard venv creation succeeded + else + # Fallback for environments where ensurepip is unavailable (e.g. Databricks + # serverless compute). Create the venv without pip, then bootstrap pip + # via get-pip.py. + echo "Standard venv failed (ensurepip likely missing); trying --without-pip ..." + rm -rf "$VENV_DIR" + "$PYTHON_BIN" -m venv --without-pip "$VENV_DIR" + echo "Bootstrapping pip via get-pip.py ..." + curl -sSL https://bootstrap.pypa.io/get-pip.py -o /tmp/_orchestra_get_pip.py + "$VENV_DIR/bin/python" /tmp/_orchestra_get_pip.py --quiet + rm -f /tmp/_orchestra_get_pip.py + fi else echo "Using existing virtual environment at $VENV_DIR ..." fi @@ -100,12 +159,47 @@ echo "Upgrading pip ..." echo "Installing dependencies from requirements.txt ..." "$VENV_PYTHON" -m pip install -r "$REQUIREMENTS" +# --------------------------------------------------------------------------- +# Databricks runtime: pre-configure workspace auth from the notebook context +# --------------------------------------------------------------------------- +# The venv Python does NOT have dbruntime (it's a system-only package), so +# workspace_downloader.py can't auto-configure auth at runtime. However, the +# system Python ($PYTHON_BIN) DOES have it. Extract host + token here once and +# write ~/.databrickscfg so all subsequent venv invocations find it immediately. +# --------------------------------------------------------------------------- +if [ -n "${DATABRICKS_RUNTIME_VERSION:-}" ]; then + CFG_PATH="${HOME}/.databrickscfg" + if [ -s "$CFG_PATH" ]; then + echo "Databricks auth already configured at $CFG_PATH" + else + echo "Databricks runtime detected; extracting workspace auth ..." + "$PYTHON_BIN" -c " +from dbruntime.databricks_repl_context import get_context +c = get_context() +host = 'https://' + c.browserHostName +token = c.apiToken +if not host or not token: + raise SystemExit('host/token unavailable from runtime context') +import pathlib +p = pathlib.Path('$CFG_PATH') +p.parent.mkdir(parents=True, exist_ok=True) +p.write_text(f'[DEFAULT]\nhost = {host}\ntoken = {token}\n') +print(f' -> {p} written (host={host})') +" 2>/dev/null && true || echo " -> skipped (runtime context unavailable)" + fi +fi + +# Record the resolved interpreter path so the skills can discover it without +# re-deriving the (environment-dependent) venv location. +echo "$VENV_PYTHON" > "$PLUGIN_ROOT/.migration-venv" + cat < + Translate parsed ADF pipeline AST into Databricks IR (intermediate representation). + Runs deterministic translators for known activity types, then invokes agentic skills + from adf-to-databricks-plugin for gaps. +triggers: + - "translate ADF" + - "convert ADF" + - "translate pipelines" + - "convert pipelines" + - "run translation" +--- + +# Convert ADF to Databricks IR + +Convert the parsed ADF inventory into Databricks intermediate representation (IR) using deterministic translators for known types and agentic fallback for unknown types. + +## Context + +This is phase 2 of the flowx migration workflow. It consumes the ADF source (profiled by the `discover` skill) and produces a translation report — a transient intermediate under `/.work/` — that the `package` skill uses to generate Databricks Declarative Automation Bundles. It shares the single migration `` with the other phases. + +The translation follows a **deterministic-first** strategy: +1. Activities with known, well-defined mappings are translated by built-in Python translators +2. Activities that require interpretation, expression conversion, or lack a Python translator are handled by agent skills from the `adf-to-databricks-plugin` + +## How to run this skill — MCP tools or venv CLI + +This phase runs one of two ways; run the **`setup`** skill first if you haven't. + +- **MCP tool (Databricks Genie Code, or a local stdio registration) — the only path in Genie Code:** + call the single **`flowx`** tool (one command per step) and run **no** `python3`/`$PY`/`bash` + commands. The `"$PY" -m …` snippets in the steps below are the **local-CLI fallback only** — ignore + them on this path. Map the steps to: + + ``` + flowx(command="convert", parameters={"output_dir": "", "pipeline": ""}) + # convert reuses the discovered output_dir on the server; only pass "adf_definitions" (inline ARM + # JSON) if you are converting without a prior discover on this server. + flowx(command="inspect", parameters={"report_path": "/.work/translation_report.json", "answers": [...]}) + flowx(command="apply_answers", parameters={"report_path": "...", "answers": ["id=value", ...], "output_dir": "", "lookup_csv": ""}) + flowx(command="merge_agentic", parameters={"report_path": "...", "agentic_results_dir": "", "output_path": ""}) + ``` + + Use the tool results in place of reading the files directly. `command="merge_agentic"` covers the + agentic `--merge-agentic` step shown later in this skill. + +- **venv CLI (local, no MCP server):** ensure the venv exists (`setup` Path B / `bootstrap.sh`), then + run the commands below with the venv interpreter (from the marker file `/.migration-venv`) + and `src/` on `PYTHONPATH` (use `$PY` anywhere a command shows `python3`): + + ```bash + export PYTHONPATH="/src" + PY="$(cat /.migration-venv)" + "$PY" -m flowx.adapter convert --output-dir + ``` + + If Python or pip is missing, `bootstrap.sh` prints a warning telling the user what to install — + relay it and stop until they have Python 3.12+ and pip. + +## Workflow + +Follow these steps in order: + +### Step 0 — Gather phase inputs + +Run the adapter inputs subcommand so the agent surfaces the free-text +options the phase needs (inventory path, ADF source dir, output +directory): + +```bash +"$PY" -m flowx.adapter inputs convert +``` + +The JSON response carries the prompts and defaults; collect answers from the user +(or fall back to the defaults). Keep them in conversation context — the same shared +`` is used by every phase. + +### Step 1 — Locate the inventory + +The discover phase wrote `/metadata/inventory.json` (and `profile_report.csv`). If the +shared `` is not already in conversation context, ask the user: + +> Which migration output directory did the discover phase use? (default: `./flowx_output`) + +Validate `/metadata/inventory.json` exists and is well-formed. + +### Step 2 — Run deterministic translation + +Execute the translation engine on all deterministic activities: + +```bash +# Unified runner (recommended): `"$PY" -m flowx.adapter convert ...` +# forwards to the engine below; --adf-source-path aliases --source-dir. +"$PY" -m flowx.translator.engine \ + --source-dir \ + --output-dir \ + [--pipeline ] +``` + +Where: +- `` is the original ADF JSON directory (the same `--source-dir` used by discover) +- `` is the **shared migration output directory** (default: `./flowx_output`) — the + same one discover used +- `` (optional) — when provided, translates only the named pipeline. **Always pass `--pipeline` when the user has specified a specific pipeline to migrate**, matching the value passed to the discover phase. + +The translation report and intermediate IR are written to the **transient** `/.work/` +folder (`translation_report.json`, per-pipeline IR, `gaps.json`). These are consumed by the steps +below and the package phase, then pruned — they are not kept artifacts. + +### Step 3 — Read the translation report + +Read `/.work/translation_report.json`. It has this structure: + +```json +{ + "inventory_path": "/path/to/inventory.json", + "generated_at": "2026-04-07T12:30:00Z", + "translations": [ + { + "pipeline": "ETL_Main", + "activity": "CopyFromBlob", + "type": "Copy", + "strategy": "deterministic", + "status": "translated", + "ir": { + "task_key": "copy_from_blob", + "task_type": "notebook_task", + "notebook_path": "notebooks/copy_from_blob.py", + "parameters": { "source": "abfss://...", "target": "..." } + } + }, + { + "pipeline": "ETL_Main", + "activity": "TransformData", + "type": "ExecuteDataFlow", + "strategy": "agentic", + "status": "pending", + "raw_activity_json": { "...": "..." }, + "target_skill": "adf-to-databricks:adf-dataflow-converter" + } + ], + "summary": { + "total": 47, + "deterministic_translated": 35, + "agentic_pending": 10, + "failed": 2 + } +} +``` + +### Step 4 — Handle agentic gaps + +For each translation with `"status": "pending"` and `"strategy": "agentic"`, invoke the appropriate skill from the `adf-to-databricks-plugin`. Route by activity type. + +Every agentic gap in the translation report carries the activity's **full ADF/ARM JSON** under `raw_activity_json` (engine field `raw_definition`), and the generated placeholder notebook embeds the same JSON in a fenced `json` block. This holds for nested activities too — an `Until` inside an `IfCondition` / `Switch` / `ForEach` is reported as its own gap. Always translate from this ARM JSON. + +**Until activities (agent-based handler):** +Databricks Lakeflow Jobs have no native repeat-until loop, so translate the `Until` from its ARM JSON into a single Python notebook task implementing a bounded polling loop. From the embedded JSON, read: +- `typeProperties.expression` — the ADF exit condition (e.g. `@or(equals(variables('jobStatus'),'succeeded'), equals(variables('jobStatus'),'failed'))`); convert it into the Python `while not ():` guard. +- `typeProperties.timeout` — wrap the loop in a wall-clock deadline (`time.monotonic()`), raising on timeout. +- `typeProperties.activities` — the loop body (e.g. a `Wait`, a polling `WebActivity`, a `SetVariable` that captures the next status); translate each child inline so the whole loop runs in one notebook. +Read the loop variables from `dbutils.widgets`, surface the final state as a task value, and write the result over the placeholder notebook's `raise NotImplementedError` cell. If the external `adf-to-databricks:adf-pipeline-converter` skill is installed you may delegate to it with the same ARM JSON; otherwise perform the translation directly. + +**ExecuteDataFlow activities:** +Invoke `adf-to-databricks:adf-dataflow-converter` with the raw activity JSON and associated data flow definition. Provide context: +- The raw `typeProperties` from the ADF activity +- The data flow JSON definition (if available in the source directory under `dataflow/`) +- The linked service configurations for source/sink connections +- Target catalog and schema for the SDP pipeline or PySpark notebook output + +**Control flow activities (Switch, Until, Wait, Filter, AppendVariable):** +Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: +- The full pipeline JSON containing the activity +- Any nested activities within the control flow +- Variable definitions from the pipeline +- The desired Databricks task type mapping + +**Stored procedures and external calls (SqlServerStoredProcedure, AzureFunction, WebHook, Custom):** +Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: +- The linked service configuration for the target system +- Connection details and authentication method +- Any parameters or request bodies + +**Complex expressions:** +If any activity (deterministic or agentic) contains ADF expressions that the deterministic translator could not resolve, invoke `adf-to-databricks:adf-expression-translator` with: +- The raw expression string (e.g., `@pipeline().parameters.inputPath`) +- The expression context (pipeline parameters, variables, activity outputs) +- The target format (Python f-string, Spark SQL, task parameter reference) + +**Trigger definitions:** +Invoke `adf-to-databricks:adf-trigger-converter` with: +- The trigger JSON definition +- The associated pipeline references +- Target: Databricks job schedule configuration (quartz_cron_expression, periodic, or file_arrival) + +### Step 5 — Collect agentic results + +Each resolved agentic gap produces one translation result. Write them into +`/agentic_results/` as one JSON file per activity (the filename is +arbitrary, e.g. `__.json`). Each file MUST use this schema: + +```json +{ + "activity_name": "", + "pipeline": "", + "task": { + "type": "NotebookActivity", + "name": "", + "task_key": "", + "notebook_path": "/Workspace/.../your_translated_notebook" + } +} +``` + +- `activity_name` (required) — matches the `name` of the placeholder task in the + report (the merge locates it by name, recursing into IfCondition / ForEach / + Switch containers, so nested gaps like an `Until` are found). +- `pipeline` (optional) — only needed to disambiguate multi-pipeline reports. +- `task` (required) — the replacement IR task. The most portable form is a + `NotebookActivity` whose `notebook_path` points at a notebook you have written + to the workspace; the package phase references it directly. `task_key` and + `depends_on` are inherited from the placeholder when omitted, so dependency + edges are preserved. + +### Step 6 — Merge agentic results + +Fold the results into the translation report (placeholders are replaced in place): + +```bash +"$PY" -m flowx.translator.engine \ + --merge-agentic \ + --report /.work/translation_report.json \ + --agentic-results +``` + +Equivalently via the unified runner: `"$PY" -m flowx.adapter convert --merge-agentic --report /.work/translation_report.json --agentic-results `. Add `--output ` to write a copy instead of overwriting the report. The command exits non-zero if any result could not be matched to a placeholder. + +This updates `/.work/translation_report.json` with the agentic results merged in, changing their status from `pending` to `translated` (or `failed` if the agentic skill could not produce a result). + +### Step 6.1 — Gather just-in-time translation configuration + +Run `inspect` **once** to get the full option schema, then drive the whole question chain yourself — +do **not** re-run `inspect` per follow-up: + +```bash +"$PY" -m flowx.adapter inspect /.work/translation_report.json +``` + +It returns every option the report can raise, each annotated with a `show_when` condition: + +```json +{"pipelines": [{"pipeline_name": "...", "options": [ + {"option_id": "notify_destination", "prompt": "...", "rationale": "...", + "choices": [{"value": "...", "label": "...", "description": "..."}], + "free_text": false, "default": "keep", "show_when": []}, + {"option_id": "notify_slack_url", "prompt": "...", "free_text": true, "default": "", + "show_when": [{"option_id": "notify_destination", "in": ["slack"]}]} +]}]} +``` + +Walk it locally: + +1. **Ask an option only when its `show_when` is satisfied** — every clause `{option_id, in:[values]}` + must match an answer you've already collected (empty `show_when` = always ask). So `notify_slack_url` + surfaces only after `notify_destination=slack`; the metadata-driven `access`/`size`/`lookup_tool` + chain surfaces only after `metadata_driven_consolidate=consolidate`, etc. Present each option's + `prompt`/`rationale` and `choices`; honor the `default`. +2. **Validate each answer** against `choices` (a `free_text` option — empty `choices` — accepts any + value; blank skips an optional one). +3. **Perform data actions inline** when an answer calls for it — e.g. when + `metadata_driven_lookup_tool=have`, run the lookup query with your database tool to get the rows. +4. When every applicable option is answered, apply them **in one `modify` call** (Step 6.2) with all + answers as `--answer OPTION_ID=VALUE` flags. `modify` validates every answer server-side. + +**Activity→Notify (`activity_and_notify`) motifs.** When **any** activity (Copy, +Notebook, Lookup, stored procedure, …) is followed by notification Web +activities, the adapter raises `notify_destination`: +`keep` (default) leaves the Web activities to translate directly — nothing is +collapsed. Any other value (`email`, `slack`, `teams`, `pagerduty`, `webhook`) +collapses the pattern: the upstream activity becomes the task and the +notifications become Databricks job-task `on_success`/`on_failure` notifications +routed to that destination (the ADF Web activity URL/body is not used). The schema includes **one +follow-up per Databricks-SDK field** of each destination, each gated by +`show_when: [{notify_destination, in:[]}]`; ask the chosen destination's fields (required +first) once the user picks it: + +| Destination | Chained field options (SDK arg) | +|-------------|---------------------------------| +| `email` | `notify_email_recipients` (`addresses`, comma-separated) | +| `slack` | `notify_slack_url` (`url`), `notify_slack_channel_id` (`channel_id`, optional), `notify_slack_oauth_token` (`oauth_token`, optional) | +| `teams` | `notify_teams_url` (`url`) | +| `pagerduty` | `notify_pagerduty_integration_key` (`integration_key`) | +| `webhook` | `notify_webhook_url` (`url`), `notify_webhook_username` (`username`, optional), `notify_webhook_password` (`password`, optional) | + +All destinations also take an optional `notify_destination_name` and +`notify_events` (both/on_failure/on_success). Optional fields left blank are +omitted so the SDK applies its defaults. For **non-email** destinations, the +`modify` phase creates (or reuses by display name) the Databricks notification +destination via the SDK **as soon as you submit the answers** — it validates the +config immediately and bakes the resolved destination id into the modified report, +so package just wires `webhook_notifications` to that id (no further SDK call). +This requires workspace auth at `modify` time; if creation fails there, the id is +left unresolved and package retries or emits a `notification_destination` setup task. +**Email** needs no destination — it uses raw `email_notifications` and is never +created via the SDK. + +When the metadata-driven flow ends with `metadata_driven_lookup_tool=have` +and the agent has a database tool (Genie, MCP SQL, or a workspace SDK), +run the lookup query directly to obtain the rows as CSV. When the answer is +`none`, ask the user for a CSV file path or a literal CSV string. Pass it inline +to `modify` via `--lookup-csv` (no intermediate JSON file): + +```bash +"$PY" -m flowx.adapter modify \ + /.work/translation_report.json \ + --output-dir \ + --answer metadata_driven_consolidate=consolidate \ + --answer metadata_driven_access=yes \ + --lookup-csv "" +``` + +When no metadata-driven motif is consolidated, `--lookup-csv` is omitted. In that default +(non-consolidated) case the motif becomes a Databricks **for-each task** that runs one Spark JDBC +read per source table — its iteration inputs are the resolved lookup rows when available, otherwise a +control-table lookup task seeds them at run time. (Consolidating instead emits one managed Lakeflow +Connect ingestion pipeline.) + +#### Legacy flow details + +Before writing the final report, surface any pipeline-modifier options the +IR raises (Copy Data paradigm, non-Databricks task compute, Lakeflow Connect +opt-in, Databricks task compute). Use the adapter CLI bridge: + +```bash +"$PY" -m flowx.adapter inspect /.work/translation_report.json +``` + +The command emits JSON: + +```json +{ + "pipelines": [ + { + "pipeline_name": "ETL_Main", + "options": [ + { + "option_id": "copy_activity_paradigm", + "prompt": "How should Copy Data activities targeting Delta be implemented?", + "rationale": "...", + "options": [{"value": "notebook", "label": "...", "description": "..."}, ...], + "affected_task_keys": ["copy_orders", "copy_customers"], + "default": "notebook" + }, + ... + ] + } + ] +} +``` + +For each option, prompt the user with the rationale, options, and the task keys it +affects. Use the default when the user defers. Then apply the collected answers as +`--answer OPTION_ID=VALUE` flags: + +```bash +"$PY" -m flowx.adapter modify \ + /.work/translation_report.json \ + --output-dir \ + --answer copy_activity_paradigm=sdp \ + --answer non_databricks_task_compute=serverless \ + --answer use_lakeflow_connectors=lakeflow_connect +``` + +`modify` writes two things under the shared ``: +- `.work/translation_report.stamped.json` — the configuration-stamped IR the package phase consumes +- `metadata/configuration.json` — the collected answers, kept as the migration's configuration record + +The package phase (next skill) reads the stamped report from `.work/` automatically. +When no options are raised, the inspect output is `{"pipelines": [{"pipeline_name": "...", "options": []},...]}` — skip `modify`; package falls back to the un-stamped report. + +### Step 7 — Present translation summary + +Display a summary to the user: + +``` +Translation Summary +=================== +Total activities: 47 +Deterministic translated: 35 (74.5%) +Agentic translated: 8 (17.0%) +Failed: 4 ( 8.5%) + +Overall coverage: 91.5% + +Failed translations: + - ETL_Main / RunSSIS (ExecuteSSISPackage) — no translator available + - ETL_Main / CustomTask (Custom) — agentic skill returned error + ... + +Generated artifacts (transient, under /.work/): + - translation_report.json + - per-pipeline IR (43 files) + - gaps.json +``` + +If coverage is below 100%, explain the options for failed translations: +1. Manual notebook creation for unsupported types +2. Retry agentic translation with additional context +3. Skip the activity and add a placeholder task in the DAB + +## Reference + +See `references/activity-mapping.md` for the complete mapping between ADF activity types and translation strategies. + +## Examples + +- "Convert the ADF pipelines" +- "Convert ADF to Databricks" +- "Run the translation on the inventory from the profile step" +- "Convert the parsed pipelines using deterministic + agentic" +- "Convert only the pl_demo_01 pipeline" + +## Output Artifacts + +The convert phase writes only **transient** intermediates, under `/.work/` (consumed +by `modify`/`package`, then pruned — not kept): + +| File | Description | +|---|---| +| `.work/translation_report.json` | Full translation report with IR for all activities | +| `.work/.json` | Per-pipeline Databricks IR | +| `.work/gaps.json` | Agentic gaps awaiting skill conversion | +| `.work/translation_report.stamped.json` | Configuration-stamped report (written by `modify`) | diff --git a/skills/translate/references/activity-mapping.md b/skills/flowx-convert/references/activity-mapping.md similarity index 99% rename from skills/translate/references/activity-mapping.md rename to skills/flowx-convert/references/activity-mapping.md index 137d980..9635220 100644 --- a/skills/translate/references/activity-mapping.md +++ b/skills/flowx-convert/references/activity-mapping.md @@ -163,7 +163,7 @@ The agentic approach trades speed for coverage — it can handle the long tail o ## Expression Function Coverage -Flowx deterministically translates 73 of 84 ADF expression functions to Python notebook code. The remaining 11 functions require agentic translation: +flowx deterministically translates 73 of 84 ADF expression functions to Python notebook code. The remaining 11 functions require agentic translation: - `dataUri`, `dataUriToBinary`, `dataUriToString`, `decodeDataUri` — Data URI encoding/decoding (rare in practice) - `uriComponentToBinary` — URI component to binary conversion diff --git a/skills/translate/references/expression-functions.md b/skills/flowx-convert/references/expression-functions.md similarity index 100% rename from skills/translate/references/expression-functions.md rename to skills/flowx-convert/references/expression-functions.md diff --git a/skills/flowx-discover/SKILL.md b/skills/flowx-discover/SKILL.md new file mode 100644 index 0000000..f33a992 --- /dev/null +++ b/skills/flowx-discover/SKILL.md @@ -0,0 +1,273 @@ +--- +name: flowx-discover +description: > + Load and parse Azure Data Factory pipeline definitions from Unity Catalog volumes or local directories. + Produces a typed inventory that classifies every activity as deterministic, agentic, or unsupported. +triggers: + - "discover ADF" + - "load ADF" + - "parse ADF" + - "import pipelines" + - "load pipelines" + - "parse pipelines" + - "inventory ADF" +--- + +# Discover ADF Pipeline Definitions + +Parse Azure Data Factory pipeline, dataset, linked service, and trigger JSON files into a typed AST and produce a classified inventory. + +## Context + +This is phase 1 of the flowx migration workflow. It takes raw ADF JSON exports and produces an `inventory.json` file that the `convert` skill consumes. The inventory classifies every ADF activity into one of three strategies: + +- **Deterministic** — a built-in translator exists (Copy, DatabricksNotebook, ForEach, IfCondition, etc.) +- **Agentic** — requires LLM-assisted translation via the `adf-to-databricks-plugin` skills (ExecuteDataFlow, Switch, Until, StoredProc, etc.) +- **Unsupported** — no known translation path; requires manual intervention + +## How to run this skill — MCP tool or venv CLI + +This phase runs one of two ways; run the **`setup`** skill first if you haven't. + +- **MCP tool (Databricks Genie Code, or a local stdio registration) — the only path in Genie Code:** + call the single **`flowx`** tool with `command="discover"` and run **no** `python3`/`$PY`/`bash` + commands. The `"$PY" -m …` snippets in the steps below are the **local-CLI fallback only** — ignore + them on this path. + + The hosted server **cannot read your workspace/volume files**, so pass the ADF JSON **inline** as + `adf_definitions` — a mapping of relative path → JSON content mirroring the ADF Git-export layout. + You (the agent) read the ARM JSON files from the source and supply them: + + ``` + flowx(command="discover", parameters={ + "adf_definitions": { + "pipeline/Foo.json": { ...ARM JSON... }, + "dataset/Bar.json": { ... }, + "linkedService/Baz.json": { ... }, + "trigger/Qux.json": { ... } + }, + "output_dir": "", "pipeline": ""}) + ``` + + For **large factories** (hundreds–thousands of pipelines), don't inline — reference the source + instead (inline `adf_definitions` is capped at ~5 MB): pass `"adf_volume_path": + "/Volumes/cat/sch/adf_export"` for a UC Volume (read via the SDK Files API) or + `"adf_workspace_path": "/Workspace/Shared/adf_export"` for an ADF Git folder in the workspace (read + via the SDK Workspace API). Locally, where the server can read + the path, you may instead pass `adf_source_path`. The tool returns the inventory summary + (pipeline/activity counts by strategy and coverage); use it in place of reading the files directly. + +- **venv CLI (local, no MCP server):** ensure the venv exists (`setup` Path B / `bootstrap.sh`), then + run the commands below with the venv interpreter and `src/` on `PYTHONPATH`. The interpreter path is + in the marker file `/.migration-venv` (use `$PY` anywhere a command shows `python3`): + + ```bash + export PYTHONPATH="/src" + PY="$(cat /.migration-venv)" + "$PY" -m flowx.adapter discover --adf-source-path --output-dir + ``` + + If Python or pip is missing, `bootstrap.sh` prints a warning telling the user what to install — + relay it and stop until they have Python 3.12+ and pip. + +## Workflow + +Follow these steps in order: + +### Step 1 — Determine the ADF source path + +Ask the user for the location of their ADF JSON exports. Accept either: +- A Unity Catalog volume path (e.g., `/Volumes/main/default/adf_export`) +- A local directory path (e.g., `./adf_export/` or `/tmp/adf_json/`) + +The directory should contain subdirectories or files for: +- `pipeline/` or `pipelines/` — pipeline definition JSON files +- `dataset/` or `datasets/` — dataset definition JSON files (optional) +- `linkedService/` or `linked_services/` — linked service JSON files (optional) +- `trigger/` or `triggers/` — trigger definition JSON files (optional) + +### Step 2 — Download from UC volumes if needed + +If the source path starts with `/Volumes/`, the files live in a Unity Catalog volume and must be downloaded to a local temp directory first. + +Use the `databricks-execution-compute` skill to run the following on the Databricks workspace: + +```python +import os, json, shutil, tempfile + +volume_path = "" +local_dir = tempfile.mkdtemp(prefix="adf_ingest_") + +# Copy from volume to local +for root, dirs, files in os.walk(volume_path): + for f in files: + if f.endswith(".json"): + src = os.path.join(root, f) + rel = os.path.relpath(src, volume_path) + dst = os.path.join(local_dir, rel) + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + +print(f"Downloaded ADF files to: {local_dir}") +``` + +Alternatively, use the Databricks CLI: +```bash +databricks fs cp -r "dbfs:" "" --overwrite +``` + +Set the working source directory to the local temp path for subsequent steps. + +### Step 3 — Run the deterministic parser + +Run the discover phase via the adapter's unified phase runner (recommended): + +```bash +"$PY" -m flowx.adapter discover \ + --adf-source-path \ + --output-dir \ + [--pipeline ] +``` + +`--adf-source-path` is accepted as an alias of `--source-dir` (it matches the +`adf_source_path` input option). This forwards to, and is equivalent to, running +the loader directly: + +```bash +"$PY" -m flowx.parser.adf_loader \ + --source-dir --output-dir [--pipeline ] +``` + +Where: +- `` is the root of the flowx plugin (the directory containing `src/`) +- `` is the local directory containing ADF JSON files +- `` is the **single shared migration output directory** used by all three phases + (default: `./flowx_output`). Discover writes its artifacts into the `metadata/` subfolder. +- `` (optional) — when provided, filters to only the named pipeline. When omitted, all pipelines in the source directory are included. + +**Always pass `--pipeline` when the user has specified a specific pipeline to migrate.** This ensures the inventory and all downstream phases are scoped to only that pipeline. + +This produces, under `/metadata/`: +- `inventory.json` — the classified activity inventory +- `profile_report.csv` — one row per pipeline with a complexity assessment (see Step 4b) +- `.arm.json` — the verbatim original ADF/ARM source for each pipeline (provenance) + +### Step 4 — Read and validate the inventory + +Read the generated `/metadata/inventory.json` file. It has this structure: + +```json +{ + "source_dir": "/path/to/adf/json", + "generated_at": "2026-04-07T12:00:00Z", + "pipelines": [ + { + "name": "PipelineName", + "file": "pipeline/PipelineName.json", + "activities": [ + { + "name": "CopyFromBlob", + "type": "Copy", + "strategy": "deterministic", + "translator": "copy.py" + }, + { + "name": "RunDataFlow", + "type": "ExecuteDataFlow", + "strategy": "agentic", + "skill": "adf-to-databricks:adf-dataflow-converter" + } + ] + } + ], + "summary": { + "pipeline_count": 12, + "activity_count": 47, + "deterministic_count": 35, + "agentic_count": 10, + "unsupported_count": 2, + "coverage_pct": 95.7 + } +} +``` + +### Step 4b — Review the complexity report + +`/metadata/profile_report.csv` carries one row per pipeline with a migration-complexity +assessment. Columns: + +| Column | Meaning | +|---|---| +| `pipeline` | Pipeline name | +| `activities` | Total activities (including nested ForEach/If/Switch children) | +| `datasets` | Distinct datasets the pipeline references | +| `linked_services` | Distinct linked services (activity-level + via referenced datasets) | +| `collapsible_patterns` | Number of motif patterns detected (auto-collapsible during convert) | +| `databricks_native_activities` | Notebook / SparkJar / SparkPython / Job activities (simplest) | +| `control_flow_activities` | ForEach / If / Switch / SetVariable / AppendVariable / Filter / Wait / Until | +| `other_activities` | Everything else — Copy, Web, Lookup, agentic types (hardest) | +| `complexity_score` | Weighted score: native×1 + control×2 + other×3 + datasets + linked_services + collapsible_patterns | +| `complexity_size` | T-shirt size from the score: **S** ≤5, **M** ≤15, **L** ≤30, **XL** >30 | + +Use it to set expectations: S/M pipelines are largely deterministic; L/XL pipelines (many "other" +activities, datasets, or linked services) warrant closer review and more agentic translation. + +### Step 5 — Present the summary + +Display a summary table to the user: + +``` +ADF Ingestion Summary +===================== +Pipelines parsed: 12 +Total activities: 47 + +Strategy Breakdown: + Deterministic: 35 (74.5%) + Agentic: 10 (21.3%) + Unsupported: 2 ( 4.3%) + +Coverage: 95.7% +``` + +### Step 6 — Detail agentic activities + +For activities classified as `agentic`, explain which skill from the `adf-to-databricks-plugin` will handle each: + +| Activity | Type | Handling Skill | +|---|---|---| +| RunDataFlow | ExecuteDataFlow | `adf-to-databricks:adf-dataflow-converter` | +| BranchLogic | Switch | `adf-to-databricks:adf-pipeline-converter` | +| ... | ... | ... | + +### Step 7 — Warn about unsupported activities + +For activities classified as `unsupported`, warn the user clearly: + +``` +WARNING: The following activities have no automated translation path: + - Pipeline "ETL_Main" / Activity "RunSSIS" (ExecuteSSISPackage) + Recommendation: Manual conversion to PySpark notebook required. +``` + +### Step 8 — Confirm output location + +Tell the user where the metadata files were written (`/metadata/`: inventory.json, profile_report.csv, and the per-pipeline `.arm.json`), summarise the complexity sizes, and confirm they can proceed to the `convert` phase using the same ``. + +## Examples + +- "Discover my ADF pipelines from /Volumes/main/default/adf_export" +- "Parse ADF definitions from ./tests/resources/json/" +- "Load the ADF pipeline JSON files and show me the inventory" +- "Import pipelines from /tmp/customer_adf_export" +- "Discover only the pl_demo_01 pipeline from /Volumes/main/default/adf_export" + +## Output Artifacts + +All under the shared `/metadata/` folder: + +| File | Description | +|---|---| +| `metadata/inventory.json` | Classified activity inventory for the convert phase | +| `metadata/profile_report.csv` | Per-pipeline complexity report (counts + T-shirt size) | +| `metadata/.arm.json` | Verbatim original ADF/ARM source for each pipeline | diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md new file mode 100644 index 0000000..9f895bb --- /dev/null +++ b/skills/flowx-migrate/SKILL.md @@ -0,0 +1,422 @@ +--- +name: flowx-migrate +description: > + End-to-end migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs. + Orchestrates discover, convert, and package phases in sequence. +triggers: + - "migrate ADF" + - "migrate pipelines" + - "ADF to Databricks" + - "migrate to Lakeflow" + - "ADF migration" + - "convert ADF to Lakeflow" + - "migrate data factory" +--- + +# End-to-End ADF to Databricks Migration + +Orchestrate the complete migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. This skill runs all three phases in sequence: discover, convert, package. + +## Context + +This is the top-level orchestration skill. It runs the full migration pipeline: + +1. **Discover** — Parse ADF JSON exports into a typed inventory +2. **Convert** — Convert ADF activities to Databricks IR (deterministic + agentic) +3. **Package** — Generate Databricks Declarative Automation Bundles for deployment + +Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. + +## How to run this skill — MCP tools or venv CLI + +This skill orchestrates all three phases. Run the **`setup`** skill first if you haven't. There are +two execution paths: + +### MCP tools (Databricks Genie Code, or a local stdio registration) + +In Genie Code this is the **only** path — the phases run on the deployed `mcp-flowx` app, so +there is **no venv, no `bootstrap.sh`, and no `.migration-venv`**. Run **no** `python3`/`$PY`/`bash` +commands on this path; the `"$PY" -m …` snippets in the steps below are the **local-CLI fallback +only**. Everything goes through the single **`flowx`** tool, one `command` per step: + +The hosted server **cannot read your workspace/volume files**, so pass the ADF JSON **inline** as +`adf_definitions` (a mapping of relative path → ARM JSON content mirroring the Git-export layout — +`pipeline/…`, `dataset/…`, `linkedService/…`, `trigger/…`). You read those files and supply them. +The **recommended Genie path is a single `migrate` call**, which avoids re-sending the payload per +phase: + +``` +flowx(command="migrate", parameters={ + "adf_definitions": {"pipeline/Foo.json": {...}, "linkedService/Bar.json": {...}, ...}, + "output_dir": ..., "catalog": ..., "schema": ..., "pipeline": ""}) +``` + +**`migrate` is interactive when configuration options exist.** After translating, if the pipeline +raises any configuration options (e.g. how to handle an `activity_and_notify` motif, metadata-driven +bulk-copy consolidation, non-Databricks task compute), it **does not package** — it returns the +**full option schema once**: `{"status": "needs_input", "pending_options": [{"pipeline_name", +"options": [{option_id, prompt, rationale, choices, free_text, default, show_when}, …]}, …], +"report_path", "output_dir"}`. You drive the whole chain locally — no per-answer round trip: + +1. **Ask an option only when its `show_when` is satisfied** — every clause `{option_id, in:[values]}` + must match an answer you've already collected (empty `show_when` = always ask). So + `notify_slack_url` surfaces only after `notify_destination=slack`; the metadata-driven + `access`/`size`/`lookup_tool` chain only after `metadata_driven_consolidate=consolidate`. Present + each option's `prompt`/`rationale`/`choices`; honor the `default`. +2. **Validate** each answer against `choices` (a `free_text` option accepts any value); collect picks + as `"option_id=value"` strings. Run any data action inline (e.g. the lookup query when + `metadata_driven_lookup_tool=have`). +3. When **every applicable option** is answered, call `migrate` **once** with the same parameters plus + `"answers": ["option_id=value", …]`. It applies them and packages (`"status": "completed"`). + +To accept all defaults and skip the prompts, pass `"interactive": false`. (Re-calling `migrate` with +`answers` reuses the existing report and skips re-running discover/convert.) + +> **Large factories (hundreds–thousands of pipelines): do not inline.** Inline `adf_definitions` +> passes through your context window and is capped (~5 MB). Instead point the server at the source by +> reference: either stage the ADF export to a **UC Volume** and pass +> `"adf_volume_path": "/Volumes/cat/sch/adf_export"` (read via the SDK Files API), or pass +> `"adf_workspace_path": "/Workspace/Shared/adf_export"` for an ADF Git folder already in the workspace +> (read via the SDK Workspace API). For output, pass `"output_volume_path": "/Volumes/cat/sch/dab"` +> **or** `"output_workspace_path": "/Workspace/Shared/dab"` so the generated bundle is written to that +> target via the SDK (returned as `bundle_uploaded` instead of inline `bundle`). Grant the +> `mcp-flowx` app's service principal read on the source and write on the output target. + +For step-by-step control, run the commands in order (the app reuses `output_dir` across calls, so +only `discover` needs `adf_definitions`): + +``` +flowx(command="inputs", parameters={"phase": "discover" | "convert" | "package"}) # learn each phase's inputs +flowx(command="discover", parameters={"adf_definitions": {...}, "output_dir": ..., "pipeline": ...}) +flowx(command="convert", parameters={"output_dir": ..., "pipeline": ...}) +flowx(command="merge_agentic", parameters={"report_path": ..., "agentic_results_dir": ..., "output_path": ...}) # if agentic results +flowx(command="inspect", parameters={"report_path": ...}) +flowx(command="apply_answers", parameters={"report_path": ..., "answers": [...], "output_dir": ...}) +flowx(command="package", parameters={"output_dir": ..., "catalog": ..., "schema": ...}) +flowx(command="record_results", parameters={...}) / flowx(command="install_dashboard", parameters={...}) +``` + +The server's `output_dir` is ephemeral and not reachable from your workspace, so **have `migrate`/ +`package` write the DAB to the target via the SDK** — pass `"output_volume_path": "/Volumes/…"` or +`"output_workspace_path": "/Workspace/…"` and the bundle is uploaded there (returned as +`bundle_uploaded`). Only when neither is set is the bundle returned inline as `bundle = {"files": +{relpath: text,…}, "truncated": [...]}` (small bundles), which you must then persist yourself. Either +way the user ends up with the DAB to validate and deploy. Each call returns a structured result +(summaries / file trees); use those in place of reading files. Wherever a step below shows `"$PY" -m flowx.adapter …`, call +`flowx(command="", parameters={...})` instead. + +> `databricks bundle validate` / `deploy` of the *generated* bundle is still a user-driven CLI step +> (web terminal / local / CI-CD); present the bundle for review. + +### venv CLI (local, no MCP server) + +Ensure the venv exists (`setup` Path B / `bootstrap.sh`), then run the commands below with the venv +interpreter (from the marker file `/.migration-venv`) and `src/` on `PYTHONPATH` (use +`$PY` anywhere a command shows `python3`): + +```bash +export PYTHONPATH="/src" +PY="$(cat /.migration-venv)" +"$PY" -m flowx.adapter inputs discover +``` + +If Python or pip is missing, `bootstrap.sh` prints a warning telling the user what to install — relay +it and stop until they have Python 3.12+ and pip. + +## Workflow + +Follow these steps in order: + +### Step 0 — Gather phase inputs via the adapter + +Before invoking discover, run the adapter inputs subcommand once per +phase so the agent surfaces the matching free-text prompts: + +```bash +"$PY" -m flowx.adapter inputs discover +"$PY" -m flowx.adapter inputs convert +"$PY" -m flowx.adapter inputs package +``` + +Each response carries the options for that phase plus their descriptions and +defaults. Collect answers from the user (or accept the defaults) and thread the +values into the downstream CLI calls. All phases share **one** migration +`` (default `./flowx_output`). + +### Step 1 — Gather inputs + +Ask the user for all required inputs upfront: + +| Parameter | Description | Required | Default | +|---|---|---|---| +| ADF source path | UC volume path or local directory with ADF JSON files | Yes | — | +| Output directory | Single shared root for all flowx output (bundle + `metadata/`) | No | `./flowx_output` | +| Target catalog | Unity Catalog catalog for tables/volumes | No | `main` | +| Target schema | Schema within the catalog | No | `default` | +| Bundle name | Name for the generated DABs project | No | derived from pipelines | + +Example prompt: + +> To migrate your ADF pipelines, I need: +> 1. Where are your ADF JSON exports? (UC volume path like `/Volumes/main/default/adf_export` or local directory) +> 2. Where should I write the output? (default: `./flowx_output/`) +> 3. What target catalog and schema? (default: `main.default`) + +### Step 2 — Phase 1: Discover + +Invoke the `flowx:flowx-discover` skill with the ADF source path and `--output-dir ` (the shared migration dir). Profile writes `/metadata/{inventory.json, profile_report.csv, .arm.json}`. + +Wait for discover to complete and present the inventory summary: + +``` +Phase 1: Discover — Complete +========================== +Pipelines parsed: 12 +Total activities: 47 + Deterministic: 35 (74.5%) + Agentic: 10 (21.3%) + Unsupported: 2 ( 4.3%) +Coverage: 95.7% +``` + +### Step 3 — Checkpoint: confirm proceed + +Ask the user to review the inventory and confirm before continuing: + +> The discover phase found 47 activities across 12 pipelines. 95.7% have a translation path (74.5% deterministic, 21.3% agentic). 2 activities are unsupported and will need manual handling. +> +> Proceed to the translation phase? (yes/no) + +If the user says no, explain the options: +- Re-run discover with a different source directory +- Review `/metadata/inventory.json` (and `profile_report.csv`) to understand unsupported activities and pipeline complexity +- Manually classify activities before proceeding + +If the user says yes, proceed to step 4. + +### Step 4 — Phase 2: Convert + +Invoke the `flowx:flowx-convert` skill with: +- ADF source dir: the original ADF source path (same `--source-dir` as discover) +- Output dir: the same shared `` (convert writes its report to `/.work/`) + +Wait for the translation to complete and present the summary: + +``` +Phase 2: Convert — Complete +============================= +Deterministic translated: 35 (74.5%) +Agentic translated: 8 (17.0%) +Failed: 4 ( 8.5%) +Overall coverage: 91.5% +``` + +### Step 5 — Present translation details + +Show the user: +1. What was translated deterministically (bulk — just counts by type) +2. What was translated via agentic skills (list each with the skill used) +3. What failed and why (list each with the failure reason) + +For failures, suggest: +- Manual notebook creation +- Retry with additional context +- Skip and add placeholder + +### Step 5.1 — Gather just-in-time translation configuration + +Run `inspect` **once** to get the full option schema (every option carries a `show_when` condition), +then drive the chain locally — ask an option only when its `show_when` clauses are all satisfied by +the answers collected so far; never re-run `inspect` per follow-up. When the user opts to consolidate +a metadata-driven motif and the agent has a database tool, run the lookup query to get CSV rows; +otherwise prompt the user for a CSV file path or literal CSV string. Apply everything in **one** +`modify` call (all `--answer OPTION_ID=VALUE` flags, plus `--lookup-csv ""` when needed — +no intermediate JSON file). + +#### Legacy flow details + +Before bundle generation, run the adapter inspect CLI on the translation +report to surface any pipeline-modifier options the IR raises: + +```bash +"$PY" -m flowx.adapter inspect /.work/translation_report.json +``` + +`inspect` returns the full option tree at once. Ask each option only when its `show_when` clauses +(`{option_id, in:[values]}`) are all satisfied by the answers collected so far (empty = always); +present its rationale, choices, and affected task keys. Then apply all collected answers in one +`modify` call as repeatable `--answer OPTION_ID=VALUE` flags: + +```bash +"$PY" -m flowx.adapter modify \ + /.work/translation_report.json \ + --output-dir \ + --answer copy_activity_paradigm=sdp \ + --answer non_databricks_task_compute=serverless \ + [--lookup-csv ""] +``` + +`modify` writes the stamped report to `/.work/translation_report.stamped.json` +and the kept answers record to `/metadata/configuration.json`. The package phase +reads the stamped report from `.work/` automatically. When inspect emits no options for any +pipeline, skip `modify` — package falls back to the un-stamped report. + +The options the adapter raises: + +| `option_id` | Allowed values | Default | +|---|---|---| +| `copy_activity_paradigm` | `notebook`, `sdp` | `notebook` | +| `non_databricks_task_compute` | `serverless`, `classic` | `serverless` | +| `use_lakeflow_connectors` | `existing`, `lakeflow_connect` | `existing` | +| `consolidate_motif:` | `keep`, `consolidate` | `keep` | + +DatabricksNotebook and DatabricksSparkPython tasks always inherit the cluster binding derived from +their source linked service. + +For each multi-activity motif the detector matches (rest_api_pagination, +incremental_load_watermark, metadata_driven_bulk_copy, ...) the adapter emits one +`consolidate_motif:` option. The user must explicitly opt in to `consolidate` +for each detected pattern. + +### Step 6 — Checkpoint: confirm proceed to bundle generation + +> Translation is 91.5% complete. 4 activities could not be translated automatically. +> Options: +> 1. Proceed to bundle generation (failed activities will get placeholder tasks) +> 2. Retry failed translations with more context +> 3. Stop here and review the translation report +> +> What would you like to do? + +### Step 6.5 — Detect workspace artifacts and authenticate + +Before invoking the package phase, run the adapter's +`workspace-paths` subcommand to detect any absolute workspace paths +the bundle would need to download: + +```bash +"$PY" -m flowx.adapter workspace-paths \ + /.work/translation_report.stamped.json \ + --source-dir +``` + +When the response carries `needs_auth: true`: + +1. Confirm the workspace host with the user, defaulting to the first + entry in `suggested_hosts` (extracted from the Databricks linked + services in the ADF export). +2. Run `databricks auth login --host ` interactively to set up + a local profile. +3. Pass `--profile ` to the prepare invocation in Step 7 so + flowx downloads the referenced notebooks and downloads them under + `bundle/src/notebooks/` with the task references rewritten to the + relative `../src/notebooks/...` paths. + +Skip this step entirely when `needs_auth` is `false`. + +### Step 7 — Phase 3: Package + +Invoke the `flowx:flowx-package` skill with: +- Output dir: the same shared `` — package reads the stamped report from + `/.work/` automatically (no report path needed) and writes the bundle here +- Catalog: user-specified or `main` +- Schema: user-specified or `default` + +Package prunes the transient `/.work/` after a successful build, leaving the +bundle (databricks.yml, resources/, src/, SETUP.md) plus the kept `metadata/` folder. + +### Step 7.5 — (Optional) Persist coverage results and install a dashboard + +When running with workspace auth (Genie Code or a configured profile), offer to record this +run's migration coverage to a Unity Catalog table and optionally install a coverage dashboard. +The `inputs package` prompts surface `results_table`, `results_warehouse_id`, and +`install_dashboard`. + +If the user supplies a `results_table`: + +```bash +"$PY" -m flowx.adapter record-results \ + --output-dir --results-table [--warehouse-id ] +``` + +Writes one row per pipeline (counts, complexity size, deterministic/agentic/unsupported +coverage), each stamped with a UUID `run_id`, `run_date` (`CURRENT_TIMESTAMP()`), and `run_by` +(`CURRENT_USER()`). If `install_dashboard = yes`: + +```bash +"$PY" -m flowx.adapter install-dashboard --results-table [--warehouse-id ] +``` + +Creates and publishes an AI/BI coverage dashboard over the table and prints its URL. Both +auto-detect a SQL warehouse when `--warehouse-id` is omitted and degrade gracefully without +workspace auth. See the `package` skill (Step 8) for details. + +### Step 8 — Present final summary + +Display the complete migration summary: + +``` +Migration Complete +================== + +Source: /Volumes/main/default/adf_export (12 ADF pipelines) +Output: ./flowx_output/ (bundle + metadata/) + +Coverage: + Total activities: 47 + Successfully translated: 43 (91.5%) + Placeholder tasks: 4 ( 8.5%) + +Generated Files (under ./flowx_output/): + databricks.yml + resources/ (3 job definitions) + src/notebooks/ (12 notebooks) + src/setup/ (3 setup scripts) + SETUP.md + metadata/ inventory.json, profile_report.csv, .arm.json, configuration.json + +Setup Required: + - Run setup/create_volumes.py to create UC volumes + - Run setup/create_secrets.py to configure secrets (review credentials first) + - Run setup/register_connections.py to register external connections + +Next Steps: + 1. cd ./flowx_output/ + 2. Review generated files, especially notebooks and setup scripts + 3. databricks bundle validate --target dev + 4. Run setup scripts on the target workspace + 5. databricks bundle deploy --target dev + 6. databricks bundle run --target dev + 7. Verify job output and promote to staging/prod +``` + +### Step 9 — Offer follow-up actions + +Ask if the user wants to: +1. Validate the bundle now (`databricks bundle validate`) +2. Deploy to dev (`databricks bundle deploy --target dev`) +3. Review specific generated files +4. Re-translate any failed activities +5. Export a migration report for documentation + +## Reference + +See `references/workflow.md` for a detailed description of the three-phase architecture. + +## Examples + +- "Migrate my ADF pipelines to Databricks" +- "Convert ADF to Lakeflow jobs" +- "ADF to Databricks migration from /Volumes/main/default/adf_export" +- "Migrate data factory pipelines to catalog analytics, schema bronze" +- "Run the full ADF migration workflow" + +## Output Artifacts + +All three phases write into a single shared `` (default `./flowx_output`): + +| Path | Phase | Contents | +|---|---|---| +| `metadata/` | Profile + Modify | `inventory.json`, `profile_report.csv`, `.arm.json`, `configuration.json` | +| `databricks.yml`, `resources/`, `src/`, `SETUP.md` | Package | The deployable DAB bundle | +| `.work/` | Convert/Modify (transient) | Translation report + IR; pruned by package | diff --git a/skills/migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md similarity index 93% rename from skills/migrate/references/workflow.md rename to skills/flowx-migrate/references/workflow.md index 93e997d..379a958 100644 --- a/skills/migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -1,16 +1,16 @@ -# Flowx Migration Workflow +# flowx Migration Workflow End-to-end architecture for migrating Azure Data Factory (ADF) pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles (DABs). ## Overview -Flowx follows a three-phase pipeline architecture. Each phase is independently runnable and produces artifacts consumed by the next phase. The design principle is **deterministic-first, agentic fallback**: well-known ADF patterns are translated by fast, reliable Python code, while complex or ambiguous patterns are handled by LLM-assisted skills. +flowx follows a three-phase pipeline architecture. Each phase is independently runnable and produces artifacts consumed by the next phase. The design principle is **deterministic-first, agentic fallback**: well-known ADF patterns are translated by fast, reliable Python code, while complex or ambiguous patterns are handled by LLM-assisted skills. ``` ADF JSON Exports | v - Phase 1: INGEST + Phase 1: PROFILE (parse + classify) | v @@ -35,9 +35,9 @@ ADF JSON Exports databricks bundle deploy ``` -## Phase 1: Ingest +## Phase 1: Discover -**Skill:** `flowx:ingest` +**Skill:** `flowx:flowx-discover` **Input:** Directory of ADF JSON export files (from ARM template export, Azure DevOps, or manual export) @@ -60,9 +60,9 @@ ADF JSON Exports - Datasets and linked services are parsed for context but not independently translated — they inform the activity translators. - Triggers are included in the inventory and translated in phase 2. -## Phase 2: Translate +## Phase 2: Convert -**Skill:** `flowx:translate` +**Skill:** `flowx:flowx-convert` **Input:** `inventory.json` from phase 1 + original ADF JSON files @@ -84,13 +84,13 @@ ADF JSON Exports **Key decisions:** - Deterministic translators run first because they are fast and reliable. Agentic skills are only invoked for gaps. -- The IR is an intermediate format that decouples translation from DABs generation. This allows the prepare phase to target different output formats in the future. +- The IR is an intermediate format that decouples translation from DABs generation. This allows the package phase to target different output formats in the future. - Each deterministic translator is a standalone Python module in `src/flowx/translator/activity_translators/`. Adding support for a new activity type means adding a new module. - Agentic results are saved separately before merging, so they can be inspected, retried, or manually overridden. -## Phase 3: Prepare +## Phase 3: Package -**Skill:** `flowx:prepare` +**Skill:** `flowx:flowx-package` **Input:** `translation_report.json` from phase 2 @@ -140,7 +140,7 @@ The Databricks IR (intermediate representation) sits between ADF semantics and D - **Semantic mapping** — translating ADF concepts to Databricks concepts - **Serialization** — writing DABs YAML and notebooks -This means the prepare phase could target different output formats (Terraform, raw API calls, etc.) without changing the translation logic. +This means the package phase could target different output formats (Terraform, raw API calls, etc.) without changing the translation logic. ## ADF Concepts to Databricks Mapping @@ -164,4 +164,4 @@ This means the prepare phase could target different output formats (Terraform, r - **Parse errors** — logged to `parse_errors.json`, skipped in inventory - **Translation failures** — marked as `failed` in translation report, get placeholder tasks in DABs - **Agentic failures** — saved with error details, can be retried with additional context -- **Unsupported activities** — warned at ingest, get placeholder tasks with TODO comments in DABs +- **Unsupported activities** — warned at discover, get placeholder tasks with TODO comments in DABs diff --git a/skills/flowx-package/SKILL.md b/skills/flowx-package/SKILL.md new file mode 100644 index 0000000..f22769e --- /dev/null +++ b/skills/flowx-package/SKILL.md @@ -0,0 +1,344 @@ +--- +name: flowx-package +description: > + Generate Databricks Declarative Automation Bundles (DABs) from translated IR, + including job definitions, notebooks, and setup scripts. +triggers: + - "package bundles" + - "generate DABs" + - "create bundles" + - "package deployment" + - "generate bundles" + - "build DABs" +--- + +# Package Databricks Declarative Automation Bundles + +Generate deployment-ready Databricks Declarative Automation Bundles (DABs) from the translated intermediate representation, including job definitions, notebooks, and infrastructure setup scripts. + +## Context + +This is phase 3 of the flowx migration workflow. It consumes the `translation_report.json` produced by the `convert` skill and generates a complete DABs project that can be validated and deployed with the Databricks CLI. + +The output is a standard DABs project with: +- `databricks.yml` — the bundle configuration +- `resources/` — job and pipeline YAML definitions +- `src/notebooks/` — generated and helper notebooks +- `setup/` — infrastructure setup scripts (volumes, secrets, connections) + +## How to run this skill — MCP tools or venv CLI + +This phase runs one of two ways; run the **`setup`** skill first if you haven't. + +- **MCP tool (Databricks Genie Code, or a local stdio registration) — the only path in Genie Code:** + call the single **`flowx`** tool (one command per step) and run **no** `python3`/`$PY`/`bash` + commands. The `"$PY" -m …` snippets in the steps below are the **local-CLI fallback only** — ignore + them on this path. Map the steps to: + + ``` + flowx(command="package", parameters={"output_dir": "", "report_path": "", "catalog": "", + "schema": "", "bundle_name": "", "profile": "", + "download_workspace_files": true, + "output_volume_path": "", + "output_workspace_path": ""}) + flowx(command="workspace_paths", parameters={"report_path": "...", "source_dir": ""}) + flowx(command="record_results", parameters={"output_dir": "", "results_table": "catalog.schema.table", "warehouse_id": ""}) + flowx(command="install_dashboard", parameters={"results_table": "catalog.schema.table", "warehouse_id": ""}) + ``` + + The server's `output_dir` is ephemeral and not reachable from your workspace, so **have `package` + write the bundle to the target itself via the SDK** — don't try to persist files yourself. Pass one + of: + - `"output_volume_path": "/Volumes/cat/sch/dab"` — uploads the DAB to that UC Volume (SDK Files API). + - `"output_workspace_path": "/Workspace/Shared/dab"` — uploads it to that workspace folder (SDK + Workspace API; files written verbatim, not imported as notebooks). + + Either returns `bundle_uploaded = {"output_volume_path"|"output_workspace_path", "files", "count"}`. + Only when **neither** is given does `package` return the contents inline as + `bundle = {"files": {relpath: text, …}, "truncated": [...]}` (small bundles only) for you to persist. + Prefer an output path so the bundle lands durably and large bundles aren't capped. Skip the + `"$PY" -m …` commands below. + +- **venv CLI (local, no MCP server):** ensure the venv exists (`setup` Path B / `bootstrap.sh`), then + run the commands below with the venv interpreter (from the marker file `/.migration-venv`) + and `src/` on `PYTHONPATH` (use `$PY` anywhere a command shows `python3`): + + ```bash + export PYTHONPATH="/src" + PY="$(cat /.migration-venv)" + "$PY" -m flowx.adapter package --output-dir --catalog --schema + ``` + + If Python or pip is missing, `bootstrap.sh` prints a warning telling the user what to install — + relay it and stop until they have Python 3.12+ and pip. + +## Workflow + +Follow these steps in order: + +### Step 1 — Locate the translation report + +The translate/modify phases left the stamped report at `/.work/translation_report.stamped.json` +(or `/.work/translation_report.json` when `modify` was not run). If the shared +`` is not in conversation context, ask the user: + +> Which migration output directory should I build the bundle in? (default: `./flowx_output`) + +`package` reads the report from `/.work/` automatically — you do not pass a report path. +Validate that a report exists there and that all required translations have status `translated`. + +### Step 2 — Gather deployment parameters + +Ask the user for the following (provide defaults): + +| Parameter | Description | Default | +|---|---|---| +| Target catalog | Unity Catalog catalog for tables/volumes | `main` | +| Target schema | Schema within the catalog | `default` | +| Output directory | Shared migration dir; bundle + `metadata/` are written here | `./flowx_output` | +| Bundle name | Name for the DABs project | derived from first pipeline name | +| Target environments | Deployment targets to configure | `dev, staging, prod` | +| Warehouse ID | SQL warehouse for SQL tasks (optional) | prompt if SQL tasks exist | +| Databricks CLI profile | Profile used to download workspace-resident notebooks / JARs / Python files (`--profile`). Required only when the bundle references absolute workspace paths. | resolved from `~/.databrickscfg` (auto-prompt if multiple) | + +### Step 2.5 — Detect workspace artifacts and authenticate + +> **Databricks runtime (serverless / cluster):** Authentication is auto-configured +> from the notebook runtime context. The `workspace_downloader` module detects +> `DATABRICKS_RUNTIME_VERSION` in the environment and writes `~/.databrickscfg` +> from `dbruntime.databricks_repl_context` automatically. You can skip the +> interactive `databricks auth login` step — just pass `--profile DEFAULT` (or +> omit `--profile` entirely) and notebook downloading will work. + +Before running the bundle writer, check whether the report references +absolute workspace paths (notebooks under `/Shared/`, SparkPython +files, SparkJar libraries) or DBFS paths that the bundle should +download to be self-contained: + +```bash +"$PY" -m flowx.adapter workspace-paths \ + /.work/translation_report.stamped.json \ + --source-dir +``` + +The command emits: + +```json +{ + "paths": ["/Shared/team/notebook_a", "/Shared/team/notebook_b"], + "suggested_hosts": ["https://adb-1234.5.azuredatabricks.net"], + "needs_auth": true +} +``` + +When `needs_auth` is `true`: + +1. Surface the suggested hosts to the user with `AskUserOption`. Use + the first `suggested_hosts` value as the default; allow the user to + override. When no host is suggested (no Databricks linked service + in the export), prompt for the host with no default. +2. Run the interactive Databricks CLI login command and wait for it to + complete: + + ```bash + databricks auth login --host + ``` + + This writes a profile into `~/.databrickscfg`. When the user has + chosen a specific profile name, append `--profile ` to both + the login and the package invocation below. + +3. Pass the resolved profile to step 3 via `--profile ` (default + profile name is `DEFAULT`). When `needs_auth` is `false` skip steps + 1–2 and omit `--profile` from step 3. + +The `paths` list is informational; you can echo it to the user so they +know which notebooks the bundle will download. + +### Step 3 — Run bundle generation + +Execute the DAB writer: + +```bash +# Unified runner (recommended): `"$PY" -m flowx.adapter package ...` +# forwards to dab_writer below. +"$PY" -m flowx.bundler.dab_writer \ + --output-dir \ + --catalog \ + --schema \ + --bundle-name \ + [--profile ] \ + [--no-download-workspace-files] \ + [--keep-intermediates] +``` + +Where: +- `` is the shared migration directory — `package` defaults `--report` to + `/.work/translation_report.stamped.json` (falling back to the un-stamped report). + Pass `--report ` only to override. +- Other parameters are from step 2 +- After a successful build, `package` **prunes the transient `/.work/`** so the + final tree contains only the bundle and the kept `metadata/` files. Pass `--keep-intermediates` + to retain `.work/` for debugging. + +**Workspace artifact downloading (default: enabled).** When the report references workspace-resident notebooks (`/Shared/...`), DBFS Spark JARs (`dbfs:/...`), or Spark Python files, the preparer downloads them via the Databricks CLI auth so the resulting bundle is self-contained and deployable across environments. Downloaded notebooks are downloaded under `src/notebooks/` and bound to the default `job_cluster` (since they may rely on classic-compute features). The original `notebook_path` in the resource YAML is rewritten to the bundle-relative path `../src/notebooks/.py`. + +If no Databricks CLI auth is detected on the host (`~/.databrickscfg` empty AND no `DATABRICKS_CONFIG_PROFILE` / `DATABRICKS_HOST`+`DATABRICKS_TOKEN` env vars), the CLI prints the workspace paths it was about to download and prompts: + +``` +Workspace downloads are enabled but no Databricks CLI auth was found. + Looked for profiles in: /Users//.databrickscfg + Artifacts to download: /Shared/ETL/transform, … + +To authenticate, run one of: + databricks auth login --host https://.cloud.databricks.com + databricks configure --token + +Continue with placeholders (downloads will be skipped)? [y/N]: +``` + +Answering `n` aborts with exit code 2 so the user can authenticate and re-run. Answering `y` continues with placeholder notebooks (legacy in-place workspace paths). In non-interactive sessions the prompt defaults to placeholders. + +Use `--no-download-workspace-files` to opt out entirely; the bundle then keeps original workspace paths exactly as in the IR. + +### Step 4 — Present the generated file tree + +Show the user what was generated: + +``` +/ # the shared migration directory + databricks.yml + resources/ + etl_main_job.yml + transform_dlt_pipeline.yml + src/ + notebooks/ + copy_from_blob.py + web_activity_call.py + setup/ + create_volumes.py + create_secrets.py + register_connections.py + SETUP.md + metadata/ # kept migration metadata (from discover + modify) + inventory.json + profile_report.csv + .arm.json # verbatim original ADF/ARM source + configuration.json # the collected configuration answers + # .work/ (transient translation report + IR) is pruned after a successful build +``` + +### Step 5 — Explain setup tasks + +If the `setup/` directory was generated, explain what each script does: + +**create_volumes.py** — Creates Unity Catalog volumes required by the migrated jobs. These volumes replace Azure Blob Storage or ADLS references from ADF. Run this once per environment. + +**create_secrets.py** — Creates Databricks secret scopes and secrets for connection credentials that were in ADF linked services. Review the secret values and populate them manually or via your secrets management system. + +**register_connections.py** — Registers Unity Catalog connections for external data sources (SQL Server, REST APIs, etc.) that were referenced in ADF linked services. + +Emphasize that the user should review these scripts before running them, especially `create_secrets.py` which will need actual credential values. + +### Step 6 — Explain the generated bundle structure + +Briefly describe: +- **databricks.yml** — The root bundle config with workspace, target environments (dev/staging/prod), and variable definitions. Variables are parameterized for environment-specific values (catalog, schema, warehouse). +- **resources/*.yml** — One YAML file per Databricks Lakeflow Job (one per ADF pipeline). Each job contains tasks mapped from ADF activities, with dependencies matching the original ADF dependency chains. +- **src/notebooks/*.py** — Python notebooks for activities that translate to notebook_task. These contain the actual data movement or transformation logic. +- **tests/*.py** — Skeleton test files for validating the migrated jobs. + +### Step 7 — Suggest next steps + +Present the following next steps: + +``` +Next Steps +========== +1. Review generated files: + cd + cat databricks.yml + +2. Validate the bundle: + databricks bundle validate --target dev + +3. Run setup scripts (if generated): + databricks bundle run setup_volumes --target dev + +4. Deploy to dev: + databricks bundle deploy --target dev + +5. Test the deployed jobs: + databricks bundle run --target dev + +6. Promote to staging/prod: + databricks bundle deploy --target staging + databricks bundle deploy --target prod +``` + +Recommend running `databricks bundle validate` first to catch any configuration issues before deployment. + +### Step 8 — (Optional) Persist coverage results and install a dashboard + +This step only applies when running with workspace auth (Genie Code, or a configured +Databricks CLI profile). The `inputs package` options surface three optional prompts: +`results_table`, `results_warehouse_id`, and `install_dashboard`. + +**Persist results.** When the user provides a `results_table` (a UC `catalog.schema.table`), +write one migration-coverage row **per pipeline** for this run: + +```bash +"$PY" -m flowx.adapter record-results \ + --output-dir \ + --results-table \ + [--warehouse-id ] +``` + +It reads `/metadata/{inventory.json, profile_report.csv}`, creates the table if +needed, and inserts a row per pipeline with the activity/dataset/linked-service counts, +collapsible-pattern count, complexity size, and the deterministic/agentic/unsupported coverage +breakdown. Every row is stamped with a shared **`run_id`** (UUID for this run), +**`run_date`** (`CURRENT_TIMESTAMP()`), and **`run_by`** (`CURRENT_USER()`). The warehouse is +auto-detected (prefers a running serverless warehouse) when `--warehouse-id` is omitted. The +command prints the `run_id` and row count. + +**Install the dashboard.** When the user answers `install_dashboard = yes`, create and publish +an AI/BI (Lakeview) coverage dashboard over that table: + +```bash +"$PY" -m flowx.adapter install-dashboard \ + --results-table \ + [--warehouse-id ] \ + [--dashboard-name ""] [--parent-path "/Workspace/Users/"] +``` + +It builds the dashboard from a template (KPI counters for pipelines / coverage % / +deterministic-agentic-unsupported activity totals, a complexity-size bar chart, a +coverage-over-runs line, and a per-pipeline coverage table), publishes it, and prints the URL. +Both commands degrade gracefully with an actionable message when workspace auth or a warehouse +is unavailable. + +## Examples + +- "Package the bundles" +- "Generate DABs for the translated pipelines" +- "Create deployment bundles targeting catalog 'analytics' and schema 'bronze'" +- "Build the DABs project in ./output/my_migration/" + +## Output Artifacts + +All under the shared ``: + +| File | Description | +|---|---| +| `databricks.yml` | Root bundle configuration | +| `resources/*.yml` | Job and pipeline YAML definitions | +| `src/notebooks/*.py` | Generated notebooks | +| `src/setup/*.py` | Infrastructure setup scripts | +| `SETUP.md` | Human-readable setup instructions | +| `metadata/inventory.json` | Activity inventory (from discover) | +| `metadata/profile_report.csv` | Per-pipeline complexity report (from profile) | +| `metadata/.arm.json` | Verbatim original ADF/ARM source (from discover) | +| `metadata/configuration.json` | Collected configuration answers (from modify) | + +> **Notification destinations.** When a `activity_and_notify` motif was opted into a Slack/Teams/PagerDuty/Generic-Webhook destination, the destination is created (or reused by display name) via the SDK at **prompt time** (the `modify` phase), and its resolved id is carried in the report; package simply wires that id into the task's `webhook_notifications`. If the report has no pre-resolved id (creation was deferred or failed earlier), package retries the create; failing that — e.g. no workspace auth — a `notification_destination` setup task is emitted in SETUP.md instead and the task ships without notifications. Email destinations use raw `email_notifications` and never create an SDK destination. diff --git a/skills/flowx-setup/SKILL.md b/skills/flowx-setup/SKILL.md new file mode 100644 index 0000000..f319580 --- /dev/null +++ b/skills/flowx-setup/SKILL.md @@ -0,0 +1,172 @@ +--- +name: flowx-setup +description: > + Prepare flowx to run its phases (discover, convert, package, migrate). In Databricks + Genie Code this deploys the phases as an MCP server (a Databricks App) and creates NO virtual + environment — all code runs through MCP. Everywhere else it provisions a Python virtual + environment for the CLI skills, and optionally a local (stdio) MCP server. Run this once before + any other flowx skill, or whenever the environment is missing. +triggers: + - "setup flowx" + - "bootstrap flowx" + - "install flowx dependencies" + - "flowx environment" + - "create flowx venv" + - "ModuleNotFoundError flowx" + - "install flowx mcp" + - "deploy flowx mcp" +--- + +# Set up flowx + +flowx runs its phases (`discover`, `convert`, `package`, `migrate`) in one of two ways. This +skill prepares whichever fits your environment, keyed on `DATABRICKS_RUNTIME_VERSION` (the same +signal the rest of the plugin uses to detect Databricks): + +- **Databricks Genie Code** (`DATABRICKS_RUNTIME_VERSION` *set*) → the phases run as **MCP tools** + hosted on a Databricks App. **No virtual environment is created** — the app vendors its own copy + of the flowx code and dependencies, so the phase skills just call the single `flowx` MCP tool. +- **Local / Claude Code / other agents** (`DATABRICKS_RUNTIME_VERSION` *unset*) → the phases run + from a **Python virtual environment** via the CLI, with an optional local (stdio) MCP server. + +## Step 1 — Pick the path + +```bash +if [ -n "${DATABRICKS_RUNTIME_VERSION:-}" ]; then + echo "Databricks / Genie Code → deploy the MCP server (Path A, no venv)" +else + echo "Local → create the virtual environment (Path B)" +fi +``` + +--- + +## Path A — Databricks Genie Code (MCP, no virtual environment) + +In Genie Code the phases run on the deployed app, so **do not run `bootstrap.sh` and do not create a +venv** — it isn't needed. Deploy the MCP server instead: + +```bash +bash /app/deploy.sh +``` + +`app/deploy.sh` stages a self-contained bundle (the app entrypoint plus a vendored copy of the +flowx source), syncs it to **`/Workspace/Shared/mcp-flowx`**, and creates/deploys the +**`mcp-flowx`** Databricks App. The script prints the app URL; the MCP endpoint is +`/mcp`. It only needs the Databricks CLI and a system `python3` (for parsing CLI output) — +**not** an flowx venv. + +> **Clone into a shared location.** The app's service principal cannot read private +> `/Workspace/Users/` folders by default, so `deploy.sh` deploys the source from +> `/Workspace/Shared/`. Clone flowx into a Git folder under **`/Workspace/Shared`** +> (e.g. `/Workspace/Shared/flowx`), not your user home. If `/Workspace/Shared` is restricted, +> use another all-users location and pass it via `APP_SOURCE_PATH`. + +After it deploys, relay these follow-up steps to the user (the script also prints them): + +1. **App access:** grant **Can use** on `mcp-flowx` to the users / service principals that will + call it (Apps UI → *Permissions*, or `databricks apps set-permissions mcp-flowx ...`). +2. **Data access:** grant the app's service principal access to the catalogs, schemas, and volumes + the migration touches (plus any SQL warehouse used by the reporting tools). +3. **Add it in Genie Code (Agent mode):** open Genie Code **Settings → MCP Servers → Add Server**, + choose **Custom MCP server**, select the `mcp-flowx` app, and **Save**. The single + `flowx` tool then appears (MCP needs Agent mode; it uses one of the 20 tool slots). + Verify via the health endpoint `/`. + +Once added, the `discover`, `convert`, `package`, and `migrate` skills run **entirely through the +`flowx` MCP tool** (`flowx(command="…", parameters={…})`) — there is no venv, no +`bootstrap.sh`, and no `.migration-venv` marker on this path. + +> **Note:** `databricks apps` deploy commands require a Databricks CLI session (workspace web +> terminal or a local machine), not serverless notebook Python. If the Genie session can't shell +> out to the CLI, run `app/deploy.sh` from the web terminal. (Same constraint as `databricks bundle +> deploy`.) If you see `Error: please specify target`, the CLI attached to a stray `databricks.yml`; +> `deploy.sh` already isolates against this, so re-run it as-is. + +--- + +## Path B — Local / Claude Code (virtual environment) + +The flowx code in `src/flowx/` depends on third-party packages (`pyyaml`, `databricks-sdk`, +`sqlglot`); running it against a bare system Python fails with `ModuleNotFoundError`. Provision an +isolated venv (created once and reused). + +### Step B1 — Run the bootstrap script + +```bash +bash /scripts/bootstrap.sh +``` + +Where `` is the flowx plugin root (the directory containing `src/`, `skills/`, and +`requirements.txt`). The script will: + +1. Check that `python3`, `pip`, and the `venv` module are available. +2. Create the venv at `/.venv`. +3. Install `requirements.txt` into that venv using `pip`. +4. Write the resolved interpreter path to `/.migration-venv` for the other skills. + +### Step B2 — Handle a missing Python or pip + +If Python, pip, or the `venv` module are **not** available, the script prints a `WARNING:` block and +exits non-zero **without** creating anything. Do **not** work around it — relay the warning and stop: + +> ⚠️ Python must be installed before I can set up the flowx environment. +> +> * On macOS: `brew install python`. +> * On Debian/Ubuntu: `sudo apt-get install python3 python3-venv python3-pip`. +> +> Let me know once it's installed and I'll re-run setup. + +Re-run this setup skill after the user confirms Python and pip are installed. + +### Step B3 — Confirm success and how to run Python code + +On success, the script writes the interpreter path to `/.migration-venv`. The phase +skills run Python with that interpreter and `src/` on `PYTHONPATH`. Resolve it from the marker file: + +```bash +export PYTHONPATH="/src" +PY="$(cat /.migration-venv)" +"$PY" -m flowx.adapter inputs discover +``` + +`$PY` resolves to `/.venv/bin/python` (on Windows, `\.venv\Scripts\python.exe`). + +### Step B4 — (Optional) Run the MCP server locally + +To drive the phases through MCP tools locally (instead of the CLI), install the MCP server stack into +the venv and register the stdio server with your MCP client: + +```bash +PY="$(cat /.migration-venv)" +"$PY" -m pip install "mcp>=1.12" "uvicorn>=0.30" "starlette>=0.40" +PYTHONPATH="/src" "$PY" -m flowx.mcp # stdio (default) +``` + +```json +{ + "mcpServers": { + "flowx": { + "command": "", + "args": ["-m", "flowx.mcp"], + "env": { "PYTHONPATH": "/src" } + } + } +} +``` + +## Output + +| Path | Artifact | Description | +|---|---|---| +| A (Genie Code) | `mcp-flowx` Databricks App | Hosts the phases as the single `flowx` MCP tool at `/mcp`; **no venv** is created | +| B (local) | venv at `/.venv` | Virtual environment with the installed dependencies | +| B (local) | `/.migration-venv` | Marker file holding the resolved interpreter path | +| B (local, optional) | local MCP server | `mcp` / `uvicorn` / `starlette` installed into the venv; run with `python -m flowx.mcp` | + +## Examples + +- "Set up flowx" (auto-detects Genie Code vs. local) +- "Deploy the flowx MCP server to Databricks / Genie Code" (Path A) +- "Bootstrap flowx so I can run a migration locally" (Path B) +- "I got a ModuleNotFoundError running discover — fix the environment" (Path B) diff --git a/skills/ingest/SKILL.md b/skills/ingest/SKILL.md deleted file mode 100644 index 0466403..0000000 --- a/skills/ingest/SKILL.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -name: ingest -description: > - Load and parse Azure Data Factory pipeline definitions from Unity Catalog volumes or local directories. - Produces a typed inventory that classifies every activity as deterministic, agentic, or unsupported. -triggers: - - "ingest ADF" - - "load ADF" - - "parse ADF" - - "import pipelines" - - "load pipelines" - - "parse pipelines" - - "inventory ADF" ---- - -# Ingest ADF Pipeline Definitions - -Parse Azure Data Factory pipeline, dataset, linked service, and trigger JSON files into a typed AST and produce a classified inventory. - -## Context - -This is phase 1 of the flowx migration workflow. It takes raw ADF JSON exports and produces an `inventory.json` file that the `translate` skill consumes. The inventory classifies every ADF activity into one of three strategies: - -- **Deterministic** — a built-in translator exists (Copy, DatabricksNotebook, ForEach, IfCondition, etc.) -- **Agentic** — requires LLM-assisted translation via the `adf-to-databricks-plugin` skills (ExecuteDataFlow, Switch, Until, StoredProc, etc.) -- **Unsupported** — no known translation path; requires manual intervention - -## Prerequisite — Python environment - -This skill runs the plugin's Python code, which depends on third-party packages. Before running -any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the -**`setup`** skill, or directly: - -```bash -bash /scripts/bootstrap.sh -``` - -This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or -pip is missing, the script prints a warning telling the user what to install — relay it and stop -until they have installed Python 3.12+ and pip. - -Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` -(use it anywhere a command below shows `python3`): - -```bash -export PYTHONPATH="/src" -"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... -``` - -## Workflow - -Follow these steps in order: - -### Step 1 — Determine the ADF source path - -Ask the user for the location of their ADF JSON exports. Accept either: -- A Unity Catalog volume path (e.g., `/Volumes/main/default/adf_export`) -- A local directory path (e.g., `./adf_export/` or `/tmp/adf_json/`) - -The directory should contain subdirectories or files for: -- `pipeline/` or `pipelines/` — pipeline definition JSON files -- `dataset/` or `datasets/` — dataset definition JSON files (optional) -- `linkedService/` or `linked_services/` — linked service JSON files (optional) -- `trigger/` or `triggers/` — trigger definition JSON files (optional) - -### Step 2 — Download from UC volumes if needed - -If the source path starts with `/Volumes/`, the files live in a Unity Catalog volume and must be downloaded to a local temp directory first. - -Use the `databricks-execution-compute` skill to run the following on the Databricks workspace: - -```python -import os, json, shutil, tempfile - -volume_path = "" -local_dir = tempfile.mkdtemp(prefix="adf_ingest_") - -# Copy from volume to local -for root, dirs, files in os.walk(volume_path): - for f in files: - if f.endswith(".json"): - src = os.path.join(root, f) - rel = os.path.relpath(src, volume_path) - dst = os.path.join(local_dir, rel) - os.makedirs(os.path.dirname(dst), exist_ok=True) - shutil.copy2(src, dst) - -print(f"Downloaded ADF files to: {local_dir}") -``` - -Alternatively, use the Databricks CLI: -```bash -databricks fs cp -r "dbfs:" "" --overwrite -``` - -Set the working source directory to the local temp path for subsequent steps. - -### Step 3 — Run the deterministic parser - -Execute the ADF loader to parse all JSON files and produce the inventory: - -```bash -python3 /src/flowx/parser/adf_loader.py \ - --source-dir \ - --output-dir -``` - -Where: -- `` is the root of the flowx plugin (the directory containing `src/`) -- `` is the local directory containing ADF JSON files -- `` is where to write the parsed output (default: `./orchestra_output/ingest/`) - -This produces: -- `inventory.json` — the classified activity inventory -- `ast/` directory — the typed AST for each pipeline -- `parse_errors.json` — any files that failed to parse - -### Step 4 — Read and validate the inventory - -Read the generated `inventory.json` file. It has this structure: - -```json -{ - "source_dir": "/path/to/adf/json", - "generated_at": "2026-04-07T12:00:00Z", - "pipelines": [ - { - "name": "PipelineName", - "file": "pipeline/PipelineName.json", - "activities": [ - { - "name": "CopyFromBlob", - "type": "Copy", - "strategy": "deterministic", - "translator": "copy.py" - }, - { - "name": "RunDataFlow", - "type": "ExecuteDataFlow", - "strategy": "agentic", - "skill": "adf-to-databricks:adf-dataflow-converter" - } - ] - } - ], - "summary": { - "pipeline_count": 12, - "activity_count": 47, - "deterministic_count": 35, - "agentic_count": 10, - "unsupported_count": 2, - "coverage_pct": 95.7 - } -} -``` - -### Step 5 — Present the summary - -Display a summary table to the user: - -``` -ADF Ingestion Summary -===================== -Pipelines parsed: 12 -Total activities: 47 - -Strategy Breakdown: - Deterministic: 35 (74.5%) - Agentic: 10 (21.3%) - Unsupported: 2 ( 4.3%) - -Coverage: 95.7% -``` - -### Step 6 — Detail agentic activities - -For activities classified as `agentic`, explain which skill from the `adf-to-databricks-plugin` will handle each: - -| Activity | Type | Handling Skill | -|---|---|---| -| RunDataFlow | ExecuteDataFlow | `adf-to-databricks:adf-dataflow-converter` | -| BranchLogic | Switch | `adf-to-databricks:adf-pipeline-converter` | -| ... | ... | ... | - -### Step 7 — Warn about unsupported activities - -For activities classified as `unsupported`, warn the user clearly: - -``` -WARNING: The following activities have no automated translation path: - - Pipeline "ETL_Main" / Activity "RunSSIS" (ExecuteSSISPackage) - Recommendation: Manual conversion to PySpark notebook required. -``` - -### Step 8 — Confirm output location - -Tell the user where the inventory and AST files were written, and confirm they can proceed to the `translate` phase. - -## Examples - -- "Ingest my ADF pipelines from /Volumes/main/default/adf_export" -- "Parse ADF definitions from ./tests/resources/json/" -- "Load the ADF pipeline JSON files and show me the inventory" -- "Import pipelines from /tmp/customer_adf_export" - -## Output Artifacts - -| File | Description | -|---|---| -| `inventory.json` | Classified activity inventory for the translate phase | -| `ast/*.json` | Typed AST for each pipeline | -| `parse_errors.json` | Any files that failed to parse | diff --git a/skills/migrate/SKILL.md b/skills/migrate/SKILL.md deleted file mode 100644 index c0f2b9b..0000000 --- a/skills/migrate/SKILL.md +++ /dev/null @@ -1,324 +0,0 @@ ---- -name: migrate -description: > - End-to-end migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs. - Orchestrates ingest, translate, and prepare phases in sequence. -triggers: - - "migrate ADF" - - "migrate pipelines" - - "ADF to Databricks" - - "migrate to Lakeflow" - - "ADF migration" - - "convert ADF to Lakeflow" - - "migrate data factory" ---- - -# End-to-End ADF to Databricks Migration - -Orchestrate the complete migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. This skill runs all three phases in sequence: ingest, translate, prepare. - -## Context - -This is the top-level orchestration skill. It runs the full migration pipeline: - -1. **Ingest** — Parse ADF JSON exports into a typed inventory -2. **Translate** — Convert ADF activities to Databricks IR (deterministic + agentic) -3. **Prepare** — Generate Databricks Declarative Automation Bundles for deployment - -Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. - -## Prerequisite — Python environment - -This skill runs the plugin's Python code, which depends on third-party packages. Before running -any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the -**`setup`** skill, or directly: - -```bash -bash /scripts/bootstrap.sh -``` - -This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or -pip is missing, the script prints a warning telling the user what to install — relay it and stop -until they have installed Python 3.12+ and pip. - -Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` -(use it anywhere a command below shows `python3`): - -```bash -export PYTHONPATH="/src" -"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... -``` - -## Workflow - -Follow these steps in order: - -### Step 0 — Gather phase inputs via the adapter - -Before invoking ingest, run the adapter inputs subcommand once per -phase so the agent surfaces the matching free-text prompts: - -```bash -python3 -m flowx.adapter inputs ingest -python3 -m flowx.adapter inputs translate -python3 -m flowx.adapter inputs prepare -``` - -Each response carries the questions for that phase plus their -descriptions and defaults. Collect answers from the user (or accept -the defaults), persist them to `//inputs.json`, and -thread the values into the downstream CLI calls. - -### Step 1 — Gather inputs - -Ask the user for all required inputs upfront: - -| Parameter | Description | Required | Default | -|---|---|---|---| -| ADF source path | UC volume path or local directory with ADF JSON files | Yes | — | -| Output directory | Root directory for all flowx output | No | `./orchestra_output/` | -| Target catalog | Unity Catalog catalog for tables/volumes | No | `main` | -| Target schema | Schema within the catalog | No | `default` | -| Bundle name | Name for the generated DABs project | No | derived from pipelines | - -Example prompt: - -> To migrate your ADF pipelines, I need: -> 1. Where are your ADF JSON exports? (UC volume path like `/Volumes/main/default/adf_export` or local directory) -> 2. Where should I write the output? (default: `./orchestra_output/`) -> 3. What target catalog and schema? (default: `main.default`) - -### Step 2 — Phase 1: Ingest - -Invoke the `flowx:ingest` skill with the ADF source path and output directory set to `/ingest/`. - -Wait for the ingest to complete and present the inventory summary: - -``` -Phase 1: Ingest — Complete -========================== -Pipelines parsed: 12 -Total activities: 47 - Deterministic: 35 (74.5%) - Agentic: 10 (21.3%) - Unsupported: 2 ( 4.3%) -Coverage: 95.7% -``` - -### Step 3 — Checkpoint: confirm proceed - -Ask the user to review the inventory and confirm before continuing: - -> The ingest phase found 47 activities across 12 pipelines. 95.7% have a translation path (74.5% deterministic, 21.3% agentic). 2 activities are unsupported and will need manual handling. -> -> Proceed to the translation phase? (yes/no) - -If the user says no, explain the options: -- Re-run ingest with a different source directory -- Review the `inventory.json` to understand unsupported activities -- Manually classify activities before proceeding - -If the user says yes, proceed to step 4. - -### Step 4 — Phase 2: Translate - -Invoke the `flowx:translate` skill with: -- Inventory path: `/ingest/inventory.json` -- ADF source dir: the original ADF source path -- Output dir: `/translate/` - -Wait for the translation to complete and present the summary: - -``` -Phase 2: Translate — Complete -============================= -Deterministic translated: 35 (74.5%) -Agentic translated: 8 (17.0%) -Failed: 4 ( 8.5%) -Overall coverage: 91.5% -``` - -### Step 5 — Present translation details - -Show the user: -1. What was translated deterministically (bulk — just counts by type) -2. What was translated via agentic skills (list each with the skill used) -3. What failed and why (list each with the failure reason) - -For failures, suggest: -- Manual notebook creation -- Retry with additional context -- Skip and add placeholder - -### Step 5.1 — Gather just-in-time translation preferences - -Drive the loop multi-pass: re-run `inspect --answers ` -after each batch of answers so the adapter can surface chained -metadata-driven prompts. When the user opts to consolidate a -metadata-driven motif and the agent has a database tool, run the -lookup query directly and persist the rows to -`/translate/lookup_values.json`; otherwise prompt the user -for a CSV file or comma-separated string and run: - -```bash -python3 -m flowx.adapter materialize-lookup "" \ - --out /translate/lookup_values.json -``` - -Pass `--lookup-values` to the modify call when the file exists. - -#### Legacy flow details - -Before bundle generation, run the adapter inspect CLI on the translation -report to surface any pipeline-modifier questions the IR raises: - -```bash -python3 -m flowx.adapter inspect /translate/translation_report.json -``` - -For each question in the JSON output, prompt the user with the rationale, -options, and the affected task keys. Collect answers into -`/translate/answers.json` keyed by `question_id`, then apply -them to a stamped report: - -```bash -python3 -m flowx.adapter modify \ - /translate/translation_report.json \ - /translate/answers.json \ - --out /translate/translation_report.stamped.json -``` - -Use the stamped report (when produced) as the input to the prepare phase. -When inspect emits no questions for any pipeline, skip modify and use the -original report. - -The questions the adapter raises: - -| `question_id` | Allowed values | Default | -|---|---|---| -| `copy_activity_paradigm` | `notebook`, `sdp` | `notebook` | -| `non_databricks_task_compute` | `serverless`, `classic` | `serverless` | -| `use_lakeflow_connectors` | `existing`, `lakeflow_connect` | `existing` | -| `consolidate_motif:` | `keep`, `consolidate` | `keep` | - -DatabricksNotebook and DatabricksSparkPython tasks always inherit the cluster binding derived from -their source linked service. - -For each multi-activity motif the detector matches (rest_api_pagination, -incremental_load_watermark, metadata_driven_bulk_copy, ...) the adapter emits one -`consolidate_motif:` question. The user must explicitly opt in to `consolidate` -for each detected pattern. - -### Step 6 — Checkpoint: confirm proceed to bundle generation - -> Translation is 91.5% complete. 4 activities could not be translated automatically. -> Options: -> 1. Proceed to bundle generation (failed activities will get placeholder tasks) -> 2. Retry failed translations with more context -> 3. Stop here and review the translation report -> -> What would you like to do? - -### Step 6.5 — Detect workspace artifacts and authenticate - -Before invoking the prepare phase, run the adapter's -`workspace-paths` subcommand to detect any absolute workspace paths -the bundle would need to vendor: - -```bash -python3 -m flowx.adapter workspace-paths \ - /translate/translation_report.stamped.json \ - --source-dir -``` - -When the response carries `needs_auth: true`: - -1. Confirm the workspace host with the user, defaulting to the first - entry in `suggested_hosts` (extracted from the Databricks linked - services in the ADF export). -2. Run `databricks auth login --host ` interactively to set up - a local profile. -3. Pass `--profile ` to the prepare invocation in Step 7 so - flowx downloads the referenced notebooks and vendors them under - `bundle/src/notebooks/` with the task references rewritten to the - relative `../src/notebooks/...` paths. - -Skip this step entirely when `needs_auth` is `false`. - -### Step 7 — Phase 3: Prepare - -Invoke the `flowx:prepare` skill with: -- Translation report: `/translate/translation_report.stamped.json` if step 5.5 produced one, otherwise `/translate/translation_report.json` -- Output dir: `/dab_output/` -- Catalog: user-specified or `main` -- Schema: user-specified or `default` - -### Step 8 — Present final summary - -Display the complete migration summary: - -``` -Migration Complete -================== - -Source: /Volumes/main/default/adf_export (12 ADF pipelines) -Output: ./orchestra_output/dab_output/ - -Coverage: - Total activities: 47 - Successfully translated: 43 (91.5%) - Placeholder tasks: 4 ( 8.5%) - -Generated Files: - dab_output/ - databricks.yml - resources/ (3 job definitions) - src/notebooks/ (12 notebooks) - setup/ (3 setup scripts) - tests/ (3 test files) - -Setup Required: - - Run setup/create_volumes.py to create UC volumes - - Run setup/create_secrets.py to configure secrets (review credentials first) - - Run setup/register_connections.py to register external connections - -Next Steps: - 1. cd ./orchestra_output/dab_output/ - 2. Review generated files, especially notebooks and setup scripts - 3. databricks bundle validate --target dev - 4. Run setup scripts on the target workspace - 5. databricks bundle deploy --target dev - 6. databricks bundle run --target dev - 7. Verify job output and promote to staging/prod -``` - -### Step 9 — Offer follow-up actions - -Ask if the user wants to: -1. Validate the bundle now (`databricks bundle validate`) -2. Deploy to dev (`databricks bundle deploy --target dev`) -3. Review specific generated files -4. Re-translate any failed activities -5. Export a migration report for documentation - -## Reference - -See `references/workflow.md` for a detailed description of the three-phase architecture. - -## Examples - -- "Migrate my ADF pipelines to Databricks" -- "Convert ADF to Lakeflow jobs" -- "ADF to Databricks migration from /Volumes/main/default/adf_export" -- "Migrate data factory pipelines to catalog analytics, schema bronze" -- "Run the full ADF migration workflow" - -## Output Artifacts - -All artifacts from all three phases are produced under the output directory: - -| Directory | Phase | Contents | -|---|---|---| -| `ingest/` | Ingest | `inventory.json`, `ast/`, `parse_errors.json` | -| `translate/` | Translate | `translation_report.json`, `ir/`, `notebooks/`, `agentic_results/` | -| `dab_output/` | Prepare | `databricks.yml`, `resources/`, `src/`, `setup/`, `tests/` | diff --git a/skills/prepare/SKILL.md b/skills/prepare/SKILL.md deleted file mode 100644 index b6eed61..0000000 --- a/skills/prepare/SKILL.md +++ /dev/null @@ -1,254 +0,0 @@ ---- -name: prepare -description: > - Generate Databricks Declarative Automation Bundles (DABs) from translated IR, - including job definitions, notebooks, and setup scripts. -triggers: - - "prepare bundles" - - "generate DABs" - - "create bundles" - - "prepare deployment" - - "generate bundles" - - "build DABs" ---- - -# Prepare Databricks Declarative Automation Bundles - -Generate deployment-ready Databricks Declarative Automation Bundles (DABs) from the translated intermediate representation, including job definitions, notebooks, and infrastructure setup scripts. - -## Context - -This is phase 3 of the flowx migration workflow. It consumes the `translation_report.json` produced by the `translate` skill and generates a complete DABs project that can be validated and deployed with the Databricks CLI. - -The output is a standard DABs project with: -- `databricks.yml` — the bundle configuration -- `resources/` — job and pipeline YAML definitions -- `src/notebooks/` — generated and helper notebooks -- `setup/` — infrastructure setup scripts (volumes, secrets, connections) - -## Prerequisite — Python environment - -This skill runs the plugin's Python code, which depends on third-party packages. Before running -any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the -**`setup`** skill, or directly: - -```bash -bash /scripts/bootstrap.sh -``` - -This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or -pip is missing, the script prints a warning telling the user what to install — relay it and stop -until they have installed Python 3.12+ and pip. - -Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` -(use it anywhere a command below shows `python3`): - -```bash -export PYTHONPATH="/src" -"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... -``` - -## Workflow - -Follow these steps in order: - -### Step 1 — Locate the translation report - -Read `translation_report.json` from the translate phase. If the path is not in conversation context, ask the user: - -> Where is the translation_report.json from the translate phase? (default: `./orchestra_output/translate/translation_report.json`) - -Validate the file exists and all required translations have status `translated`. - -### Step 2 — Gather deployment parameters - -Ask the user for the following (provide defaults): - -| Parameter | Description | Default | -|---|---|---| -| Target catalog | Unity Catalog catalog for tables/volumes | `main` | -| Target schema | Schema within the catalog | `default` | -| Output directory | Where to write the DABs project | `./dab_output/` | -| Bundle name | Name for the DABs project | derived from first pipeline name | -| Target environments | Deployment targets to configure | `dev, staging, prod` | -| Warehouse ID | SQL warehouse for SQL tasks (optional) | prompt if SQL tasks exist | -| Databricks CLI profile | Profile used to download workspace-resident notebooks / JARs / Python files (`--profile`). Required only when the bundle references absolute workspace paths. | resolved from `~/.databrickscfg` (auto-prompt if multiple) | - -### Step 2.5 — Detect workspace artifacts and authenticate - -Before running the bundle writer, check whether the report references -absolute workspace paths (notebooks under `/Shared/`, SparkPython -files, SparkJar libraries) or DBFS paths that the bundle should -download to be self-contained: - -```bash -python3 -m flowx.adapter workspace-paths \ - \ - --source-dir -``` - -The command emits: - -```json -{ - "paths": ["/Shared/team/notebook_a", "/Shared/team/notebook_b"], - "suggested_hosts": ["https://adb-1234.5.azuredatabricks.net"], - "needs_auth": true -} -``` - -When `needs_auth` is `true`: - -1. Surface the suggested hosts to the user with `AskUserQuestion`. Use - the first `suggested_hosts` value as the default; allow the user to - override. When no host is suggested (no Databricks linked service - in the export), prompt for the host with no default. -2. Run the interactive Databricks CLI login command and wait for it to - complete: - - ```bash - databricks auth login --host - ``` - - This writes a profile into `~/.databrickscfg`. When the user has - chosen a specific profile name, append `--profile ` to both - the login and the prepare invocation below. - -3. Pass the resolved profile to step 3 via `--profile ` (default - profile name is `DEFAULT`). When `needs_auth` is `false` skip steps - 1–2 and omit `--profile` from step 3. - -The `paths` list is informational; you can echo it to the user so they -know which notebooks the bundle will vendor. - -### Step 3 — Run bundle generation - -Execute the DAB writer: - -```bash -python3 /src/flowx/bundler/dab_writer.py \ - --report \ - --output-dir \ - --catalog \ - --schema \ - --bundle-name \ - [--profile ] \ - [--no-vendor-workspace-files] -``` - -Where: -- `` is the root of the flowx plugin -- `` is the path to `translation_report.json` -- Other parameters are from step 2 - -**Workspace artifact vendoring (default: enabled).** When the report references workspace-resident notebooks (`/Shared/...`), DBFS Spark JARs (`dbfs:/...`), or Spark Python files, the preparer downloads them via the Databricks CLI auth so the resulting bundle is self-contained and deployable across environments. Downloaded notebooks are vendored under `src/notebooks/` and bound to the default `job_cluster` (since they may rely on classic-compute features). The original `notebook_path` in the resource YAML is rewritten to the bundle-relative path `../src/notebooks/.py`. - -If no Databricks CLI auth is detected on the host (`~/.databrickscfg` empty AND no `DATABRICKS_CONFIG_PROFILE` / `DATABRICKS_HOST`+`DATABRICKS_TOKEN` env vars), the CLI prints the workspace paths it was about to download and prompts: - -``` -Workspace downloads are enabled but no Databricks CLI auth was found. - Looked for profiles in: /Users//.databrickscfg - Artifacts to vendor: /Shared/ETL/transform, … - -To authenticate, run one of: - databricks auth login --host https://.cloud.databricks.com - databricks configure --token - -Continue with placeholders (downloads will be skipped)? [y/N]: -``` - -Answering `n` aborts with exit code 2 so the user can authenticate and re-run. Answering `y` continues with placeholder notebooks (legacy in-place workspace paths). In non-interactive sessions the prompt defaults to placeholders. - -Use `--no-vendor-workspace-files` to opt out entirely; the bundle then keeps original workspace paths exactly as in the IR. - -### Step 4 — Present the generated file tree - -Show the user what was generated: - -``` -dab_output/ - databricks.yml - resources/ - etl_main_job.yml - etl_secondary_job.yml - transform_dlt_pipeline.yml - src/ - notebooks/ - copy_from_blob.py - lookup_config.py - web_activity_call.py - set_variable_helper.py - setup/ - create_volumes.py - create_secrets.py - register_connections.py - tests/ - test_etl_main.py -``` - -### Step 5 — Explain setup tasks - -If the `setup/` directory was generated, explain what each script does: - -**create_volumes.py** — Creates Unity Catalog volumes required by the migrated jobs. These volumes replace Azure Blob Storage or ADLS references from ADF. Run this once per environment. - -**create_secrets.py** — Creates Databricks secret scopes and secrets for connection credentials that were in ADF linked services. Review the secret values and populate them manually or via your secrets management system. - -**register_connections.py** — Registers Unity Catalog connections for external data sources (SQL Server, REST APIs, etc.) that were referenced in ADF linked services. - -Emphasize that the user should review these scripts before running them, especially `create_secrets.py` which will need actual credential values. - -### Step 6 — Explain the generated bundle structure - -Briefly describe: -- **databricks.yml** — The root bundle config with workspace, target environments (dev/staging/prod), and variable definitions. Variables are parameterized for environment-specific values (catalog, schema, warehouse). -- **resources/*.yml** — One YAML file per Databricks Lakeflow Job (one per ADF pipeline). Each job contains tasks mapped from ADF activities, with dependencies matching the original ADF dependency chains. -- **src/notebooks/*.py** — Python notebooks for activities that translate to notebook_task. These contain the actual data movement or transformation logic. -- **tests/*.py** — Skeleton test files for validating the migrated jobs. - -### Step 7 — Suggest next steps - -Present the following next steps: - -``` -Next Steps -========== -1. Review generated files: - cd - cat databricks.yml - -2. Validate the bundle: - databricks bundle validate --target dev - -3. Run setup scripts (if generated): - databricks bundle run setup_volumes --target dev - -4. Deploy to dev: - databricks bundle deploy --target dev - -5. Test the deployed jobs: - databricks bundle run --target dev - -6. Promote to staging/prod: - databricks bundle deploy --target staging - databricks bundle deploy --target prod -``` - -Recommend running `databricks bundle validate` first to catch any configuration issues before deployment. - -## Examples - -- "Prepare the bundles" -- "Generate DABs for the translated pipelines" -- "Create deployment bundles targeting catalog 'analytics' and schema 'bronze'" -- "Build the DABs project in ./output/my_migration/" - -## Output Artifacts - -| File | Description | -|---|---| -| `databricks.yml` | Root bundle configuration | -| `resources/*.yml` | Job and pipeline YAML definitions | -| `src/notebooks/*.py` | Generated notebooks | -| `setup/*.py` | Infrastructure setup scripts | -| `tests/*.py` | Skeleton test files | diff --git a/skills/setup/SKILL.md b/skills/setup/SKILL.md deleted file mode 100644 index 3d262c8..0000000 --- a/skills/setup/SKILL.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -name: setup -description: > - Setup the Python environment for the flowx plugin. Creates a .venv virtual environment - and installs the Python dependencies (from requirements.txt via pip) needed for each phase. - Run this once before any other flowx skill, or whenever dependencies are missing. -triggers: - - "setup flowx" - - "bootstrap flowx" - - "install flowx dependencies" - - "flowx environment" - - "create flowx venv" - - "ModuleNotFoundError flowx" ---- - -# Create the Python Environment - -Create a virtual environment (`.venv`) for the plugin and install its Python dependencies. This -is the prerequisite for the `ingest`, `translate`, `prepare`, and `migrate` skills which run Python -from this environment. - -## Context - -The flowx plugin ships Python code (in `src/flowx/`) that the skills invoke (e.g. -`python -m flowx.adapter ...`, `adf_loader.py`, `engine.py`, `dab_writer.py`). Some code depends -on third-party packages (`pyyaml`, `databricks-sdk`, `sqlglot`). Running it against a bare system -Python fails with `ModuleNotFoundError`. This step provisions an isolated `.venv` with the required -dependencies installed via `pip` from `requirements.txt`. - -The environment is created once and reused. Re-running the bootstrapscript confirms the venv exists -and ensures that dependencies are satisfied. - -## Workflow - -### Step 1 — Run the bootstrap script - -From the plugin root, run: - -```bash -bash /scripts/bootstrap.sh -``` - -Where `` is the root of the flowx plugin (the directory containing `src/`, -`skills/`, and `requirements.txt`). - -The script will: -1. Check that `python3`, `pip`, and the `venv` module are available. -2. Create `/.venv` if it does not already exist. -3. Install dependencies listed in `requirements.txt` into that venv using `pip`. - -### Step 2 — Handle a missing Python or pip - -If Python, pip, or the `venv` module are **not** available, the script prints a `WARNING:` block -explaining what to install and exits non-zero **without** creating anything. - -When this happens, **do not attempt to work around it**. Relay the warning to the user, ask them -to install, and stop: - -> ⚠️ Python must be installed before I can set up the flowx environment. -> -> * On macOS: `brew install python`. -> * On Debian/Ubuntu: `sudo apt-get install python3 python3-venv python3-pip`. -> -> Let me know once it's installed and I'll re-run setup. - -Re-run this setup skill after the user confirms Python and pip are installed. - -### Step 3 — Confirm success and how to run Python code - -On success, the script prints the interpreter path and a usage example. After this, every -Python command in the flowx skills **must** be run with the venv interpreter and `src/` -on `PYTHONPATH`: - -```bash -export PYTHONPATH="/src" -"/.venv/bin/python" -m flowx.adapter inputs ingest -``` - -(On Windows the interpreter is `\.venv\Scripts\python.exe`.) - -Use `/.venv/bin/python` anywhere the other skills show `python3`. - -## Output - -| Artifact | Description | -|---|---| -| `/.venv/` | Virtual environment containing the installed dependencies | -| `requirements.txt` | The dependency list installed into the venv | - -## Examples - -- "Set up the flowx environment" -- "Bootstrap flowx so I can run a migration" -- "I got a ModuleNotFoundError running ingest — fix the environment" diff --git a/skills/translate/SKILL.md b/skills/translate/SKILL.md deleted file mode 100644 index 0ce95b9..0000000 --- a/skills/translate/SKILL.md +++ /dev/null @@ -1,329 +0,0 @@ ---- -name: translate -description: > - Translate parsed ADF pipeline AST into Databricks IR (intermediate representation). - Runs deterministic translators for known activity types, then invokes agentic skills - from adf-to-databricks-plugin for gaps. -triggers: - - "translate ADF" - - "convert ADF" - - "translate pipelines" - - "convert pipelines" - - "run translation" ---- - -# Translate ADF to Databricks IR - -Convert the parsed ADF inventory into Databricks intermediate representation (IR) using deterministic translators for known types and agentic fallback for unknown types. - -## Context - -This is phase 2 of the flowx migration workflow. It consumes the `inventory.json` produced by the `ingest` skill and produces a `translation_report.json` that the `prepare` skill uses to generate Databricks Declarative Automation Bundles. - -The translation follows a **deterministic-first** strategy: -1. Activities with known, well-defined mappings are translated by built-in Python translators -2. Activities that require interpretation, expression conversion, or lack a Python translator are handled by agent skills from the `adf-to-databricks-plugin` - -## Prerequisite — Python environment - -This skill runs the plugin's Python code, which depends on third-party packages. Before running -any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the -**`setup`** skill, or directly: - -```bash -bash /scripts/bootstrap.sh -``` - -This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or -pip is missing, the script prints a warning telling the user what to install — relay it and stop -until they have installed Python 3.12+ and pip. - -Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` -(use it anywhere a command below shows `python3`): - -```bash -export PYTHONPATH="/src" -"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... -``` - -## Workflow - -Follow these steps in order: - -### Step 0 — Gather phase inputs - -Run the adapter inputs subcommand so the agent surfaces the free-text -questions the phase needs (inventory path, ADF source dir, output -directory): - -```bash -python3 -m flowx.adapter inputs translate -``` - -The JSON response carries the prompts and defaults; collect answers -from the user (or fall back to the defaults) and persist them to -`/translate/inputs.json` so later steps and subsequent -phases can read the same values. - -### Step 1 — Locate the inventory - -Read `inventory.json` from the ingest phase. If the path is not already in conversation context, ask the user: - -> Where is the inventory.json from the ingest phase? (default: `./orchestra_output/ingest/inventory.json`) - -Validate the file exists and is well-formed. - -### Step 2 — Run deterministic translation - -Execute the translation engine on all deterministic activities: - -```bash -python3 /src/flowx/translator/engine.py \ - --inventory \ - --source-dir \ - --output-dir -``` - -Where: -- `` is the root of the flowx plugin -- `` is the path to `inventory.json` -- `` is the original ADF JSON directory (from the ingest phase) -- `` is the translation output path (default: `./orchestra_output/translate/`) - -This produces: -- `translation_report.json` — results for deterministic activities + placeholders for agentic gaps -- `ir/` directory — Databricks IR for each translated activity -- `notebooks/` directory — generated helper notebooks - -### Step 3 — Read the translation report - -Read `translation_report.json`. It has this structure: - -```json -{ - "inventory_path": "/path/to/inventory.json", - "generated_at": "2026-04-07T12:30:00Z", - "translations": [ - { - "pipeline": "ETL_Main", - "activity": "CopyFromBlob", - "type": "Copy", - "strategy": "deterministic", - "status": "translated", - "ir": { - "task_key": "copy_from_blob", - "task_type": "notebook_task", - "notebook_path": "notebooks/copy_from_blob.py", - "parameters": { "source": "abfss://...", "target": "..." } - } - }, - { - "pipeline": "ETL_Main", - "activity": "TransformData", - "type": "ExecuteDataFlow", - "strategy": "agentic", - "status": "pending", - "raw_activity_json": { "...": "..." }, - "target_skill": "adf-to-databricks:adf-dataflow-converter" - } - ], - "summary": { - "total": 47, - "deterministic_translated": 35, - "agentic_pending": 10, - "failed": 2 - } -} -``` - -### Step 4 — Handle agentic gaps - -For each translation with `"status": "pending"` and `"strategy": "agentic"`, invoke the appropriate skill from the `adf-to-databricks-plugin`. Route by activity type: - -**ExecuteDataFlow activities:** -Invoke `adf-to-databricks:adf-dataflow-converter` with the raw activity JSON and associated data flow definition. Provide context: -- The raw `typeProperties` from the ADF activity -- The data flow JSON definition (if available in the source directory under `dataflow/`) -- The linked service configurations for source/sink connections -- Target catalog and schema for the SDP pipeline or PySpark notebook output - -**Control flow activities (Switch, Until, Wait, Filter, AppendVariable):** -Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: -- The full pipeline JSON containing the activity -- Any nested activities within the control flow -- Variable definitions from the pipeline -- The desired Databricks task type mapping - -**Stored procedures and external calls (SqlServerStoredProcedure, AzureFunction, WebHook, Custom):** -Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: -- The linked service configuration for the target system -- Connection details and authentication method -- Any parameters or request bodies - -**Complex expressions:** -If any activity (deterministic or agentic) contains ADF expressions that the deterministic translator could not resolve, invoke `adf-to-databricks:adf-expression-translator` with: -- The raw expression string (e.g., `@pipeline().parameters.inputPath`) -- The expression context (pipeline parameters, variables, activity outputs) -- The target format (Python f-string, Spark SQL, task parameter reference) - -**Trigger definitions:** -Invoke `adf-to-databricks:adf-trigger-converter` with: -- The trigger JSON definition -- The associated pipeline references -- Target: Databricks job schedule configuration (quartz_cron_expression, periodic, or file_arrival) - -### Step 5 — Collect agentic results - -Each agentic skill invocation produces a translation result. Collect all results into `/agentic_results/`: -- Save each result as `__.json` -- Include the generated IR, any notebooks, and metadata - -### Step 6 — Merge agentic results - -Run the merge step to combine deterministic and agentic translations: - -```bash -python3 /src/flowx/translator/engine.py \ - --merge-agentic \ - --report \ - --agentic-results -``` - -This updates `translation_report.json` with the agentic results merged in, changing their status from `pending` to `translated` (or `failed` if the agentic skill could not produce a result). - -### Step 6.1 — Gather just-in-time translation preferences - -The adapter raises several preference questions plus a chained set for -metadata-driven motifs. Every time the user answers a question whose value -gates further prompts, re-run `inspect --answers ` to surface -the next batch. - -When the metadata-driven flow ends with `metadata_driven_lookup_tool=have` -and the agent has a database tool (Genie, MCP SQL, or a workspace SDK), -run the lookup query directly and write the rows to -`/lookup_values.json`. When the answer is `none`, prompt -the user for a CSV file or comma-separated string and call: - -```bash -python3 -m flowx.adapter materialize-lookup "" \ - --out /lookup_values.json -``` - -Then call `modify` with the lookup values: - -```bash -python3 -m flowx.adapter modify \ - \ - /answers.json \ - --lookup-values /lookup_values.json \ - --out /translation_report.stamped.json -``` - -When no metadata-driven motif is consolidated, `--lookup-values` is omitted. - -#### Legacy flow details - -Before writing the final report, surface any pipeline-modifier questions the -IR raises (Copy Data paradigm, non-Databricks task compute, Lakeflow Connect -opt-in, Databricks task compute). Use the adapter CLI bridge: - -```bash -python3 -m flowx.adapter inspect -``` - -The command emits JSON: - -```json -{ - "pipelines": [ - { - "pipeline_name": "ETL_Main", - "questions": [ - { - "question_id": "copy_activity_paradigm", - "prompt": "How should Copy Data activities targeting Delta be implemented?", - "rationale": "...", - "options": [{"value": "notebook", "label": "...", "description": "..."}, ...], - "affected_task_keys": ["copy_orders", "copy_customers"], - "default": "notebook" - }, - ... - ] - } - ] -} -``` - -For each question, prompt the user with the rationale, options, and the -task keys it affects. Use the default when the user defers. Collect the -answers into a JSON file (`/answers.json`) shaped like: - -```json -{ - "copy_activity_paradigm": "sdp", - "non_databricks_task_compute": "serverless", - "use_lakeflow_connectors": "lakeflow_connect" -} -``` - -Then apply the answers to produce a stamped report the prepare phase consumes: - -```bash -python3 -m flowx.adapter modify \ - \ - /answers.json \ - --out /translation_report.stamped.json -``` - -The prepare phase (next skill) must be pointed at the stamped report. -When no questions are raised, the inspect output is `{"pipelines": [{"pipeline_name": "...", "questions": []}, ...]}` — skip the modify step and pass the original report straight through. - -### Step 7 — Present translation summary - -Display a summary to the user: - -``` -Translation Summary -=================== -Total activities: 47 -Deterministic translated: 35 (74.5%) -Agentic translated: 8 (17.0%) -Failed: 4 ( 8.5%) - -Overall coverage: 91.5% - -Failed translations: - - ETL_Main / RunSSIS (ExecuteSSISPackage) — no translator available - - ETL_Main / CustomTask (Custom) — agentic skill returned error - ... - -Generated artifacts: - - translation_report.json - - ir/ (43 files) - - notebooks/ (12 files) -``` - -If coverage is below 100%, explain the options for failed translations: -1. Manual notebook creation for unsupported types -2. Retry agentic translation with additional context -3. Skip the activity and add a placeholder task in the DAB - -## Reference - -See `references/activity-mapping.md` for the complete mapping between ADF activity types and translation strategies. - -## Examples - -- "Translate the ADF pipelines" -- "Convert ADF to Databricks" -- "Run the translation on the inventory from the ingest step" -- "Translate the parsed pipelines using deterministic + agentic" - -## Output Artifacts - -| File | Description | -|---|---| -| `translation_report.json` | Full translation report with IR for all activities | -| `ir/*.json` | Databricks IR for each translated activity | -| `notebooks/*.py` | Generated helper notebooks | -| `agentic_results/*.json` | Raw results from agentic skill invocations | diff --git a/src/AGENTS.md b/src/AGENTS.md index 2198208..e0cd18d 100644 --- a/src/AGENTS.md +++ b/src/AGENTS.md @@ -25,13 +25,15 @@ neighbouring module. This is a companion to the top-level [AGENTS.md](../AGENTS. - First line is **third-person present indicative** describing what the function does: ``"""Converts X to Y."""`` rather than ``"""Convert X..."""`` or ``"""This function converts..."""``. -- Keep docstrings **brief** -- usually one sentence. Add extra detail - only when behaviour is non-obvious (edge cases, surprising - invariants). Don't restate the type signature or repeat parameter - names; ``Args:`` / ``Returns:`` / ``Raises:`` blocks are optional and - should appear only when the type is genuinely ambiguous. -- Don't write multi-paragraph design rationale in docstrings; that - belongs in commit messages or pull-request descriptions. +- **Public** classes and functions use the **Google docstring format**: + a one-line summary, then ``Attributes:`` (classes), ``Args:``, + ``Returns:``, ``Raises:``, and ``Notes:`` sections as applicable. +- Private helpers keep a brief one-line summary; add ``Args:`` / + ``Returns:`` only when the types are genuinely ambiguous. +- Non-obvious design rationale (a constraint, a prior bug, an external + spec) goes in a ``Notes:`` section or a one-line comment that points to + the relevant ``AGENTS.md`` design-notes entry -- never as a + multi-paragraph comment block. ## Comments diff --git a/src/flowx/__init__.py b/src/flowx/__init__.py new file mode 100644 index 0000000..2723f34 --- /dev/null +++ b/src/flowx/__init__.py @@ -0,0 +1,3 @@ +"""flowx - ADF to Databricks translation plugin for Claude Code.""" + +__version__ = "0.1.0" diff --git a/src/flowx/adapter/__init__.py b/src/flowx/adapter/__init__.py new file mode 100644 index 0000000..780bdbd --- /dev/null +++ b/src/flowx/adapter/__init__.py @@ -0,0 +1,75 @@ +"""Agent-facing surfaces and the matching pipeline modifier for flowx translation. + +Two roles live here, kept deliberately separate: the **agent adapter** (:mod:`~flowx.adapter.session` +plus the option shapes in :mod:`~flowx.adapter.models`) converts tool-call arguments into +deterministic calls and maps "need more input" into structured objects, while the **pipeline modifier** +(:mod:`~flowx.adapter.operations`) consumes a validated :class:`TranslationConfiguration` and stamps +concrete decisions onto a Pipeline IR with no awareness of agents. ``constants`` holds shared strings and +``predicates`` holds pure IR predicates used by both ``operations`` and the bundler. +""" + +from __future__ import annotations + +from flowx.adapter.models import ( + DEFAULT_CONFIGURATION, + CopyActivityParadigm, + LakeflowConnectorType, + MetadataDrivenAccess, + MetadataDrivenConsolidate, + MetadataDrivenLookupTool, + MetadataDrivenSize, + MigrationInputOption, + MotifConsolidate, + NonDatabricksTaskCompute, + OptionChoice, + PendingMigrationInputs, + PendingOptions, + TranslationConfiguration, + TranslationOption, + UseLakeflowConnectors, +) +from flowx.adapter.operations import ( + allowed_values_for, + apply_configuration, + collect_workspace_artifact_paths, + detect_databricks_hosts, + enum_for, + gather_options, + validate_answer, +) +from flowx.adapter.session import ( + MigrationInputSession, + TranslationInputRequired, + TranslationSession, + UnknownMigrationPhaseError, +) + +__all__ = [ + "DEFAULT_CONFIGURATION", + "CopyActivityParadigm", + "LakeflowConnectorType", + "MetadataDrivenAccess", + "MetadataDrivenConsolidate", + "MetadataDrivenLookupTool", + "MetadataDrivenSize", + "MigrationInputOption", + "MigrationInputSession", + "MotifConsolidate", + "NonDatabricksTaskCompute", + "PendingMigrationInputs", + "PendingOptions", + "OptionChoice", + "TranslationInputRequired", + "TranslationConfiguration", + "TranslationOption", + "TranslationSession", + "UnknownMigrationPhaseError", + "UseLakeflowConnectors", + "allowed_values_for", + "apply_configuration", + "collect_workspace_artifact_paths", + "detect_databricks_hosts", + "enum_for", + "gather_options", + "validate_answer", +] diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py new file mode 100644 index 0000000..1d214aa --- /dev/null +++ b/src/flowx/adapter/__main__.py @@ -0,0 +1,792 @@ +"""Unified CLI entry point that the flowx skills and MCP tools drive via subprocesses. + +Exposes stateless subcommands -- the ``discover``/``convert``/``package`` phase runners plus +``inspect``, ``modify``, ``inputs``, ``materialize-lookup``, ``workspace-paths``, ``record-results``, +and ``install-dashboard`` -- so each agent turn runs as an independent process holding no session +state across user prompts. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from flowx.adapter.constants import MOTIF_CONSOLIDATE_OPTION_PREFIX +from flowx.adapter.models import ( + DEFAULT_CONFIGURATION, + CopyActivityParadigm, + LakeflowConnectorType, + MetadataDrivenAccess, + MetadataDrivenConsolidate, + MetadataDrivenLookupTool, + MetadataDrivenSize, + MotifConsolidate, + NonDatabricksTaskCompute, + NotifyDestination, + NotifyEvents, + TranslationConfiguration, + UseLakeflowConnectors, +) +from flowx.adapter.operations import ( + apply_configuration, + build_option_schema, + collect_notify_args, + collect_workspace_artifact_paths, + detect_databricks_hosts, + provision_notification_destinations, + validate_answer, +) + +# bundler.dab_writer + translator.engine (sqlglot) are imported lazily inside inspect/modify only, so +# the cheap commands (inputs, phase pass-throughs, materialize-lookup, workspace-paths) skip ~0.15s of +# unused import cost on every adapter subprocess. + +# Maps the unified phase runner subcommands to the module CLI they forward to. +_PHASE_MODULES: dict[str, str] = { + "discover": "flowx.parser.adf_loader", + "convert": "flowx.translator.engine", + "package": "flowx.bundler.dab_writer", +} +# Aliases so the inputs option ids double as CLI flags on the phase runners. +_PHASE_FLAG_ALIASES: dict[str, str] = { + "--adf-source-path": "--source-dir", +} + + +def main(argv: list[str] | None = None) -> int: + """Dispatches an ``inspect`` or ``modify`` subcommand. + + Args: + argv: CLI arguments to parse. Defaults to :data:`sys.argv` when + ``None``. + + Returns: + Exit code (0 on success, non-zero on usage or runtime errors). + """ + raw_args = list(sys.argv[1:]) if argv is None else list(argv) + if raw_args and raw_args[0] in _PHASE_MODULES: + # Phase runners are pure pass-through to the underlying phase CLI; + # bypass argparse so forwarded --flags aren't misparsed at this level. + return _run_phase(raw_args[0], raw_args[1:]) + + parser = _build_parser() + args = parser.parse_args(argv) + if args.command == "inspect": + return _run_inspect(args) + if args.command == "modify": + return _run_modify(args) + if args.command == "materialize-lookup": + return _run_materialize_lookup(args) + if args.command == "inputs": + return _run_inputs(args) + if args.command == "workspace-paths": + return _run_workspace_paths(args) + if args.command == "record-results": + return _run_record_results(args) + if args.command == "install-dashboard": + return _run_install_dashboard(args) + parser.print_help(sys.stderr) + return 2 + + +def _run_record_results(args: argparse.Namespace) -> int: + """Implements ``record-results``: write per-pipeline coverage to a UC table. + + Returns 0 on success, 1 when the metadata cannot be read or the write fails. + """ + from flowx.reporting.results import write_results + + metadata_dir = args.output_dir / "metadata" + if not (metadata_dir / "inventory.json").exists(): + print(f"No inventory.json under {metadata_dir}; run the discover phase first.", file=sys.stderr) + return 1 + try: + run_id, rows = write_results(metadata_dir, args.results_table, warehouse_id=args.warehouse_id) + except Exception as error: # noqa: BLE001 - surface an actionable message to the agent + print(f"Failed to record results to {args.results_table}: {error}", file=sys.stderr) + return 1 + if not rows: + print("No pipelines found to record.", file=sys.stderr) + return 1 + print(f"Recorded {rows} pipeline row(s) to {args.results_table} (run_id={run_id}).") + return 0 + + +def _run_install_dashboard(args: argparse.Namespace) -> int: + """Implements ``install-dashboard``: create + publish the coverage dashboard. + + Returns 0 on success, 1 when the dashboard could not be created. + """ + from flowx.reporting.dashboard import install_dashboard + + try: + dashboard_id, url = install_dashboard( + args.results_table, + warehouse_id=args.warehouse_id, + display_name=args.dashboard_name, + parent_path=args.parent_path, + ) + except Exception as error: # noqa: BLE001 - surface an actionable message to the agent + print(f"Failed to install dashboard for {args.results_table}: {error}", file=sys.stderr) + return 1 + print(f"Installed coverage dashboard (id={dashboard_id}).") + if url: + print(f" {url}") + return 0 + + +def _run_workspace_paths(args: argparse.Namespace) -> int: + """Implements the ``workspace-paths`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``report``, ``source_dir``, + and ``out``. + + Returns: + ``0`` on success. The command always succeeds when the report + can be read; missing or unreadable inputs simply produce empty + path / host lists so the skill can detect the no-op case. + """ + paths = collect_workspace_artifact_paths(args.report) + suggested_hosts = detect_databricks_hosts(args.source_dir) if args.source_dir else [] + payload = { + "paths": paths, + "suggested_hosts": suggested_hosts, + "needs_auth": bool(paths), + } + _emit_json(payload, args.out) + return 0 + + +def _run_inputs(args: argparse.Namespace) -> int: + """Implements the ``inputs`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``phase`` and ``out``. + + Returns: + ``0`` on success. The CLI never raises here because the phase + argument is constrained by argparse. + """ + from flowx.adapter.session import MigrationInputSession + + session = MigrationInputSession(phase=args.phase) + pending = session.pending() + payload = { + "phase": pending.phase, + "options": [ + { + "option_id": option.option_id, + "prompt": option.prompt, + "description": option.description, + "default": option.default, + "required": option.required, + } + for option in pending.options + ], + } + _emit_json(payload, args.out) + return 0 + + +def _build_parser() -> argparse.ArgumentParser: + """Builds the top-level argparse parser with the two subcommands. + + Returns: + Configured :class:`argparse.ArgumentParser`. + """ + parser = argparse.ArgumentParser( + prog="python -m flowx.adapter", + description="Inspect and modify a translated flowx pipeline IR.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + inspect = subparsers.add_parser( + "inspect", + help="Emit the full translation-option schema for a report as JSON.", + ) + inspect.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") + inspect.add_argument( + "--answer", + action="append", + default=[], + metavar="OPTION_ID=VALUE", + help=( + "Deprecated/no-op: the full option tree (every option with a `show_when` condition) is " + "always emitted now, so the agent walks the chain locally. Accepted for back-compat and " + "validated for format only." + ), + ) + inspect.add_argument( + "--out", + type=Path, + default=None, + help="Optional output file; defaults to stdout.", + ) + + modify = subparsers.add_parser( + "modify", + help="Apply collected answers to a translation report and write the stamped IR.", + ) + modify.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") + modify.add_argument( + "--answer", + action="append", + default=[], + metavar="OPTION_ID=VALUE", + help=( + "A collected answer as OPTION_ID=VALUE (e.g. --answer notify_destination=email). " + "Repeatable; pass one per option the user answered. Values may contain '=' (only the " + "first '=' splits the pair)." + ), + ) + modify.add_argument( + "--output-dir", + type=Path, + default=None, + help=( + "Migration output directory. The stamped IR is written to its transient " + ".work/translation_report.stamped.json and the collected answers are written to " + "metadata/configuration.json. Either --output-dir or --out is required." + ), + ) + modify.add_argument( + "--out", + type=Path, + default=None, + help=("Explicit destination for the configuration-stamped IR JSON (overrides the --output-dir convention)."), + ) + modify.add_argument( + "--config-out", + type=Path, + default=None, + help=( + "Explicit destination for configuration.json (the collected answers). Defaults to " + "/metadata/configuration.json." + ), + ) + modify.add_argument( + "--lookup-csv", + type=str, + default=None, + help=( + "Optional CSV file path or literal CSV string of lookup-value rows that consolidated " + "metadata-driven motifs should ingest. The header row names the columns; each " + "subsequent row becomes one dict." + ), + ) + + workspace_paths = subparsers.add_parser( + "workspace-paths", + help=( + "Detect absolute workspace paths in a stamped report and suggest " + "Databricks workspace hosts from the ADF linked services." + ), + ) + workspace_paths.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") + workspace_paths.add_argument( + "--source-dir", + type=Path, + default=None, + help=( + "Optional path to the ADF JSON export directory. When supplied, " + "the command reads ``linked_services/*.json`` to suggest the " + "workspace host that ``databricks auth login --host`` should use." + ), + ) + workspace_paths.add_argument( + "--out", + type=Path, + default=None, + help="Optional output file; defaults to stdout.", + ) + + inputs = subparsers.add_parser( + "inputs", + help="Emit the migration-phase input options for an flowx phase as JSON.", + ) + inputs.add_argument( + "phase", + choices=("discover", "convert", "package"), + help="Migration phase whose input prompts the agent should surface.", + ) + inputs.add_argument( + "--out", + type=Path, + default=None, + help="Optional output file; defaults to stdout.", + ) + + materialize = subparsers.add_parser( + "materialize-lookup", + help="Parse CSV-shaped lookup values into the JSON shape modify consumes.", + ) + materialize.add_argument( + "source", + help=( + "Either a path to a CSV file or a literal CSV string. The first row " + "is treated as headers and every subsequent row is emitted as one dict." + ), + ) + materialize.add_argument( + "--out", + type=Path, + required=True, + help="Destination path for the lookup-values JSON list.", + ) + + record = subparsers.add_parser( + "record-results", + help="Write per-pipeline migration coverage for this run to a Unity Catalog table.", + ) + record.add_argument( + "--output-dir", + type=Path, + required=True, + help="Migration output directory (reads metadata/inventory.json + metadata/profile_report.csv).", + ) + record.add_argument( + "--results-table", + type=str, + required=True, + help="Target UC table as catalog.schema.table.", + ) + record.add_argument( + "--warehouse-id", + type=str, + default=None, + help="SQL warehouse id for the write. Auto-detected (prefers running serverless) when omitted.", + ) + + dashboard = subparsers.add_parser( + "install-dashboard", + help="Create and publish an AI/BI dashboard visualizing coverage from the results table.", + ) + dashboard.add_argument( + "--results-table", + type=str, + required=True, + help="UC table the dashboard reads (catalog.schema.table).", + ) + dashboard.add_argument( + "--warehouse-id", + type=str, + default=None, + help="SQL warehouse backing the dashboard. Auto-detected when omitted.", + ) + dashboard.add_argument( + "--dashboard-name", + type=str, + default=None, + help="Dashboard display name (defaults to 'Migration Coverage \u2014 ').", + ) + dashboard.add_argument( + "--parent-path", + type=str, + default=None, + help="Workspace folder for the dashboard (defaults to the current user's home).", + ) + + # Unified phase runners: `adapter -- ` forwards to the phase CLI (one entry point); + # --adf-source-path is accepted as an alias of the loader/translator --source-dir flag. + for _phase in ("discover", "convert", "package"): + _runner = subparsers.add_parser( + _phase, + help=f"Run the {_phase} phase (forwards flags to the underlying phase CLI).", + ) + _runner.add_argument( + "forward", + nargs=argparse.REMAINDER, + help="Flags forwarded to the phase CLI (e.g. --adf-source-path/--source-dir, --output-dir, --pipeline).", + ) + + return parser + + +def _run_phase(phase: str, forward: list[str]) -> int: + """Forward a phase runner subcommand to the underlying phase module, **in-process**. + + ``python -m flowx.adapter discover --adf-source-path X --output-dir Y`` runs + ``flowx.parser.adf_loader.main(["--source-dir", "X", "--output-dir", "Y"])`` in this same + interpreter -- no second ``python -m`` spawn. The module's ``main(argv)`` reuses the existing, + tested phase CLI surface, so there is a single entry point with no argument-surface duplication. + Collapsing the former double-spawn (adapter process -> module process) shaves an interpreter + start + re-import off every ``discover``/``convert``/``package`` call. + + Args: + phase: One of ``"discover"`` / ``"convert"`` / ``"package"``. + forward: Tokens after the phase name (flags for the phase CLI). + + Returns: + The phase's exit code (0 on success). + """ + import importlib + + module = importlib.import_module(_PHASE_MODULES[phase]) + mapped = [_PHASE_FLAG_ALIASES.get(token, token) for token in (forward or [])] + try: + return module.main(mapped) or 0 + except SystemExit as exit_signal: # e.g. argparse usage error -> parser.error() raises SystemExit + code = exit_signal.code + if isinstance(code, int): + return code + return 0 if code is None else 1 + + +def _run_inspect(args: argparse.Namespace) -> int: + """Implements the ``inspect`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``report``, ``answers``, and + ``out``. + + Returns: + ``0`` when the report was inspected successfully, ``1`` when the + report could not be loaded. + """ + pipelines = _load_pipelines(args.report) + if pipelines is None: + return 1 + try: + # The agent now walks the full option tree locally, so --answer no longer filters the output; + # we still validate its format so a malformed pair is reported rather than silently ignored. + _parse_answer_args(getattr(args, "answer", []) or []) + except ValueError as error: + print(f"Invalid --answer: {error}", file=sys.stderr) + return 2 + payload = { + "pipelines": [ + {"pipeline_name": pipeline.name, "options": build_option_schema(pipeline)} for pipeline in pipelines + ], + } + _emit_json(payload, args.out) + return 0 + + +def _parse_answer_args(pairs: list[str]) -> dict[str, str]: + """Parses repeatable ``--answer OPTION_ID=VALUE`` CLI args into a mapping. + + Only the first ``=`` splits each pair, so values may themselves contain + ``=`` (e.g. a query string or base64 token). + + Args: + pairs: Raw ``OPTION_ID=VALUE`` strings from argparse. + + Returns: + Mapping of option_id to answer string (later values win on duplicates). + + Raises: + ValueError: When a token has no ``=`` or an empty option id. + """ + answers: dict[str, str] = {} + for pair in pairs: + if "=" not in pair: + raise ValueError(f"expected OPTION_ID=VALUE, got {pair!r}") + key, value = pair.split("=", 1) + key = key.strip() + if not key: + raise ValueError(f"empty option id in {pair!r}") + answers[key] = value + return answers + + +def _resolve_modify_outputs(args: argparse.Namespace) -> tuple[Path, Path] | None: + """Resolves the (stamped_report_path, configuration_json_path) for ``modify``. + + Honors explicit ``--out`` / ``--config-out`` overrides, otherwise derives both + from ``--output-dir`` (stamped -> ``.work/``, configuration.json -> ``metadata/``). + Returns ``None`` when neither ``--output-dir`` nor ``--out`` was supplied. + """ + output_dir: Path | None = args.output_dir + stamped = args.out + if stamped is None: + if output_dir is None: + return None + stamped = output_dir / ".work" / "translation_report.stamped.json" + config_out = args.config_out + if config_out is None: + if output_dir is not None: + config_out = output_dir / "metadata" / "configuration.json" + elif stamped.parent.name == ".work": + config_out = stamped.parent.parent / "metadata" / "configuration.json" + else: + config_out = stamped.parent / "configuration.json" + return stamped, config_out + + +def _run_modify(args: argparse.Namespace) -> int: + """Implements the ``modify`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``report``, ``answers``, + ``out``, and the optional ``lookup_values``. + + Returns: + ``0`` when the modified IR was written successfully, ``1`` when + the report could not be loaded, ``2`` when the answers failed + validation. + """ + outputs = _resolve_modify_outputs(args) + if outputs is None: + print("modify requires --output-dir (or an explicit --out)", file=sys.stderr) + return 2 + stamped_out, config_out = outputs + + pipelines = _load_pipelines(args.report) + if pipelines is None: + return 1 + try: + answers = _parse_answer_args(args.answer or []) + configuration = _configuration_from_answers(answers) + except ValueError as error: + print(f"Invalid answers: {error}", file=sys.stderr) + return 2 + try: + lookup_values = _parse_csv_source(args.lookup_csv) if args.lookup_csv else [] + except ValueError as error: + print(f"Invalid --lookup-csv: {error}", file=sys.stderr) + return 2 + + stamped_pipelines = [ + _stamp_lookup_values_into_metadata_driven_motifs(apply_configuration(pipeline, configuration), lookup_values) + for pipeline in pipelines + ] + # Prompt-time provisioning: create/reuse the Databricks notification destination for any non-email + # activity_and_notify spec now, so its resolved id is baked into the report (email needs none). + provisioned_pipelines = [] + for pipeline in stamped_pipelines: + provisioned, messages = provision_notification_destinations(pipeline) + provisioned_pipelines.append(provisioned) + for message in messages: + print(message, file=sys.stderr) + from flowx.translator.engine import _pipeline_to_dict # lazy: heavy import (sqlglot) + + modified = [_pipeline_to_dict(pipeline) for pipeline in provisioned_pipelines] + _write_modified_report(args.report, modified, stamped_out) + + # Persist the collected answers as the kept configuration record. + config_out.parent.mkdir(parents=True, exist_ok=True) + config_out.write_text(json.dumps(answers, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"Wrote stamped IR to {stamped_out}", file=sys.stderr) + print(f"Wrote configuration to {config_out}", file=sys.stderr) + return 0 + + +def _run_materialize_lookup(args: argparse.Namespace) -> int: + """Implements the ``materialize-lookup`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``source`` (file path or + literal CSV string) and ``out``. + + Returns: + ``0`` when the JSON was written successfully, ``2`` when the + source could not be parsed as CSV. + """ + try: + rows = _parse_csv_source(args.source) + except ValueError as error: + print(f"Invalid CSV source: {error}", file=sys.stderr) + return 2 + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _parse_csv_source(source: str) -> list[dict[str, str]]: + """Parses a CSV file path or literal CSV string into a list of row dicts. + + Args: + source: Either a path to a CSV file or a literal CSV string with + a header row. + + Returns: + List of dicts, one per data row, keyed by the header names. + + Raises: + ValueError: When the CSV has no header row or is empty. + """ + import csv + + source_path = Path(source) + text = source_path.read_text(encoding="utf-8") if source_path.exists() else source + reader = csv.DictReader(text.splitlines()) + if reader.fieldnames is None: + raise ValueError("Source CSV is empty or missing a header row") + return [dict(row) for row in reader] + + +def _stamp_lookup_values_into_metadata_driven_motifs(pipeline, lookup_values: list[dict[str, Any]]): + """Stamps lookup values onto every metadata-driven motif marked for consolidation. + + Args: + pipeline: Configuration-stamped pipeline IR. + lookup_values: Rows materialised by the agent or the user. + + Returns: + A new :class:`Pipeline` whose metadata-driven motif activities + carry the supplied lookup rows. When *lookup_values* is empty + the pipeline is returned unchanged. + """ + if not lookup_values: + return pipeline + import dataclasses as _dataclasses + + from flowx.models.ir import MotifActivity as _MotifActivity + + stamped_tasks = [] + for task in pipeline.tasks: + if isinstance(task, _MotifActivity) and task.consolidate_metadata_driven: + stamped_tasks.append(_dataclasses.replace(task, lookup_values=list(lookup_values))) + else: + stamped_tasks.append(task) + return _dataclasses.replace(pipeline, tasks=stamped_tasks) + + +def _load_pipelines(report_path: Path) -> list[Any] | None: + """Loads every pipeline IR contained in a report file. + + Args: + report_path: Path to a translation report or pipeline IR JSON. + + Returns: + List of rehydrated :class:`Pipeline` objects, or ``None`` when + the file could not be parsed. + """ + from flowx.bundler.dab_writer import pipeline_dict_to_ir # lazy: heavy import, only inspect/modify + + try: + raw = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + print(f"Failed to read {report_path}: {error}", file=sys.stderr) + return None + pipeline_dicts = _extract_pipeline_dicts(raw) + return [pipeline_dict_to_ir(pipeline_dict)[0] for pipeline_dict in pipeline_dicts] + + +def _extract_pipeline_dicts(raw: Any) -> list[dict[str, Any]]: + """Normalises a translation report into a list of pipeline IR dicts. + + Args: + raw: Parsed JSON content from a report file. + + Returns: + List of dicts, each in the shape ``engine._pipeline_to_dict`` + produces. Empty when *raw* does not contain a recognisable + pipeline payload. + """ + # Single pipeline IR dict (has both "tasks" and "name" at top level) + if isinstance(raw, dict) and "tasks" in raw and "name" in raw: + return [raw] + # Multi-pipeline wrapper written by engine.py ({"pipelines": [...]}) + if isinstance(raw, dict) and "pipelines" in raw and isinstance(raw["pipelines"], list): + return [p for p in raw["pipelines"] if isinstance(p, dict) and "tasks" in p and "name" in p] + # Legacy aggregated translation report shape + if isinstance(raw, dict) and "translations" in raw: + return [ + {"name": entry["pipeline"], **entry["ir"]} + for entry in raw.get("translations", []) + if entry.get("status") == "translated" and entry.get("ir") + ] + return [] + + +def _configuration_from_answers(answers: dict[str, str]) -> TranslationConfiguration: + """Builds a :class:`TranslationConfiguration` from a validated answers dict. + + Args: + answers: Validated mapping of option_id to answer string. + + Returns: + Configuration with every answered field overridden and every + unanswered field defaulted. + + Raises: + ValueError: When an answer is not in the allowed set for its + option. + """ + validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} + motif_consolidations: dict[str, MotifConsolidate] = {} + for qid, value in validated.items(): + if qid.startswith(MOTIF_CONSOLIDATE_OPTION_PREFIX): + motif_consolidations[qid[len(MOTIF_CONSOLIDATE_OPTION_PREFIX) :]] = MotifConsolidate(value) + return TranslationConfiguration( + copy_activity_paradigm=CopyActivityParadigm( + validated.get("copy_activity_paradigm", DEFAULT_CONFIGURATION.copy_activity_paradigm) + ), + non_databricks_task_compute=NonDatabricksTaskCompute( + validated.get("non_databricks_task_compute", DEFAULT_CONFIGURATION.non_databricks_task_compute) + ), + use_lakeflow_connectors=UseLakeflowConnectors( + validated.get("use_lakeflow_connectors", DEFAULT_CONFIGURATION.use_lakeflow_connectors) + ), + lakeflow_connector_type=LakeflowConnectorType( + validated.get("lakeflow_connector_type", DEFAULT_CONFIGURATION.lakeflow_connector_type) + ), + metadata_driven_consolidate=MetadataDrivenConsolidate( + validated.get("metadata_driven_consolidate", DEFAULT_CONFIGURATION.metadata_driven_consolidate) + ), + metadata_driven_access=MetadataDrivenAccess( + validated.get("metadata_driven_access", DEFAULT_CONFIGURATION.metadata_driven_access) + ), + metadata_driven_size=MetadataDrivenSize( + validated.get("metadata_driven_size", DEFAULT_CONFIGURATION.metadata_driven_size) + ), + metadata_driven_lookup_tool=MetadataDrivenLookupTool( + validated.get("metadata_driven_lookup_tool", DEFAULT_CONFIGURATION.metadata_driven_lookup_tool) + ), + notify_destination=NotifyDestination( + validated.get("notify_destination", DEFAULT_CONFIGURATION.notify_destination) + ), + notify_events=NotifyEvents(validated.get("notify_events", DEFAULT_CONFIGURATION.notify_events)), + notify_destination_name=validated.get("notify_destination_name", ""), + notify_args=collect_notify_args(validated), + motif_consolidations=motif_consolidations, + ) + + +def _emit_json(payload: dict[str, Any], out: Path | None) -> None: + """Writes a JSON payload to a file or to stdout. + + Args: + payload: JSON-serialisable mapping to emit. + out: Destination path; ``None`` selects stdout. + """ + encoded = json.dumps(payload, indent=2, default=str) + if out is None: + print(encoded) + return + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(encoded + "\n", encoding="utf-8") + + +def _write_modified_report(report_path: Path, pipelines: list[dict[str, Any]], out: Path) -> None: + """Writes the configuration-stamped IR to *out* using the input report's shape. + + Args: + report_path: Path the modified report was sourced from. Used + only to detect whether the input was a single pipeline IR + or an aggregated translation report. + pipelines: Stamped pipeline IR dicts to write. + out: Destination path for the modified report. + """ + raw = json.loads(report_path.read_text(encoding="utf-8")) + if isinstance(raw, dict) and "translations" in raw: + by_name = {pipeline["name"]: pipeline for pipeline in pipelines} + for entry in raw.get("translations", []): + stamped = by_name.get(entry.get("pipeline")) + if stamped is not None and entry.get("ir") is not None: + entry["ir"] = {key: value for key, value in stamped.items() if key != "name"} + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(raw, indent=2, default=str) + "\n", encoding="utf-8") + return + payload = pipelines[0] if len(pipelines) == 1 else {"pipelines": pipelines} + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/orchestra/adapter/constants.py b/src/flowx/adapter/constants.py similarity index 51% rename from src/orchestra/adapter/constants.py rename to src/flowx/adapter/constants.py index 40c3e3c..02e30aa 100644 --- a/src/orchestra/adapter/constants.py +++ b/src/flowx/adapter/constants.py @@ -1,36 +1,31 @@ """String constants shared across the adapter and its bundler consumers. -Every adapter-side string the modifier stamps onto an IR field or that -the bundler reads back from one is defined here. Modules in -``flowx.adapter``, ``flowx.bundler``, and the test suite import -from this module to avoid string-literal drift between the producer and -consumer ends of the same value. +Defining each stamped/read-back value here keeps the producer (the modifier) and consumer (the +bundler) ends from drifting on string literals. """ from __future__ import annotations from typing import Final -QUESTION_COPY_ACTIVITY_PARADIGM: Final[str] = "copy_activity_paradigm" -QUESTION_NON_DATABRICKS_TASK_COMPUTE: Final[str] = "non_databricks_task_compute" -QUESTION_USE_LAKEFLOW_CONNECTORS: Final[str] = "use_lakeflow_connectors" -QUESTION_LAKEFLOW_CONNECTOR_TYPE: Final[str] = "lakeflow_connector_type" -QUESTION_METADATA_DRIVEN_CONSOLIDATE: Final[str] = "metadata_driven_consolidate" -QUESTION_METADATA_DRIVEN_ACCESS: Final[str] = "metadata_driven_access" -QUESTION_METADATA_DRIVEN_SIZE: Final[str] = "metadata_driven_size" -QUESTION_METADATA_DRIVEN_LOOKUP_TOOL: Final[str] = "metadata_driven_lookup_tool" - -# Per-detected-motif consolidation question_ids carry the motif_id as a suffix -# (e.g. ``consolidate_motif:rest_api_pagination``) so each detected motif gets -# its own question. Validation strips the prefix and validates the answer -# against the :class:`MotifConsolidate` enum. -MOTIF_CONSOLIDATE_QUESTION_PREFIX: Final[str] = "consolidate_motif:" +OPTION_COPY_ACTIVITY_PARADIGM: Final[str] = "copy_activity_paradigm" +OPTION_NON_DATABRICKS_TASK_COMPUTE: Final[str] = "non_databricks_task_compute" +OPTION_USE_LAKEFLOW_CONNECTORS: Final[str] = "use_lakeflow_connectors" +OPTION_LAKEFLOW_CONNECTOR_TYPE: Final[str] = "lakeflow_connector_type" +OPTION_METADATA_DRIVEN_CONSOLIDATE: Final[str] = "metadata_driven_consolidate" +OPTION_METADATA_DRIVEN_ACCESS: Final[str] = "metadata_driven_access" +OPTION_METADATA_DRIVEN_SIZE: Final[str] = "metadata_driven_size" +OPTION_METADATA_DRIVEN_LOOKUP_TOOL: Final[str] = "metadata_driven_lookup_tool" + +# Per-motif consolidation option ids suffix the motif_id (e.g. consolidate_motif:rest_api_pagination) +# so each motif gets its own option; validation strips this prefix and checks against MotifConsolidate. +MOTIF_CONSOLIDATE_OPTION_PREFIX: Final[str] = "consolidate_motif:" METADATA_DRIVEN_MOTIF_ID: Final[str] = "metadata_driven_bulk_copy" -PHASE_INGEST: Final[str] = "ingest" -PHASE_TRANSLATE: Final[str] = "translate" -PHASE_PREPARE: Final[str] = "prepare" +PHASE_DISCOVER: Final[str] = "discover" +PHASE_CONVERT: Final[str] = "convert" +PHASE_PACKAGE: Final[str] = "package" INPUT_ADF_SOURCE_PATH: Final[str] = "adf_source_path" INPUT_ADF_RESOURCE_URL: Final[str] = "adf_resource_url" @@ -42,6 +37,9 @@ INPUT_SCHEMA: Final[str] = "schema" INPUT_BUNDLE_NAME: Final[str] = "bundle_name" INPUT_DATABRICKS_PROFILE: Final[str] = "databricks_profile" +INPUT_RESULTS_TABLE: Final[str] = "results_table" +INPUT_RESULTS_WAREHOUSE: Final[str] = "results_warehouse_id" +INPUT_INSTALL_DASHBOARD: Final[str] = "install_dashboard" LAKEFLOW_CONNECTOR_TYPE_QUERY_BASED: Final[str] = "query_based" LAKEFLOW_CONNECTOR_TYPE_CDC: Final[str] = "cdc" @@ -77,3 +75,9 @@ ) DATABASE_SOURCE_TYPE_HINT: Final[str] = "database" + +OPTION_NOTIFY_DESTINATION: Final[str] = "notify_destination" +OPTION_NOTIFY_EVENTS: Final[str] = "notify_events" +OPTION_NOTIFY_DESTINATION_NAME: Final[str] = "notify_destination_name" +# Per-field notification follow-up option ids (e.g. notify_email_recipients, +# notify_slack_url) are defined by the _NOTIFY_FIELDS registry in operations.py. diff --git a/src/orchestra/adapter/models.py b/src/flowx/adapter/models.py similarity index 72% rename from src/orchestra/adapter/models.py rename to src/flowx/adapter/models.py index 70f5abc..a54e626 100644 --- a/src/orchestra/adapter/models.py +++ b/src/flowx/adapter/models.py @@ -34,7 +34,7 @@ class LakeflowConnectorType(StrEnum): Used only when ``use_lakeflow_connectors`` is ``lakeflow_connect``. The modifier still routes Copy activities that read from a SQL query into - the query-based connector regardless of this preference; this enum + the query-based connector regardless of this configuration; this enum controls the default for table-based Copy activities. """ @@ -91,6 +91,32 @@ class MotifConsolidate(StrEnum): CONSOLIDATE = "consolidate" +class NotifyDestination(StrEnum): + """How an activity->Notify (activity_and_notify) motif's notifications are handled. + + ``KEEP`` preserves the current behaviour (the WebActivity notify + activities translate directly; the motif is not collapsed). Any other + value collapses the motif: the upstream activity (Copy, Notebook, Lookup, + …) becomes the task and the downstream notifications become Databricks + job-task notifications routed to the chosen destination. + """ + + KEEP = "keep" + EMAIL = "email" + SLACK = "slack" + TEAMS = "teams" + PAGERDUTY = "pagerduty" + WEBHOOK = "webhook" + + +class NotifyEvents(StrEnum): + """Which job-task events fire the collapsed notification.""" + + ON_FAILURE = "on_failure" + ON_SUCCESS = "on_success" + BOTH = "both" + + FIELD_TO_ENUM: Final[MappingProxyType[str, type[StrEnum]]] = MappingProxyType( { "copy_activity_paradigm": CopyActivityParadigm, @@ -101,12 +127,14 @@ class MotifConsolidate(StrEnum): "metadata_driven_access": MetadataDrivenAccess, "metadata_driven_size": MetadataDrivenSize, "metadata_driven_lookup_tool": MetadataDrivenLookupTool, + "notify_destination": NotifyDestination, + "notify_events": NotifyEvents, } ) @dataclass(frozen=True, slots=True, kw_only=True) -class TranslationPreferences: +class TranslationConfiguration: """Snapshot of user choices that shape downstream IR transformations. Each field accepts either a raw string or the corresponding enum @@ -122,11 +150,10 @@ class TranslationPreferences: value is a partial mapping of the fields above; only the keys present win over the pipeline-wide defaults. - ADF DatabricksNotebook and DatabricksSparkPython tasks always keep - the cluster binding derived from the source linked service -- the - serverless replacement option was removed because it silently - discarded init scripts and DBR-version constraints that the source - pipeline relied on. + Notes: + ADF DatabricksNotebook and DatabricksSparkPython tasks always keep the cluster binding from + the source linked service; the serverless-replacement option was removed because it silently + discarded init scripts and DBR-version constraints the source pipeline relied on. """ copy_activity_paradigm: CopyActivityParadigm = CopyActivityParadigm.NOTEBOOK @@ -137,6 +164,12 @@ class TranslationPreferences: metadata_driven_access: MetadataDrivenAccess = MetadataDrivenAccess.NO metadata_driven_size: MetadataDrivenSize = MetadataDrivenSize.LARGE metadata_driven_lookup_tool: MetadataDrivenLookupTool = MetadataDrivenLookupTool.NONE + notify_destination: NotifyDestination = NotifyDestination.KEEP + notify_events: NotifyEvents = NotifyEvents.BOTH + notify_destination_name: str = "" + # SDK config kwargs for the chosen destination (e.g. addresses/url/integration_key), keyed by SDK + # arg name; populated from the per-field follow-up answers via collect_notify_args. + notify_args: dict[str, str] = field(default_factory=dict) motif_consolidations: dict[str, MotifConsolidate] = field(default_factory=dict) per_task: dict[str, dict[str, str]] = field(default_factory=dict) @@ -151,29 +184,27 @@ def __post_init__(self) -> None: value = getattr(self, field_name) if not isinstance(value, enum_cls): object.__setattr__(self, field_name, enum_cls(value)) - # motif_consolidations is keyed by dynamic motif_id rather than a - # fixed field name, so it is not in FIELD_TO_ENUM. Coerce its - # values to MotifConsolidate members here. + # motif_consolidations is keyed by dynamic motif_id (not in FIELD_TO_ENUM), so coerce here. coerced: dict[str, MotifConsolidate] = {} for motif_id, choice in self.motif_consolidations.items(): coerced[motif_id] = choice if isinstance(choice, MotifConsolidate) else MotifConsolidate(choice) object.__setattr__(self, "motif_consolidations", coerced) - def effective_for(self, task_key: str) -> TranslationPreferences: - """Returns a preferences view where per-task overrides for *task_key* win. + def effective_for(self, task_key: str) -> TranslationConfiguration: + """Returns a configuration view where per-task overrides for *task_key* win. Args: task_key: Sanitised task key of the activity being prepared. Returns: - A new :class:`TranslationPreferences` with overrides for + A new :class:`TranslationConfiguration` with overrides for *task_key* applied on top of the pipeline-wide values, or ``self`` unchanged when no overrides exist for *task_key*. """ override = self.per_task.get(task_key) if not override: return self - return TranslationPreferences( + return TranslationConfiguration( copy_activity_paradigm=CopyActivityParadigm( override.get("copy_activity_paradigm", self.copy_activity_paradigm) ), @@ -201,12 +232,12 @@ def effective_for(self, task_key: str) -> TranslationPreferences: ) -DEFAULT_PREFERENCES: Final[TranslationPreferences] = TranslationPreferences() +DEFAULT_CONFIGURATION: Final[TranslationConfiguration] = TranslationConfiguration() @dataclass(frozen=True, slots=True, kw_only=True) -class QuestionOption: - """One allowed answer to a :class:`TranslationQuestion`. +class OptionChoice: + """One allowed answer to a :class:`TranslationOption`. Attributes: value: Machine-readable identifier matching the backing enum member. @@ -221,63 +252,63 @@ class QuestionOption: @dataclass(frozen=True, slots=True, kw_only=True) -class TranslationQuestion: - """A single just-in-time question raised by the IR inspector. +class TranslationOption: + """A single just-in-time option raised by the IR inspector. Attributes: - question_id: Stable identifier matching the preferences field. - prompt: Human-readable question text. - rationale: One- or two-sentence explanation of why the question + option_id: Stable identifier matching the configuration field. + prompt: Human-readable option text. + rationale: One- or two-sentence explanation of why the option is being raised. options: Allowed answers; the first option is the conservative default and is also exposed via ``default``. affected_task_keys: Activity task keys impacted by the answer. - default: Default value applied when the caller skips the question. - conditions: Tuples of ``(question_id, expected_value)`` that must + default: Default value applied when the caller skips the option. + conditions: Tuples of ``(option_id, expected_value)`` that must already be answered with the expected value before this - question surfaces. An empty tuple means the question is + option surfaces. An empty tuple means the option is evaluated solely on its IR/motif preconditions. """ - question_id: str + option_id: str prompt: str rationale: str - options: tuple[QuestionOption, ...] + options: tuple[OptionChoice, ...] affected_task_keys: tuple[str, ...] default: str conditions: tuple[tuple[str, str], ...] = () @dataclass(slots=True, kw_only=True) -class PendingQuestions: - """Outstanding questions for a single pipeline translation. +class PendingOptions: + """Outstanding options for a single pipeline translation. Attributes: - pipeline_name: Name of the pipeline these questions belong to. - questions: Ordered list of questions still awaiting an answer. + pipeline_name: Name of the pipeline these options belong to. + options: Ordered list of options still awaiting an answer. """ pipeline_name: str - questions: list[TranslationQuestion] = field(default_factory=list) + options: list[TranslationOption] = field(default_factory=list) @dataclass(frozen=True, slots=True, kw_only=True) -class MigrationInputQuestion: +class MigrationInputOption: """A free-text input gathered before an flowx phase runs. Attributes: - question_id: Stable identifier the skill uses to key the answer. - prompt: Human-readable question text. + option_id: Stable identifier the skill uses to key the answer. + prompt: Human-readable option text. description: One-sentence explanation of what the value is used for and what shape is expected (path, URL, identifier). default: Default value applied when the caller skips the - question; ``None`` when the field is required and has no + option; ``None`` when the field is required and has no sensible default. required: When ``True`` the skill must collect a value; when ``False`` the default (which may be ``None``) is permitted. """ - question_id: str + option_id: str prompt: str description: str default: str | None = None @@ -286,13 +317,13 @@ class MigrationInputQuestion: @dataclass(slots=True, kw_only=True) class PendingMigrationInputs: - """Outstanding migration-phase input questions for a single phase. + """Outstanding migration-phase input options for a single phase. Attributes: - phase: The migration phase name (``"ingest"``, ``"translate"``, - ``"prepare"``). - questions: Ordered list of questions still awaiting an answer. + phase: The migration phase name (``"discover"``, ``"convert"``, + ``"package"``). + options: Ordered list of options still awaiting an answer. """ phase: str - questions: list[MigrationInputQuestion] = field(default_factory=list) + options: list[MigrationInputOption] = field(default_factory=list) diff --git a/src/orchestra/adapter/operations.py b/src/flowx/adapter/operations.py similarity index 51% rename from src/orchestra/adapter/operations.py rename to src/flowx/adapter/operations.py index fe21cbc..fb292fe 100644 --- a/src/orchestra/adapter/operations.py +++ b/src/flowx/adapter/operations.py @@ -1,7 +1,7 @@ -"""Standalone operations: question gathering, validation, and IR modification. +"""Standalone operations: option gathering, validation, and IR modification. The agent adapter and the CLI bridge call into these functions; nothing -here is stateful. Preference dataclasses, StrEnums, and question shapes +here is stateful. Configuration dataclasses, StrEnums, and option shapes live in :mod:`flowx.adapter.models`. """ @@ -23,14 +23,17 @@ LAKEFLOW_CONNECT_REPLACEMENT, LAKEFLOW_CONNECTOR_TYPE_QUERY_BASED, METADATA_DRIVEN_MOTIF_ID, - MOTIF_CONSOLIDATE_QUESTION_PREFIX, - QUESTION_COPY_ACTIVITY_PARADIGM, - QUESTION_METADATA_DRIVEN_ACCESS, - QUESTION_METADATA_DRIVEN_CONSOLIDATE, - QUESTION_METADATA_DRIVEN_LOOKUP_TOOL, - QUESTION_METADATA_DRIVEN_SIZE, - QUESTION_NON_DATABRICKS_TASK_COMPUTE, - QUESTION_USE_LAKEFLOW_CONNECTORS, + MOTIF_CONSOLIDATE_OPTION_PREFIX, + OPTION_COPY_ACTIVITY_PARADIGM, + OPTION_METADATA_DRIVEN_ACCESS, + OPTION_METADATA_DRIVEN_CONSOLIDATE, + OPTION_METADATA_DRIVEN_LOOKUP_TOOL, + OPTION_METADATA_DRIVEN_SIZE, + OPTION_NON_DATABRICKS_TASK_COMPUTE, + OPTION_NOTIFY_DESTINATION, + OPTION_NOTIFY_DESTINATION_NAME, + OPTION_NOTIFY_EVENTS, + OPTION_USE_LAKEFLOW_CONNECTORS, ) from flowx.adapter.models import ( FIELD_TO_ENUM, @@ -42,10 +45,12 @@ MetadataDrivenSize, MotifConsolidate, NonDatabricksTaskCompute, - PendingQuestions, - QuestionOption, - TranslationPreferences, - TranslationQuestion, + NotifyDestination, + NotifyEvents, + OptionChoice, + PendingOptions, + TranslationConfiguration, + TranslationOption, UseLakeflowConnectors, ) from flowx.adapter.predicates import ( @@ -59,65 +64,73 @@ from flowx.models.ir import ( Activity, CopyActivity, + Dependency, ForEachActivity, IfConditionActivity, MotifActivity, Pipeline, SwitchActivity, SwitchCase, + WebActivity, ) from flowx.models.motifs import MOTIF_LAKEFLOW_CONNECT_DATABASE +# Free-text option ids (no backing enum) that validate_answer accepts with any value, vs a genuinely +# unknown id which it rejects -- the set is the NOTIFY_FREE_TEXT_OPTION_IDS registry defined below. -def enum_for(question_id: str) -> type[StrEnum] | None: - """Returns the enum class backing a preference field. + +def enum_for(option_id: str) -> type[StrEnum] | None: + """Returns the enum class backing a configuration field. Args: - question_id: Field name (e.g. ``"copy_activity_paradigm"``) or + option_id: Field name (e.g. ``"copy_activity_paradigm"``) or per-motif id (e.g. ``"consolidate_motif:rest_api_pagination"``). Returns: The :class:`StrEnum` subclass that defines the allowed values, or - ``None`` when the question_id is unknown. + ``None`` when the option_id is unknown. """ - if question_id.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): + if option_id.startswith(MOTIF_CONSOLIDATE_OPTION_PREFIX): return MotifConsolidate - return FIELD_TO_ENUM.get(question_id) + return FIELD_TO_ENUM.get(option_id) -def allowed_values_for(question_id: str) -> tuple[str, ...]: - """Returns the allowed string values for a preference field. +def allowed_values_for(option_id: str) -> tuple[str, ...]: + """Returns the allowed string values for a configuration field. Args: - question_id: Field name (e.g. ``"copy_activity_paradigm"``). + option_id: Field name (e.g. ``"copy_activity_paradigm"``). Returns: Tuple of allowed string values in declaration order. Empty when the field is unknown. """ - enum_cls = enum_for(question_id) + enum_cls = enum_for(option_id) return tuple(member.value for member in enum_cls) if enum_cls else () -def validate_answer(question_id: str, value: str) -> str: - """Returns *value* when it is an allowed answer for *question_id*. +def validate_answer(option_id: str, value: str) -> str: + """Returns *value* when it is an allowed answer for *option_id*. Args: - question_id: Stable question identifier. + option_id: Stable option identifier. value: Caller-supplied answer string. Returns: The validated value, unchanged. Raises: - ValueError: When *question_id* is not known or *value* is not in - the allowed set for the question. + ValueError: When *option_id* is not known or *value* is not in + the allowed set for the option. """ - allowed = allowed_values_for(question_id) - if not allowed: - raise ValueError(f"Unknown question_id {question_id!r}") - if value not in allowed: - raise ValueError(f"Invalid answer {value!r} for {question_id!r}; allowed: {sorted(allowed)}") + allowed = allowed_values_for(option_id) + if allowed: + if value not in allowed: + raise ValueError(f"Invalid answer {value!r} for {option_id!r}; allowed: {sorted(allowed)}") + return value + if option_id in NOTIFY_FREE_TEXT_OPTION_IDS: + return value + raise ValueError(f"Unknown option_id {option_id!r}") return value @@ -213,118 +226,539 @@ def _walk_workspace_paths(tasks: list[dict[str, Any]] | None, candidates: list[s _walk_workspace_paths(task.get("default_activities"), candidates) -def gather_questions( +def gather_options( pipeline: Pipeline, motifs: list | None = None, *, answers: dict[str, str] | None = None, -) -> PendingQuestions: - """Walks the IR and returns the questions that apply to *pipeline*. +) -> PendingOptions: + """Walks the IR and returns the options that apply to *pipeline*. Args: pipeline: Translated pipeline IR after motif collapsing. motifs: Detected motifs, used to surface the Lakeflow Connect - question for multi-step database ingestion patterns. - answers: Answers the caller has already collected. Questions - whose ``question_id`` is in this mapping are filtered out, - and questions whose ``conditions`` reference earlier answers + option for multi-step database ingestion patterns. + answers: Answers the caller has already collected. Options + whose ``option_id`` is in this mapping are filtered out, + and options whose ``conditions`` reference earlier answers are evaluated against this mapping. Returns: - A :class:`PendingQuestions` instance carrying the questions whose + A :class:`PendingOptions` instance carrying the options whose IR preconditions and answer-dependent conditions are met but - whose ``question_id`` has not yet been answered. + whose ``option_id`` has not yet been answered. """ motif_list = motifs or [] answer_map = answers or {} builders = ( - _build_use_lakeflow_connectors_question, - _build_lakeflow_connector_type_question, - _build_copy_activity_paradigm_question, - _build_non_databricks_task_compute_question, - _build_metadata_driven_consolidate_question, - _build_metadata_driven_access_question, - _build_metadata_driven_size_question, - _build_metadata_driven_lookup_tool_question, + _build_use_lakeflow_connectors_option, + _build_lakeflow_connector_type_option, + _build_copy_activity_paradigm_option, + _build_non_databricks_task_compute_option, + _build_metadata_driven_consolidate_option, + _build_metadata_driven_access_option, + _build_metadata_driven_size_option, + _build_metadata_driven_lookup_tool_option, ) candidates = (builder(pipeline, motif_list, answers=answer_map) for builder in builders) pending = [ - question - for question in candidates - if question is not None - and question.question_id not in answer_map - and _conditions_met(question.conditions, answer_map) + option + for option in candidates + if option is not None and option.option_id not in answer_map and _conditions_met(option.conditions, answer_map) ] - # Per-motif "consolidate?" questions: one per detected motif. Each - # gets its own question_id ``consolidate_motif:`` so the - # adapter can solicit and validate them independently. Default is - # ``keep`` -- nothing is collapsed without an explicit yes. - for motif_question in _build_motif_consolidation_questions(motif_list): - if motif_question.question_id in answer_map: + # Per-motif consolidate options: one per detected motif (consolidate_motif:), default keep. + for motif_option in _build_motif_consolidation_options(motif_list): + if motif_option.option_id in answer_map: + continue + pending.append(motif_option) + # activity->Notify chain: destination choice, then one follow-up per SDK field once chosen. + for notify_option in _build_notify_options(pipeline, answer_map): + if notify_option.option_id in answer_map: continue - pending.append(motif_question) - return PendingQuestions(pipeline_name=pipeline.name, questions=pending) + pending.append(notify_option) + return PendingOptions(pipeline_name=pipeline.name, options=pending) def _conditions_met(conditions: tuple[tuple[str, str], ...], answers: dict[str, str]) -> bool: """Returns True when every condition is satisfied by *answers*. Args: - conditions: Tuples of ``(question_id, expected_value)`` from a - :class:`TranslationQuestion`. - answers: Mapping of question_id to the caller-supplied answer. + conditions: Tuples of ``(option_id, expected_value)`` from a + :class:`TranslationOption`. + answers: Mapping of option_id to the caller-supplied answer. Returns: - ``True`` when every condition's question has been answered with + ``True`` when every condition's option has been answered with the expected value (or when *conditions* is empty); ``False`` otherwise. """ return all(answers.get(qid) == expected for qid, expected in conditions) -def apply_preferences(pipeline: Pipeline, pipeline_preferences: TranslationPreferences) -> Pipeline: - """Returns a copy of *pipeline* with preferences stamped onto each activity. +# Notify destinations other than KEEP — the destination-name and events follow-ups apply to all of them. +_NOTIFY_NON_KEEP: tuple[str, ...] = tuple(d.value for d in NotifyDestination if d is not NotifyDestination.KEEP) + + +def _show_when_from_conditions(conditions: tuple[tuple[str, str], ...]) -> list[dict[str, Any]]: + """Translates a TranslationOption's equality ``conditions`` into ``show_when`` clauses.""" + return [{"option_id": qid, "in": [value]} for qid, value in conditions] + + +def _option_schema(option: TranslationOption, show_when: list[dict[str, Any]]) -> dict[str, Any]: + """Serialises one option into a declarative schema entry the agent can walk locally.""" + return { + "option_id": option.option_id, + "prompt": option.prompt, + "rationale": option.rationale, + "choices": [ + {"value": choice.value, "label": choice.label, "description": choice.description} + for choice in option.options + ], + "free_text": not option.options, + "default": option.default, + "affected_task_keys": list(option.affected_task_keys), + "show_when": show_when, + } + + +def _build_notify_schema(pipeline: Pipeline) -> list[dict[str, Any]]: + """The full activity_and_notify chain as schema entries (every destination's follow-ups).""" + affected = _notify_present(pipeline) + if not affected: + return [] + entries = [_option_schema(_build_notify_destination_option(affected), [])] + for dest, fields in _NOTIFY_FIELDS.items(): + dest_clause = [{"option_id": OPTION_NOTIFY_DESTINATION, "in": [dest]}] + entries.extend(_option_schema(_notify_field_option(dest, field, affected), dest_clause) for field in fields) + non_keep_clause = [{"option_id": OPTION_NOTIFY_DESTINATION, "in": list(_NOTIFY_NON_KEEP)}] + entries.append(_option_schema(_notify_name_option(affected), non_keep_clause)) + entries.append(_option_schema(_build_notify_events_option(affected), non_keep_clause)) + return entries + + +def build_option_schema(pipeline: Pipeline, motifs: list | None = None) -> list[dict[str, Any]]: + """Returns the full declarative option tree for *pipeline* as schema dicts. + + Unlike :func:`gather_options` -- which filters to the options *currently* pending given the + answers collected so far -- this returns **every** option the pipeline can raise, each annotated + with a ``show_when`` condition: a conjunction of ``{"option_id", "in": [values]}`` clauses (empty + list = always shown). The agent walks this tree locally, asking only the options whose + ``show_when`` clauses are all satisfied by the answers gathered so far, so a multi-step chain + (e.g. notify destination -> per-field follow-ups, or metadata-driven consolidate -> access -> + size -> lookup-tool) needs no per-follow-up round trip back to the server. ``apply_answers`` + remains the single validate-and-apply path. + """ + motif_list = motifs or [] + schema: list[dict[str, Any]] = [] + builders = ( + _build_use_lakeflow_connectors_option, + _build_lakeflow_connector_type_option, + _build_copy_activity_paradigm_option, + _build_non_databricks_task_compute_option, + _build_metadata_driven_consolidate_option, + _build_metadata_driven_access_option, + _build_metadata_driven_size_option, + _build_metadata_driven_lookup_tool_option, + ) + for builder in builders: + option = builder(pipeline, motif_list, answers={}) + if option is not None: + schema.append(_option_schema(option, _show_when_from_conditions(option.conditions))) + for motif_option in _build_motif_consolidation_options(motif_list): + schema.append(_option_schema(motif_option, _show_when_from_conditions(motif_option.conditions))) + schema.extend(_build_notify_schema(pipeline)) + return schema + + +def apply_configuration(pipeline: Pipeline, pipeline_configuration: TranslationConfiguration) -> Pipeline: + """Returns a copy of *pipeline* with configuration stamped onto each activity. Args: pipeline: Translated pipeline IR after motif collapsing. - pipeline_preferences: Validated pipeline-wide preferences. + pipeline_configuration: Validated pipeline-wide configuration. Returns: A new :class:`Pipeline` whose activities carry concrete decisions about compute, target format, and Lakeflow Connect replacement. The input pipeline is not mutated. """ - stamped_tasks = [_stamp_activity(activity, pipeline_preferences) for activity in pipeline.tasks] - return dataclasses.replace( + stamped_tasks = [_stamp_activity(activity, pipeline_configuration) for activity in pipeline.tasks] + stamped = dataclasses.replace( pipeline, tasks=stamped_tasks, - translation_preferences=pipeline_preferences, + translation_configuration=pipeline_configuration, + ) + if pipeline_configuration.notify_destination is not NotifyDestination.KEEP: + stamped = _collapse_notify(stamped, pipeline_configuration) + return stamped + + +def provision_notification_destinations(pipeline: Pipeline) -> tuple[Pipeline, list[str]]: + """Create the Databricks notification destinations for *pipeline* at prompt time. + + Walks the collapsed tasks and, for every non-email ``activity_and_notify`` + notification spec, creates (or reuses) the destination via the SDK now and stamps + the resolved ``destination_id`` back onto the task. Email specs are left untouched + -- email wires raw ``email_notifications`` and needs no destination. This runs in + the adapter ``modify`` phase so the destination exists (and validates) as soon as + the user answers, rather than at prepare time. + + Returns: + ``(pipeline, messages)`` -- a copy of *pipeline* with resolved ids stamped in, + and human-readable status lines (one per destination) for surfacing to the user. + When no non-email notifications are present the pipeline is returned unchanged + with an empty message list and no SDK call is made. + """ + from flowx.preparer.notifications import provision_destination + + messages: list[str] = [] + new_tasks: list = [] + changed = False + for task in pipeline.tasks: + spec = getattr(task, "notifications", None) + if spec and spec.get("destination") not in (None, "", "email"): + new_spec, message = provision_destination(spec) + if message: + messages.append(message) + if new_spec is not spec: + task = dataclasses.replace(task, notifications=new_spec) + changed = True + new_tasks.append(task) + if not changed: + return pipeline, messages + return dataclasses.replace(pipeline, tasks=new_tasks), messages + + +_NOTIFY_WEBHOOK_DESTS: frozenset[str] = frozenset({"slack", "teams", "webhook"}) + + +@dataclasses.dataclass(frozen=True, slots=True) +class _NotifyField: + """One Databricks-SDK config field of a notification destination. + + Surfaced as a chained follow-up option after the user picks a destination. + + Attributes: + option_id: Adapter option id for the follow-up question. + sdk_arg: The kwarg on the SDK config class (e.g. ``url``, ``addresses``). + prompt: The question text. + required: Whether the SDK config needs this field for the destination. + is_list: When True the comma-separated answer is split into a list + (used for email ``addresses``). + """ + + option_id: str + sdk_arg: str + prompt: str + required: bool = True + is_list: bool = False + + +# Per-destination SDK config fields (required first), from the SDK notification-destination config +# classes; the adapter chains one follow-up option per field after the destination is chosen. +_NOTIFY_FIELDS: dict[str, tuple[_NotifyField, ...]] = { + NotifyDestination.EMAIL.value: ( + _NotifyField( + "notify_email_recipients", + "addresses", + "Recipient email address(es)? (comma-separated)", + is_list=True, + ), + ), + NotifyDestination.SLACK.value: ( + _NotifyField("notify_slack_url", "url", "Slack incoming webhook URL?"), + _NotifyField("notify_slack_channel_id", "channel_id", "Slack channel id? (optional)", required=False), + _NotifyField("notify_slack_oauth_token", "oauth_token", "Slack OAuth token? (optional)", required=False), + ), + NotifyDestination.TEAMS.value: (_NotifyField("notify_teams_url", "url", "Microsoft Teams incoming webhook URL?"),), + NotifyDestination.PAGERDUTY.value: ( + _NotifyField("notify_pagerduty_integration_key", "integration_key", "PagerDuty integration key?"), + ), + NotifyDestination.WEBHOOK.value: ( + _NotifyField("notify_webhook_url", "url", "Generic webhook URL?"), + _NotifyField("notify_webhook_username", "username", "Webhook basic-auth username? (optional)", required=False), + _NotifyField("notify_webhook_password", "password", "Webhook basic-auth password? (optional)", required=False), + ), +} + +# Free-text follow-up option ids (no backing enum). ``validate_answer`` accepts +# any value for these, distinct from a genuinely unknown option id. +NOTIFY_FREE_TEXT_OPTION_IDS: frozenset[str] = frozenset( + {field.option_id for fields in _NOTIFY_FIELDS.values() for field in fields} | {OPTION_NOTIFY_DESTINATION_NAME} +) + + +def _find_notify_groups(tasks: list) -> dict[str, list[tuple[Any, str]]]: + """Find activity->notify groups in the IR: any non-Web task with WebActivity dependents. + + Returns a mapping of ``upstream_task_key`` -> list of ``(web_activity, outcome)`` where + outcome is the dependency condition (``Succeeded`` / ``Failed`` / ...). + + Any activity -- Copy, Notebook, Lookup, SparkPython, a stored procedure, etc. -- that is directly + followed by one or more WebActivity calls is a candidate: collapsing turns those Web calls into + native Databricks job-task notifications on the upstream task. The upstream is any non-Web task + (a WebActivity that merely follows another WebActivity is not treated as a notify target). + Mirrors the activity_and_notify motif on the (non-collapsed) IR. + """ + upstream = {t.task_key for t in tasks if not isinstance(t, WebActivity)} + groups: dict[str, list[tuple[Any, str]]] = {} + for task in tasks: + if isinstance(task, WebActivity) and task.depends_on: + for dep in task.depends_on: + if dep.task_key in upstream: + groups.setdefault(dep.task_key, []).append((task, dep.outcome or "Succeeded")) + break + return groups + + +def _notify_present(pipeline: Pipeline) -> tuple[str, ...]: + """Task keys of activities that have notify WebActivity dependents.""" + return tuple(sorted(_find_notify_groups(pipeline.tasks).keys())) + + +def _notify_dest(answers: dict[str, str] | None) -> str: + return (answers or {}).get(OPTION_NOTIFY_DESTINATION, "") + + +def _freetext_option(option_id: str, prompt: str, rationale: str, affected: tuple[str, ...]) -> TranslationOption: + """Builds a free-text option (no enum choices).""" + return TranslationOption( + option_id=option_id, + prompt=prompt, + rationale=rationale, + options=(), + affected_task_keys=affected, + default="", ) -def _build_copy_activity_paradigm_question( +def _build_notify_destination_option(affected: tuple[str, ...]) -> TranslationOption: + """Asks whether/how to route an activity->Notify motif to a Databricks destination.""" + return TranslationOption( + option_id=OPTION_NOTIFY_DESTINATION, + prompt=( + "One or more activities are followed by notification Web activities. " + "Route these to a Databricks destination?" + ), + rationale=( + "Choosing a destination collapses the pattern: the upstream activity becomes the task " + "and the downstream notifications become Databricks job-task success/failure " + "notifications (email_notifications or webhook_notifications). The ADF Web activity's own " + "URL/body is not used. Keeping preserves the current per-activity Web activity translation." + ), + options=( + OptionChoice( + value=NotifyDestination.KEEP.value, + label="Keep current behavior", + description="Do not collapse; translate the Web activities directly.", + ), + OptionChoice( + value=NotifyDestination.EMAIL.value, + label="Email", + description="Wire email_notifications with recipient addresses.", + ), + OptionChoice( + value=NotifyDestination.SLACK.value, + label="Slack", + description="Create a Slack notification destination and wire webhook_notifications.", + ), + OptionChoice( + value=NotifyDestination.TEAMS.value, + label="Microsoft Teams", + description="Create a Teams notification destination and wire webhook_notifications.", + ), + OptionChoice( + value=NotifyDestination.PAGERDUTY.value, + label="PagerDuty", + description="Create a PagerDuty notification destination and wire webhook_notifications.", + ), + OptionChoice( + value=NotifyDestination.WEBHOOK.value, + label="Generic Webhook", + description="Create a generic webhook destination and wire webhook_notifications.", + ), + ), + affected_task_keys=affected, + default=NotifyDestination.KEEP.value, + ) + + +def _build_notify_events_option(affected: tuple[str, ...]) -> TranslationOption: + return TranslationOption( + option_id=OPTION_NOTIFY_EVENTS, + prompt="Which events should notify?", + rationale="Defaults to both (whatever the source notify activities covered). Restrict if desired.", + options=( + OptionChoice( + value=NotifyEvents.BOTH.value, + label="Both success and failure", + description="Wire on_success and on_failure (as the source activities had).", + ), + OptionChoice( + value=NotifyEvents.ON_FAILURE.value, label="On failure only", description="Only wire on_failure." + ), + OptionChoice( + value=NotifyEvents.ON_SUCCESS.value, label="On success only", description="Only wire on_success." + ), + ), + affected_task_keys=affected, + default=NotifyEvents.BOTH.value, + ) + + +def _notify_field_option(dest: str, field: _NotifyField, affected: tuple[str, ...]) -> TranslationOption: + """Builds the free-text follow-up for one SDK config field of a notification destination.""" + suffix = "" if field.required else " (optional -- leave blank to skip)" + return _freetext_option( + field.option_id, + field.prompt, + f"Maps to the Databricks SDK {dest} config field `{field.sdk_arg}`.{suffix}", + affected, + ) + + +def _notify_name_option(affected: tuple[str, ...]) -> TranslationOption: + """Builds the optional destination-display-name follow-up.""" + return _freetext_option( + OPTION_NOTIFY_DESTINATION_NAME, + "Display name for the notification destination? (optional; default derived)", + "Reused if a destination with this name already exists, so prepare is idempotent.", + affected, + ) + + +def _build_notify_options(pipeline: Pipeline, answers: dict[str, str] | None) -> list[TranslationOption]: + """Returns the chained activity_and_notify options. + + The first prompt is the destination choice. Once a (non-keep) destination is + answered, one follow-up option is surfaced per SDK field of that destination + (required first; optional fields flagged), so the agent prompts for each field + sequentially, followed by an optional display name and the events selector. + """ + affected = _notify_present(pipeline) + if not affected: + return [] + options: list[TranslationOption] = [_build_notify_destination_option(affected)] + dest = _notify_dest(answers) + if dest in ("", NotifyDestination.KEEP.value): + return options + options.extend(_notify_field_option(dest, field, affected) for field in _NOTIFY_FIELDS.get(dest, ())) + options.append(_notify_name_option(affected)) + options.append(_build_notify_events_option(affected)) + return options + + +def collect_notify_args(answers: dict[str, str]) -> dict[str, str]: + """Collect the per-field notification answers into ``{sdk_arg: value}``. + + Reads only the fields belonging to the chosen destination so that, e.g., the + Slack ``url`` answer and a Webhook ``url`` answer never collide. + """ + dest = answers.get(OPTION_NOTIFY_DESTINATION, "") + args: dict[str, str] = {} + for field in _NOTIFY_FIELDS.get(dest, ()): + value = answers.get(field.option_id) + if value: + args[field.sdk_arg] = value + return args + + +def _notification_spec(config: TranslationConfiguration) -> dict[str, Any]: + """Builds the notification spec stamped onto the collapsed Copy task. + + ``args`` carries the resolved SDK config kwargs for the destination; email + ``addresses`` is split into a list, other fields pass through as strings. + """ + dest = config.notify_destination.value + spec: dict[str, Any] = { + "destination": dest, + "destination_name": config.notify_destination_name or f"flowx-{dest}", + "args": {}, + } + for field in _NOTIFY_FIELDS.get(dest, ()): + raw = config.notify_args.get(field.sdk_arg, "") + if not raw: + continue + spec["args"][field.sdk_arg] = ( + [item.strip() for item in raw.split(",") if item.strip()] if field.is_list else raw + ) + return spec + + +def _collapse_notify(pipeline: Pipeline, config: TranslationConfiguration) -> Pipeline: + """Collapse activity->notify groups: drop the notify Web activities and stamp a + notification spec (events + chosen destination) onto each upstream task. + + Works for any upstream activity type (Copy, Notebook, Lookup, …). Dependents of a dropped + notify activity are rewired to the upstream task so the DAG stays connected. The destination is + created (and its id resolved) later, at prepare time. + """ + groups = _find_notify_groups(pipeline.tasks) + if not groups: + return pipeline + spec_base = _notification_spec(config) + notify_to_upstream: dict[str, str] = {} + events_by_task: dict[str, list[str]] = {} + drop: set[str] = set() + for upstream_key, web_list in groups.items(): + events: set[str] = set() + for web, outcome in web_list: + events.add("on_failure" if outcome == "Failed" else "on_success") + drop.add(web.task_key) + notify_to_upstream[web.task_key] = upstream_key + chosen = config.notify_events + if chosen is NotifyEvents.ON_FAILURE: + events &= {"on_failure"} + elif chosen is NotifyEvents.ON_SUCCESS: + events &= {"on_success"} + events_by_task[upstream_key] = sorted(events) or ["on_failure"] + + new_tasks: list = [] + for task in pipeline.tasks: + if task.task_key in drop: + continue + deps = task.depends_on + if deps: + rewired: list = [] + seen: set[str] = set() + for dep in deps: + key = notify_to_upstream.get(dep.task_key, dep.task_key) + if key not in seen: + seen.add(key) + rewired.append(Dependency(task_key=key, outcome=dep.outcome)) + deps = rewired + if task.task_key in events_by_task: + spec = {**spec_base, "events": events_by_task[task.task_key]} + task = dataclasses.replace(task, depends_on=deps, notifications=spec) + else: + task = dataclasses.replace(task, depends_on=deps) + new_tasks.append(task) + return dataclasses.replace(pipeline, tasks=new_tasks) + + +def _build_copy_activity_paradigm_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None -) -> TranslationQuestion | None: - """Builds the SDP-vs-notebook question for Copy activities targeting Delta. +) -> TranslationOption | None: + """Builds the SDP-vs-notebook option for Copy activities targeting Delta. Args: pipeline: Translated pipeline IR. motifs: Detected motifs (unused; accepted for builder uniformity). answers: Answers already supplied for prior prompts. When the - user opted into Lakeflow Connect, this question only fires + user opted into Lakeflow Connect, this option only fires for Copy activities that are *not* LFC-eligible -- the paradigm choice is moot for Copies that will become managed LFC pipelines. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when no + The constructed :class:`TranslationOption`, or ``None`` when no Copy activity needs a paradigm choice. Copies whose source query is unfit for both LFC and SDP (joins, aggregates, etc.) are forced to PySpark notebook and excluded from the affected set. """ answers = answers or {} - going_to_lfc = answers.get(QUESTION_USE_LAKEFLOW_CONNECTORS) == UseLakeflowConnectors.LAKEFLOW_CONNECT.value + going_to_lfc = answers.get(OPTION_USE_LAKEFLOW_CONNECTORS) == UseLakeflowConnectors.LAKEFLOW_CONNECT.value affected = tuple( activity.task_key for activity in walk_activities(pipeline.tasks) @@ -334,8 +768,8 @@ def _build_copy_activity_paradigm_question( ) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_COPY_ACTIVITY_PARADIGM, + return TranslationOption( + option_id=OPTION_COPY_ACTIVITY_PARADIGM, prompt="How should Copy Data activities targeting Delta be implemented?", rationale=( "One or more Copy Data activities write to a Delta table. " @@ -343,12 +777,12 @@ def _build_copy_activity_paradigm_question( "a PySpark notebook stays closer to the original ADF activity shape." ), options=( - QuestionOption( + OptionChoice( value=CopyActivityParadigm.NOTEBOOK.value, label="PySpark notebook", description="Generates a notebook task that reads the source and writes Delta directly.", ), - QuestionOption( + OptionChoice( value=CopyActivityParadigm.SDP.value, label="Lakeflow Spark Declarative Pipeline", description="Emits an SDP pipeline resource with declarative table definitions.", @@ -359,10 +793,10 @@ def _build_copy_activity_paradigm_question( ) -def _build_non_databricks_task_compute_question( +def _build_non_databricks_task_compute_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None -) -> TranslationQuestion | None: - """Builds the serverless-vs-classic question for non-Databricks tasks. +) -> TranslationOption | None: + """Builds the serverless-vs-classic option for non-Databricks tasks. Args: pipeline: Translated pipeline IR. @@ -372,12 +806,12 @@ def _build_non_databricks_task_compute_question( because LFC pipelines always use serverless compute. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when + The constructed :class:`TranslationOption`, or ``None`` when every non-Databricks task in the pipeline is going to LFC (no compute choice to make). """ answers = answers or {} - going_to_lfc = answers.get(QUESTION_USE_LAKEFLOW_CONNECTORS) == UseLakeflowConnectors.LAKEFLOW_CONNECT.value + going_to_lfc = answers.get(OPTION_USE_LAKEFLOW_CONNECTORS) == UseLakeflowConnectors.LAKEFLOW_CONNECT.value affected = tuple( activity.task_key for activity in walk_activities(pipeline.tasks) @@ -385,8 +819,8 @@ def _build_non_databricks_task_compute_question( ) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_NON_DATABRICKS_TASK_COMPUTE, + return TranslationOption( + option_id=OPTION_NON_DATABRICKS_TASK_COMPUTE, prompt="What compute should the non-Databricks tasks use?", rationale=( "Tasks such as Copy Data, Web, Lookup, and Wait can run on serverless " @@ -394,12 +828,12 @@ def _build_non_databricks_task_compute_question( "tasks and a larger fixed-size cluster for Copy Data." ), options=( - QuestionOption( + OptionChoice( value=NonDatabricksTaskCompute.SERVERLESS.value, label="Serverless", description="Runs every non-Databricks task on serverless compute.", ), - QuestionOption( + OptionChoice( value=NonDatabricksTaskCompute.CLASSIC.value, label="Classic job_cluster", description="Provisions classic job_clusters sized per task type.", @@ -410,24 +844,24 @@ def _build_non_databricks_task_compute_question( ) -def _build_use_lakeflow_connectors_question( +def _build_use_lakeflow_connectors_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None -) -> TranslationQuestion | None: - """Builds the Lakeflow Connect question for eligible database ingestions. +) -> TranslationOption | None: + """Builds the Lakeflow Connect option for eligible database ingestions. Args: pipeline: Translated pipeline IR. motifs: Detected motifs, scanned for database-source ingestion patterns. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when no + The constructed :class:`TranslationOption`, or ``None`` when no Copy activity or motif qualifies for Lakeflow Connect. """ affected = _affected_task_keys_for_lakeflow_connect(pipeline, motifs) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_USE_LAKEFLOW_CONNECTORS, + return TranslationOption( + option_id=OPTION_USE_LAKEFLOW_CONNECTORS, prompt="Migrate eligible SQL Server, MySQL, and PostgreSQL ingestions to Lakeflow Connect?", rationale=( "One or more Copy Data activities ingest from SQL Server, MySQL, or " @@ -436,12 +870,12 @@ def _build_use_lakeflow_connectors_question( "the ADF-shaped activity intact." ), options=( - QuestionOption( + OptionChoice( value=UseLakeflowConnectors.EXISTING.value, label="Keep existing translation", description="Preserves the Copy Data activity as a notebook or SDP task.", ), - QuestionOption( + OptionChoice( value=UseLakeflowConnectors.LAKEFLOW_CONNECT.value, label="Use Lakeflow Connect", description="Replaces eligible ingestions with a managed Lakeflow Connect pipeline.", @@ -452,10 +886,10 @@ def _build_use_lakeflow_connectors_question( ) -def _build_lakeflow_connector_type_question( +def _build_lakeflow_connector_type_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None -) -> TranslationQuestion | None: - """Builds the CDC-vs-query connector question, suppressed when not actionable. +) -> TranslationOption | None: + """Builds the CDC-vs-query connector option, suppressed when not actionable. Args: pipeline: Translated pipeline IR. @@ -468,7 +902,7 @@ def _build_lakeflow_connector_type_question( (table-based reads → CDC because the query-based connector requires a cursor column; queries with a cursor → query-based because CDC requires direct table access). A pipeline-wide - preference between CDC and query-based therefore has no + configuration between CDC and query-based therefore has no actionable effect; the modifier picks the eligible connector per Copy. """ @@ -476,42 +910,42 @@ def _build_lakeflow_connector_type_question( return None -def _build_metadata_driven_consolidate_question( +def _build_metadata_driven_consolidate_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None, -) -> TranslationQuestion | None: - """Builds the consolidate-or-keep question for metadata-driven motifs. +) -> TranslationOption | None: + """Builds the consolidate-or-keep option for metadata-driven motifs. Args: pipeline: Translated pipeline IR. - motifs: Detected motifs; the question only surfaces when at + motifs: Detected motifs; the option only surfaces when at least one matches the metadata-driven bulk copy pattern. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when + The constructed :class:`TranslationOption`, or ``None`` when the pipeline contains no metadata-driven motif. """ affected = _metadata_driven_motif_task_keys(pipeline, motifs) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_METADATA_DRIVEN_CONSOLIDATE, + return TranslationOption( + option_id=OPTION_METADATA_DRIVEN_CONSOLIDATE, prompt="Consolidate the metadata-driven ingestions into one managed pipeline?", rationale=( "A Lookup feeds a ForEach that copies each row's table. Consolidating " "replaces this loop with a single Lakeflow Connect or Lakeflow Spark " "Declarative Pipeline whose objects list materialises each source as " - "its own streaming table. Keeping the loop preserves the existing " - "per-row Copy translation." + "its own streaming table. Keeping it emits a Databricks for-each task " + "that runs one Spark JDBC read per source table (no managed pipeline)." ), options=( - QuestionOption( + OptionChoice( value=MetadataDrivenConsolidate.KEEP.value, - label="Keep the per-row loop", - description="Preserves the ForEach + Copy translation as a motif scaffold.", + label="Keep the per-table loop", + description="Emits a for-each task running one Spark JDBC read per source table.", ), - QuestionOption( + OptionChoice( value=MetadataDrivenConsolidate.CONSOLIDATE.value, label="Consolidate into one pipeline", description="Emits one pipeline resource that ingests every source from the lookup.", @@ -522,26 +956,26 @@ def _build_metadata_driven_consolidate_question( ) -def _build_metadata_driven_access_question( +def _build_metadata_driven_access_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None, -) -> TranslationQuestion | None: - """Builds the metadata-source access question, gated on consolidate=yes. +) -> TranslationOption | None: + """Builds the metadata-source access option, gated on consolidate=yes. Args: pipeline: Translated pipeline IR. motifs: Detected motifs. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when no + The constructed :class:`TranslationOption`, or ``None`` when no metadata-driven motif applies. """ affected = _metadata_driven_motif_task_keys(pipeline, motifs) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_METADATA_DRIVEN_ACCESS, + return TranslationOption( + option_id=OPTION_METADATA_DRIVEN_ACCESS, prompt="Do you have access to query the metadata source and approve doing so?", rationale=( "Consolidating a metadata-driven ingestion requires materialising the " @@ -550,12 +984,12 @@ def _build_metadata_driven_access_question( "translation pass; answering no falls back to the per-row scaffold." ), options=( - QuestionOption( + OptionChoice( value=MetadataDrivenAccess.YES.value, label="Yes, query is allowed", description="The metadata source is reachable and approved for read during translation.", ), - QuestionOption( + OptionChoice( value=MetadataDrivenAccess.NO.value, label="No, skip materialising the lookup", description="Keeps the per-row motif scaffold without inlining the configuration.", @@ -563,30 +997,30 @@ def _build_metadata_driven_access_question( ), affected_task_keys=affected, default=MetadataDrivenAccess.NO.value, - conditions=((QUESTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value),), + conditions=((OPTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value),), ) -def _build_metadata_driven_size_question( +def _build_metadata_driven_size_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None, -) -> TranslationQuestion | None: - """Builds the t-shirt sizing question, gated on consolidate=yes. +) -> TranslationOption | None: + """Builds the t-shirt sizing option, gated on consolidate=yes. Args: pipeline: Translated pipeline IR. motifs: Detected motifs. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when no + The constructed :class:`TranslationOption`, or ``None`` when no metadata-driven motif applies. """ affected = _metadata_driven_motif_task_keys(pipeline, motifs) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_METADATA_DRIVEN_SIZE, + return TranslationOption( + option_id=OPTION_METADATA_DRIVEN_SIZE, prompt="Roughly how many configuration rows feed the metadata-driven ingestion?", rationale=( "The size determines whether the modifier inlines every lookup row into " @@ -595,17 +1029,17 @@ def _build_metadata_driven_size_question( "avoid generating an unwieldy pipeline definition." ), options=( - QuestionOption( + OptionChoice( value=MetadataDrivenSize.SMALL.value, label="S (under 50 rows)", description="Lookup feeds fewer than 50 ingestion targets.", ), - QuestionOption( + OptionChoice( value=MetadataDrivenSize.MEDIUM.value, label="M (under 250 rows)", description="Lookup feeds 50 to 249 ingestion targets.", ), - QuestionOption( + OptionChoice( value=MetadataDrivenSize.LARGE.value, label="L (250 or more rows)", description="Lookup feeds 250+ targets; skip inline consolidation.", @@ -613,30 +1047,30 @@ def _build_metadata_driven_size_question( ), affected_task_keys=affected, default=MetadataDrivenSize.LARGE.value, - conditions=((QUESTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value),), + conditions=((OPTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value),), ) -def _build_metadata_driven_lookup_tool_question( +def _build_metadata_driven_lookup_tool_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None, -) -> TranslationQuestion | None: - """Builds the agent-tool question for the lookup query, gated on size != L. +) -> TranslationOption | None: + """Builds the agent-tool option for the lookup query, gated on size != L. Args: pipeline: Translated pipeline IR. motifs: Detected motifs. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when no + The constructed :class:`TranslationOption`, or ``None`` when no metadata-driven motif applies. """ affected = _metadata_driven_motif_task_keys(pipeline, motifs) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_METADATA_DRIVEN_LOOKUP_TOOL, + return TranslationOption( + option_id=OPTION_METADATA_DRIVEN_LOOKUP_TOOL, prompt="Does the agent have a tool that can run the lookup query?", rationale=( "When the agent has a Genie skill, an MCP database tool, or a SQL " @@ -646,12 +1080,12 @@ def _build_metadata_driven_lookup_tool_question( "comma-separated string of values and the modifier ingests that." ), options=( - QuestionOption( + OptionChoice( value=MetadataDrivenLookupTool.HAVE.value, label="Yes, the agent can run the lookup", description="Agent executes the lookup query via its own tool.", ), - QuestionOption( + OptionChoice( value=MetadataDrivenLookupTool.NONE.value, label="No, ask the user for the values", description="Agent prompts the user for a CSV file or string of values.", @@ -660,22 +1094,22 @@ def _build_metadata_driven_lookup_tool_question( affected_task_keys=affected, default=MetadataDrivenLookupTool.NONE.value, conditions=( - (QUESTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value), - (QUESTION_METADATA_DRIVEN_ACCESS, MetadataDrivenAccess.YES.value), + (OPTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value), + (OPTION_METADATA_DRIVEN_ACCESS, MetadataDrivenAccess.YES.value), ), ) -def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuestion]: - """Builds one ``consolidate_motif:`` question per detected motif. +def _build_motif_consolidation_options(motifs: list) -> list[TranslationOption]: + """Builds one ``consolidate_motif:`` option per detected motif. Args: motifs: Detected :class:`~flowx.models.motifs.DetectedMotif` instances from :func:`flowx.motifs.detector.detect_motifs`. Returns: - A list of :class:`TranslationQuestion` instances, one per - detected motif. Each question uses a unique question_id of the + A list of :class:`TranslationOption` instances, one per + detected motif. Each option uses a unique option_id of the form ``consolidate_motif:`` so multiple distinct motif types in the same pipeline (e.g. ``rest_api_pagination`` *and* ``metadata_driven_bulk_copy``) each get their own prompt. @@ -688,7 +1122,7 @@ def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuesti the safer default. - When the same motif type is detected more than once in the same pipeline (rare in practice but possible) the builder - emits a single question covering all instances of that type. + emits a single option covering all instances of that type. Per-instance overrides can still be expressed by adding more fine-grained gating in :class:`MotifActivity`. - The ``affected_task_keys`` field lists the *underlying* ADF @@ -699,7 +1133,7 @@ def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuesti if not motifs: return [] seen: set[str] = set() - questions: list[TranslationQuestion] = [] + options: list[TranslationOption] = [] for motif in motifs: definition = motif.definition motif_id = definition.motif_id @@ -707,13 +1141,13 @@ def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuesti continue seen.add(motif_id) affected = tuple(motif.matched_activities) - question_id = f"{MOTIF_CONSOLIDATE_QUESTION_PREFIX}{motif_id}" + option_id = f"{MOTIF_CONSOLIDATE_OPTION_PREFIX}{motif_id}" confidence_suffix = "" if motif.confidence_notes: confidence_suffix = " Detector notes: " + " | ".join(motif.confidence_notes) - questions.append( - TranslationQuestion( - question_id=question_id, + options.append( + TranslationOption( + option_id=option_id, prompt=f"Consolidate the {definition.display_name!r} motif into a single task?", rationale=( f"{definition.description} " @@ -722,12 +1156,12 @@ def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuesti f"them with a single {definition.databricks_replacement!r} task." ), options=( - QuestionOption( + OptionChoice( value=MotifConsolidate.KEEP.value, label="Keep individual activities", description="Preserves the per-activity translation; no motif collapse.", ), - QuestionOption( + OptionChoice( value=MotifConsolidate.CONSOLIDATE.value, label="Consolidate into one task", description=f"Replaces matched activities with a {definition.databricks_replacement!r} task.", @@ -737,7 +1171,7 @@ def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuesti default=MotifConsolidate.KEEP.value, ) ) - return questions + return options def _metadata_driven_motif_task_keys( @@ -790,7 +1224,7 @@ def _copy_paradigm_decided_by_lfc(activity: CopyActivity, going_to_lfc: bool) -> Args: activity: Copy activity to inspect. - going_to_lfc: ``True`` when the caller answered the LFC question + going_to_lfc: ``True`` when the caller answered the LFC option with ``lakeflow_connect``. Returns: @@ -809,7 +1243,7 @@ def _task_compute_decided_by_lfc(activity, going_to_lfc: bool) -> bool: Args: activity: Activity to inspect. - going_to_lfc: ``True`` when the caller answered the LFC question + going_to_lfc: ``True`` when the caller answered the LFC option with ``lakeflow_connect``. Returns: @@ -862,13 +1296,13 @@ def _motif_task_keys_for_lakeflow_connect( ] -def _stamp_activity(activity: Activity, pipeline_preferences: TranslationPreferences) -> Activity: - """Stamps preference-derived decisions onto an activity. +def _stamp_activity(activity: Activity, pipeline_configuration: TranslationConfiguration) -> Activity: + """Stamps configuration-derived decisions onto an activity. Args: activity: Source activity from the IR. - pipeline_preferences: Pipeline-wide preferences; per-task overrides - apply via :meth:`TranslationPreferences.effective_for`. + pipeline_configuration: Pipeline-wide configuration; per-task overrides + apply via :meth:`TranslationConfiguration.effective_for`. Returns: A new activity instance with ``compute_mode``, ``target_format``, @@ -876,32 +1310,32 @@ def _stamp_activity(activity: Activity, pipeline_preferences: TranslationPrefere flow activities are recursed into so their inner bodies are stamped too. """ - activity_preferences = pipeline_preferences.effective_for(activity.task_key) + activity_configuration = pipeline_configuration.effective_for(activity.task_key) if isinstance(activity, ForEachActivity): - return _stamp_for_each_activity(activity, pipeline_preferences, activity_preferences) + return _stamp_for_each_activity(activity, pipeline_configuration, activity_configuration) if isinstance(activity, IfConditionActivity): - return _stamp_if_condition_activity(activity, pipeline_preferences, activity_preferences) + return _stamp_if_condition_activity(activity, pipeline_configuration, activity_configuration) if isinstance(activity, SwitchActivity): - return _stamp_switch_activity(activity, pipeline_preferences, activity_preferences) + return _stamp_switch_activity(activity, pipeline_configuration, activity_configuration) if isinstance(activity, CopyActivity): - return _stamp_copy_activity(activity, activity_preferences) + return _stamp_copy_activity(activity, activity_configuration) if isinstance(activity, MotifActivity): - return _stamp_motif_activity(activity, activity_preferences) - return dataclasses.replace(activity, compute_mode=_resolve_compute_mode(activity, activity_preferences)) + return _stamp_motif_activity(activity, activity_configuration) + return dataclasses.replace(activity, compute_mode=_resolve_compute_mode(activity, activity_configuration)) def _stamp_for_each_activity( activity: ForEachActivity, - pipeline_preferences: TranslationPreferences, - activity_preferences: TranslationPreferences, + pipeline_configuration: TranslationConfiguration, + activity_configuration: TranslationConfiguration, ) -> ForEachActivity: """Stamps a ForEach activity and recurses into its inner body. Args: activity: Source ForEach activity. - pipeline_preferences: Pipeline-wide preferences threaded into + pipeline_configuration: Pipeline-wide configuration threaded into inner activities so they re-resolve their own overrides. - activity_preferences: Preferences after per-task overrides for + activity_configuration: Configuration after per-task overrides for *activity*. Returns: @@ -909,23 +1343,23 @@ def _stamp_for_each_activity( """ return dataclasses.replace( activity, - inner_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.inner_activities], - compute_mode=_resolve_compute_mode(activity, activity_preferences), + inner_activities=[_stamp_activity(inner, pipeline_configuration) for inner in activity.inner_activities], + compute_mode=_resolve_compute_mode(activity, activity_configuration), ) def _stamp_if_condition_activity( activity: IfConditionActivity, - pipeline_preferences: TranslationPreferences, - activity_preferences: TranslationPreferences, + pipeline_configuration: TranslationConfiguration, + activity_configuration: TranslationConfiguration, ) -> IfConditionActivity: """Stamps an IfCondition activity and recurses into both branches. Args: activity: Source IfCondition activity. - pipeline_preferences: Pipeline-wide preferences threaded into + pipeline_configuration: Pipeline-wide configuration threaded into inner activities so they re-resolve their own overrides. - activity_preferences: Preferences after per-task overrides for + activity_configuration: Configuration after per-task overrides for *activity*. Returns: @@ -933,24 +1367,24 @@ def _stamp_if_condition_activity( """ return dataclasses.replace( activity, - if_true_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.if_true_activities], - if_false_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.if_false_activities], - compute_mode=_resolve_compute_mode(activity, activity_preferences), + if_true_activities=[_stamp_activity(inner, pipeline_configuration) for inner in activity.if_true_activities], + if_false_activities=[_stamp_activity(inner, pipeline_configuration) for inner in activity.if_false_activities], + compute_mode=_resolve_compute_mode(activity, activity_configuration), ) def _stamp_switch_activity( activity: SwitchActivity, - pipeline_preferences: TranslationPreferences, - activity_preferences: TranslationPreferences, + pipeline_configuration: TranslationConfiguration, + activity_configuration: TranslationConfiguration, ) -> SwitchActivity: """Stamps a Switch activity and recurses into every case and the default. Args: activity: Source Switch activity. - pipeline_preferences: Pipeline-wide preferences threaded into + pipeline_configuration: Pipeline-wide configuration threaded into inner activities so they re-resolve their own overrides. - activity_preferences: Preferences after per-task overrides for + activity_configuration: Configuration after per-task overrides for *activity*. Returns: @@ -959,46 +1393,46 @@ def _stamp_switch_activity( stamped_cases = [ SwitchCase( value=case.value, - activities=[_stamp_activity(inner, pipeline_preferences) for inner in case.activities], + activities=[_stamp_activity(inner, pipeline_configuration) for inner in case.activities], ) for case in activity.cases ] return dataclasses.replace( activity, cases=stamped_cases, - default_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.default_activities], - compute_mode=_resolve_compute_mode(activity, activity_preferences), + default_activities=[_stamp_activity(inner, pipeline_configuration) for inner in activity.default_activities], + compute_mode=_resolve_compute_mode(activity, activity_configuration), ) def _stamp_copy_activity( activity: CopyActivity, - activity_preferences: TranslationPreferences, + activity_configuration: TranslationConfiguration, ) -> CopyActivity: """Stamps a Copy activity with paradigm, compute, and Lakeflow Connect flags. Args: activity: Source Copy activity. - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. Returns: A new :class:`CopyActivity` whose ``target_format``, ``compute_mode``, and ``use_lakeflow_connector`` fields reflect the user's choices. Copies whose query is unfit for LFC and SDP (joins, aggregates, etc.) are forced to the notebook - paradigm regardless of preference because the alternative + paradigm regardless of configuration because the alternative paradigms cannot represent arbitrary SQL. """ - user_picked_lfc = activity_preferences.use_lakeflow_connectors is UseLakeflowConnectors.LAKEFLOW_CONNECT + user_picked_lfc = activity_configuration.use_lakeflow_connectors is UseLakeflowConnectors.LAKEFLOW_CONNECT use_lakeflow_connector = user_picked_lfc and copy_eligible_for_any_lfc_connector(activity) - paradigm = _resolve_paradigm(activity, activity_preferences, use_lakeflow_connector) + paradigm = _resolve_paradigm(activity, activity_configuration, use_lakeflow_connector) connector_type = ( - _resolve_lakeflow_connector_type(activity, activity_preferences) if use_lakeflow_connector else None + _resolve_lakeflow_connector_type(activity, activity_configuration) if use_lakeflow_connector else None ) return dataclasses.replace( activity, target_format=paradigm.value, - compute_mode=_resolve_compute_mode(activity, activity_preferences), + compute_mode=_resolve_compute_mode(activity, activity_configuration), use_lakeflow_connector=use_lakeflow_connector, lakeflow_connector_type=connector_type, ) @@ -1006,14 +1440,14 @@ def _stamp_copy_activity( def _resolve_paradigm( activity: CopyActivity, - activity_preferences: TranslationPreferences, + activity_configuration: TranslationConfiguration, use_lakeflow_connector: bool, ) -> CopyActivityParadigm: """Resolves the paradigm (notebook vs SDP) for a Copy that won't go to LFC. Args: activity: Source Copy activity. - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. use_lakeflow_connector: ``True`` when the modifier already routed the Copy to a managed LFC pipeline; the paradigm is informational in that case. @@ -1029,20 +1463,20 @@ def _resolve_paradigm( return CopyActivityParadigm.NOTEBOOK if not copy_targets_delta(activity): return CopyActivityParadigm.NOTEBOOK - return activity_preferences.copy_activity_paradigm + return activity_configuration.copy_activity_paradigm -def _resolve_lakeflow_connector_type(activity: CopyActivity, activity_preferences: TranslationPreferences) -> str: +def _resolve_lakeflow_connector_type(activity: CopyActivity, activity_configuration: TranslationConfiguration) -> str: """Resolves which Lakeflow Connect connector to use for an eligible Copy. Args: activity: Source Copy activity (already known to be LFC-eligible). - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. Returns: Always the connector flavour the Copy is actually eligible for. Query-based eligibility (parseable query + cursor column) wins - over the user's CDC preference because no cursor candidate + over the user's CDC configuration because no cursor candidate exists for a CDC connector to use on a query-only Copy. Table-based Copies route to CDC because the query-based connector requires a cursor column and there is none. @@ -1054,13 +1488,13 @@ def _resolve_lakeflow_connector_type(activity: CopyActivity, activity_preference def _stamp_motif_activity( activity: MotifActivity, - activity_preferences: TranslationPreferences, + activity_configuration: TranslationConfiguration, ) -> MotifActivity: """Stamps a Motif activity, swapping in Lakeflow Connect when eligible. Args: activity: Source motif activity. - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. Returns: A new :class:`MotifActivity` whose ``databricks_replacement`` is @@ -1071,7 +1505,7 @@ def _stamp_motif_activity( consolidation, granted access, and the size bucket is S or M. """ qualifies_for_lakeflow_connect = ( - activity_preferences.use_lakeflow_connectors is UseLakeflowConnectors.LAKEFLOW_CONNECT + activity_configuration.use_lakeflow_connectors is UseLakeflowConnectors.LAKEFLOW_CONNECT and activity.source_type_hint == DATABASE_SOURCE_TYPE_HINT ) replacement = LAKEFLOW_CONNECT_REPLACEMENT if qualifies_for_lakeflow_connect else activity.databricks_replacement @@ -1080,25 +1514,25 @@ def _stamp_motif_activity( if qualifies_for_lakeflow_connect else activity.notebook_template ) - consolidate = _should_consolidate_metadata_driven(activity, activity_preferences) + consolidate = _should_consolidate_metadata_driven(activity, activity_configuration) return dataclasses.replace( activity, databricks_replacement=replacement, notebook_template=notebook_template, - compute_mode=_resolve_compute_mode(activity, activity_preferences), + compute_mode=_resolve_compute_mode(activity, activity_configuration), consolidate_metadata_driven=consolidate, ) def _should_consolidate_metadata_driven( activity: MotifActivity, - activity_preferences: TranslationPreferences, + activity_configuration: TranslationConfiguration, ) -> bool: """Returns True when the modifier should consolidate a metadata-driven motif. Args: activity: Source motif activity. - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. Returns: ``True`` when the motif matches the metadata-driven bulk-copy @@ -1109,19 +1543,19 @@ def _should_consolidate_metadata_driven( """ if activity.motif_id != "metadata_driven_bulk_copy": return False - if activity_preferences.metadata_driven_consolidate is not MetadataDrivenConsolidate.CONSOLIDATE: + if activity_configuration.metadata_driven_consolidate is not MetadataDrivenConsolidate.CONSOLIDATE: return False - if activity_preferences.metadata_driven_access is not MetadataDrivenAccess.YES: + if activity_configuration.metadata_driven_access is not MetadataDrivenAccess.YES: return False - return activity_preferences.metadata_driven_size is not MetadataDrivenSize.LARGE + return activity_configuration.metadata_driven_size is not MetadataDrivenSize.LARGE -def _resolve_compute_mode(activity: Activity, activity_preferences: TranslationPreferences) -> str: +def _resolve_compute_mode(activity: Activity, activity_configuration: TranslationConfiguration) -> str: """Resolves the compute mode an activity should run on. Args: activity: Source activity. - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. Returns: One of :data:`COMPUTE_MODE_SERVERLESS`, @@ -1135,7 +1569,7 @@ def _resolve_compute_mode(activity: Activity, activity_preferences: TranslationP """ if not is_non_databricks_task(activity): return COMPUTE_MODE_INHERIT - if activity_preferences.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS: + if activity_configuration.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS: return COMPUTE_MODE_SERVERLESS if isinstance(activity, CopyActivity): return COMPUTE_MODE_CLASSIC_MULTI_NODE diff --git a/src/orchestra/adapter/predicates.py b/src/flowx/adapter/predicates.py similarity index 99% rename from src/orchestra/adapter/predicates.py rename to src/flowx/adapter/predicates.py index aa211e9..7c69bac 100644 --- a/src/orchestra/adapter/predicates.py +++ b/src/flowx/adapter/predicates.py @@ -182,7 +182,7 @@ def copy_query_unfit_for_lfc(activity: CopyActivity) -> bool: contains JOIN, GROUP BY, aggregates, UNION, window functions, subqueries, or column expressions). Such Copies should be translated through PySpark notebooks regardless of paradigm - preference because LFC's query-based connector and SDP's + configuration because LFC's query-based connector and SDP's declarative table form both reject the query. """ if not copy_has_source_query(activity): diff --git a/src/flowx/adapter/session.py b/src/flowx/adapter/session.py new file mode 100644 index 0000000..2b7509c --- /dev/null +++ b/src/flowx/adapter/session.py @@ -0,0 +1,486 @@ +"""Agent adapter that drives the ask-validate-resume loop. + +:class:`TranslationSession` is the entry point an agent uses to +translate tool-call arguments into validated configuration. When the IR +raises options the agent cannot answer from context alone, the +session surfaces them as structured :class:`TranslationOption` +objects (and, via :exc:`TranslationInputRequired`, as exceptions) so +the agent can route them back to the user. The pipeline modifier is +invoked only once every option has an answer. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from flowx.adapter.constants import ( + INPUT_ADF_RESOURCE_URL, + INPUT_ADF_SOURCE_PATH, + INPUT_BUNDLE_NAME, + INPUT_CATALOG, + INPUT_DATABRICKS_PROFILE, + INPUT_INSTALL_DASHBOARD, + INPUT_INVENTORY_PATH, + INPUT_OUTPUT_BUNDLE_PATH, + INPUT_OUTPUT_DIR, + INPUT_RESULTS_TABLE, + INPUT_RESULTS_WAREHOUSE, + INPUT_SCHEMA, + INPUT_TRANSLATION_REPORT_PATH, + MOTIF_CONSOLIDATE_OPTION_PREFIX, + PHASE_CONVERT, + PHASE_DISCOVER, + PHASE_PACKAGE, +) +from flowx.adapter.models import ( + DEFAULT_CONFIGURATION, + CopyActivityParadigm, + LakeflowConnectorType, + MetadataDrivenAccess, + MetadataDrivenConsolidate, + MetadataDrivenLookupTool, + MetadataDrivenSize, + MigrationInputOption, + MotifConsolidate, + NonDatabricksTaskCompute, + PendingMigrationInputs, + PendingOptions, + TranslationConfiguration, + TranslationOption, + UseLakeflowConnectors, +) +from flowx.adapter.operations import ( + apply_configuration, + gather_options, + validate_answer, +) +from flowx.models.ir import Pipeline +from flowx.models.motifs import DetectedMotif + + +class TranslationInputRequired(Exception): + """Raised by :meth:`TranslationSession.run` when answers are still missing. + + Attributes: + pending: The outstanding options the agent should route to the + user before retrying :meth:`TranslationSession.run`. + """ + + def __init__(self, pending: PendingOptions) -> None: + """Stores the pending options on the exception. + + Args: + pending: Outstanding options surfaced by the session. + """ + super().__init__( + f"{len(pending.options)} translation option(s) require user input for pipeline {pending.pipeline_name!r}" + ) + self.pending = pending + + +@dataclass(slots=True, kw_only=True) +class TranslationSession: + """Coordinates the ask-validate-resume loop for one translated pipeline. + + A session is single-use: the caller drives it by either polling via + :meth:`pending` and :meth:`answer`, or calling :meth:`run` and + handling :exc:`TranslationInputRequired`. When every option is + answered, :meth:`run` (or :meth:`resume`) returns the + configuration-stamped pipeline. + + Attributes: + pipeline: Translated pipeline IR after motif collapsing. + motifs: Detected motifs for the pipeline. Optional; only used to + decide whether the Lakeflow Connect option applies. + defaults: Baseline configuration applied when the caller skips a + option. Per-task overrides on this object are preserved + verbatim when :meth:`build_configuration` composes the final + snapshot. + """ + + pipeline: Pipeline + motifs: list[DetectedMotif] = field(default_factory=list) + defaults: TranslationConfiguration = DEFAULT_CONFIGURATION + _answers: dict[str, str] = field(default_factory=dict) + + def pending(self) -> PendingOptions: + """Returns the options still awaiting an answer. + + Returns: + A :class:`PendingOptions` instance containing only the + options whose preconditions are met by the IR and whose + IDs are not yet in the answer set. + """ + return gather_options( + self.pipeline, + self.motifs, + answers=self._answers, + ) + + def answer(self, option_id: str, value: str) -> None: + """Validates and records a single answer. + + Args: + option_id: Stable option identifier from + :class:`TranslationOption`. + value: Caller-supplied answer string. + + Raises: + ValueError: When *option_id* is unknown or *value* is not + in the allowed set for the option. + """ + self._answers[option_id] = validate_answer(option_id, value) + + def answer_many(self, answers: dict[str, str]) -> None: + """Validates and records multiple answers atomically. + + Args: + answers: Mapping of option_id to the caller-supplied answer. + + Raises: + ValueError: When any pair fails validation. No answers from + the batch are recorded when the call raises. + """ + validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} + self._answers.update(validated) + + def find_option(self, option_id: str) -> TranslationOption | None: + """Looks up a pending option by its identifier. + + Args: + option_id: Stable option identifier. + + Returns: + The matching :class:`TranslationOption` if it is still + pending, otherwise ``None``. + """ + return next( + (option for option in self.pending().options if option.option_id == option_id), + None, + ) + + def build_configuration(self) -> TranslationConfiguration: + """Composes the validated configuration snapshot from collected answers. + + Returns: + A :class:`TranslationConfiguration` where every answered field + takes the caller-supplied value and every unanswered field + falls back to the corresponding value on ``defaults``. + """ + return TranslationConfiguration( + copy_activity_paradigm=CopyActivityParadigm( + self._answers.get("copy_activity_paradigm", self.defaults.copy_activity_paradigm) + ), + non_databricks_task_compute=NonDatabricksTaskCompute( + self._answers.get("non_databricks_task_compute", self.defaults.non_databricks_task_compute) + ), + use_lakeflow_connectors=UseLakeflowConnectors( + self._answers.get("use_lakeflow_connectors", self.defaults.use_lakeflow_connectors) + ), + lakeflow_connector_type=LakeflowConnectorType( + self._answers.get("lakeflow_connector_type", self.defaults.lakeflow_connector_type) + ), + metadata_driven_consolidate=MetadataDrivenConsolidate( + self._answers.get("metadata_driven_consolidate", self.defaults.metadata_driven_consolidate) + ), + metadata_driven_access=MetadataDrivenAccess( + self._answers.get("metadata_driven_access", self.defaults.metadata_driven_access) + ), + metadata_driven_size=MetadataDrivenSize( + self._answers.get("metadata_driven_size", self.defaults.metadata_driven_size) + ), + metadata_driven_lookup_tool=MetadataDrivenLookupTool( + self._answers.get("metadata_driven_lookup_tool", self.defaults.metadata_driven_lookup_tool) + ), + motif_consolidations=self._collect_motif_consolidations(), + per_task=self.defaults.per_task, + ) + + def resume(self) -> Pipeline: + """Returns the configuration-stamped pipeline IR. + + Returns: + A new :class:`Pipeline` produced by applying the composed + configuration to ``self.pipeline``. The input pipeline is not + mutated. + """ + return apply_configuration(self.pipeline, self.build_configuration()) + + def run(self) -> Pipeline: + """Returns the modified pipeline, raising when input is still required. + + Returns: + The configuration-stamped pipeline IR when every applicable + option has an answer. + + Raises: + TranslationInputRequired: When one or more options are + still outstanding. The exception carries the pending + options so the agent can route them to the user. + """ + pending = self.pending() + if pending.options: + raise TranslationInputRequired(pending) + return self.resume() + + def _collect_motif_consolidations(self) -> dict[str, MotifConsolidate]: + """Returns the per-motif consolidation answers gathered so far. + + Returns: + Dict mapping ``motif_id`` to the user's :class:`MotifConsolidate` answer. Motifs the user + did not answer fall back to ``self.defaults`` (default :data:`MotifConsolidate.KEEP`). + """ + consolidations: dict[str, MotifConsolidate] = dict(self.defaults.motif_consolidations) + for option_id, answer in self._answers.items(): + if not option_id.startswith(MOTIF_CONSOLIDATE_OPTION_PREFIX): + continue + motif_id = option_id[len(MOTIF_CONSOLIDATE_OPTION_PREFIX) :] + consolidations[motif_id] = MotifConsolidate(answer) + return consolidations + + +_DISCOVER_OPTIONS: tuple[MigrationInputOption, ...] = ( + MigrationInputOption( + option_id=INPUT_ADF_SOURCE_PATH, + prompt="Where are the ADF JSON exports?", + description=( + "Unity Catalog volume path (``/Volumes///``) " + "or a local directory containing the ADF ARM/JSON export." + ), + required=True, + ), + MigrationInputOption( + option_id=INPUT_ADF_RESOURCE_URL, + prompt="ADF resource URL?", + description=( + "Azure portal URL of the source Data Factory. Captured for " + "traceability and surfaced in the generated bundle README; " + "leave blank when the source is exported from a local copy." + ), + default="", + required=False, + ), + MigrationInputOption( + option_id=INPUT_OUTPUT_DIR, + prompt="Which migration output directory should flowx use?", + description=( + "Single shared migration directory used by every phase (default ``./flowx_output``). " + "Discover writes ``metadata/inventory.json``, ``metadata/profile_report.csv``, and the " + "verbatim ``metadata/.arm.json`` into it." + ), + default="./flowx_output", + required=False, + ), +) + +_CONVERT_OPTIONS: tuple[MigrationInputOption, ...] = ( + MigrationInputOption( + option_id=INPUT_INVENTORY_PATH, + prompt="Path to the inventory.json from the discover phase?", + description="Inventory produced by the discover phase (under the shared migration dir's metadata/).", + default="./flowx_output/metadata/inventory.json", + required=False, + ), + MigrationInputOption( + option_id=INPUT_ADF_SOURCE_PATH, + prompt="Path to the ADF JSON exports?", + description="Same source directory the discover phase consumed; needed for cross-references.", + required=True, + ), + MigrationInputOption( + option_id=INPUT_OUTPUT_DIR, + prompt="Which migration output directory should flowx use?", + description=( + "The same shared migration directory the discover phase used (default ``./flowx_output``). " + "Convert writes its transient report and IR to the directory's ``.work/`` subfolder." + ), + default="./flowx_output", + required=False, + ), +) + +_PACKAGE_OPTIONS: tuple[MigrationInputOption, ...] = ( + MigrationInputOption( + option_id=INPUT_TRANSLATION_REPORT_PATH, + prompt="Path to the translation report?", + description=( + "Configuration-stamped report from `python -m flowx.adapter modify`, " + "or the raw convert-phase report when no configuration were applied." + ), + default="./flowx_output/.work/translation_report.stamped.json", + required=False, + ), + MigrationInputOption( + option_id=INPUT_OUTPUT_BUNDLE_PATH, + prompt="Which migration output directory should flowx use?", + description=( + "The same shared migration directory used by discover/convert (default ``./flowx_output``). " + "Package writes the DAB bundle at its top level and prunes the transient ``.work/`` folder." + ), + default="./flowx_output", + required=False, + ), + MigrationInputOption( + option_id=INPUT_CATALOG, + prompt="Target Unity Catalog catalog?", + description="Default ``catalog`` bundle variable used by emitted notebooks and pipelines.", + default="main", + required=False, + ), + MigrationInputOption( + option_id=INPUT_SCHEMA, + prompt="Target Unity Catalog schema?", + description="Default ``schema`` bundle variable used by emitted notebooks and pipelines.", + default="default", + required=False, + ), + MigrationInputOption( + option_id=INPUT_BUNDLE_NAME, + prompt="Bundle name override?", + description="Defaults to the first translated pipeline's resource key when blank.", + default="", + required=False, + ), + MigrationInputOption( + option_id=INPUT_DATABRICKS_PROFILE, + prompt="Databricks CLI profile?", + description=( + "Profile used to download workspace-resident notebooks during the " + "package phase. Leave blank to use the default profile from " + "``~/.databrickscfg`` or the active ``DATABRICKS_*`` env vars." + ), + default="", + required=False, + ), + MigrationInputOption( + option_id=INPUT_RESULTS_TABLE, + prompt="Record migration coverage to a Unity Catalog table? If so, the table (catalog.schema.table)?", + description=( + "Optional. When set, the package phase writes one coverage row per pipeline to this UC " + "table, stamped with a UUID run_id, run_date (CURRENT_TIMESTAMP()), and run_by " + "(CURRENT_USER()). Leave blank to skip. Requires workspace auth (Genie Code / a profile)." + ), + default="", + required=False, + ), + MigrationInputOption( + option_id=INPUT_RESULTS_WAREHOUSE, + prompt="SQL warehouse id for writing the results table / backing the dashboard?", + description=( + "Optional. Warehouse used to run the CREATE/INSERT and back the dashboard. Leave blank to " + "auto-detect (prefers a running, serverless warehouse)." + ), + default="", + required=False, + ), + MigrationInputOption( + option_id=INPUT_INSTALL_DASHBOARD, + prompt="Install a published AI/BI coverage dashboard over the results table? (yes/no)", + description=( + "Optional. When 'yes' (and a results table is set), installs and publishes a Lakeview " + "dashboard that visualizes migration coverage from the table." + ), + default="no", + required=False, + ), +) + +_OPTIONS_BY_PHASE: dict[str, tuple[MigrationInputOption, ...]] = { + PHASE_DISCOVER: _DISCOVER_OPTIONS, + PHASE_CONVERT: _CONVERT_OPTIONS, + PHASE_PACKAGE: _PACKAGE_OPTIONS, +} + + +class UnknownMigrationPhaseError(ValueError): + """Raised when a MigrationInputSession is constructed with an unrecognised phase.""" + + +@dataclass(slots=True, kw_only=True) +class MigrationInputSession: + """Coordinates the free-text input prompts at the top of an flowx phase. + + A session is single-use: the caller drives it by polling + :meth:`pending` and recording answers via :meth:`answer`, then reads + them out with :meth:`collected` once every required input has a + value. The session is intentionally distinct from + :class:`TranslationSession` because the inputs it gathers are + free-text paths and identifiers rather than enum-backed choices. + + Attributes: + phase: One of ``"discover"``, ``"convert"``, ``"package"``. + """ + + phase: str + _answers: dict[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Validates that *phase* is one of the supported migration phases. + + Raises: + UnknownMigrationPhaseError: When *phase* is not registered in + :data:`_OPTIONS_BY_PHASE`. + """ + if self.phase not in _OPTIONS_BY_PHASE: + raise UnknownMigrationPhaseError( + f"Unknown migration phase {self.phase!r}; expected one of {sorted(_OPTIONS_BY_PHASE)}" + ) + + def pending(self) -> PendingMigrationInputs: + """Returns the input options still awaiting an answer. + + Returns: + A :class:`PendingMigrationInputs` with the unanswered + options for ``self.phase`` in registration order. + """ + options = [option for option in _OPTIONS_BY_PHASE[self.phase] if option.option_id not in self._answers] + return PendingMigrationInputs(phase=self.phase, options=options) + + def answer(self, option_id: str, value: str) -> None: + """Records an answer to one input option. + + Args: + option_id: Stable identifier of the option. + value: Caller-supplied string value. + + Raises: + ValueError: When *option_id* is not a known input for the + session's phase. + """ + if not any(option.option_id == option_id for option in _OPTIONS_BY_PHASE[self.phase]): + raise ValueError(f"Unknown input option {option_id!r} for phase {self.phase!r}") + self._answers[option_id] = value + + def answer_many(self, answers: dict[str, str]) -> None: + """Records multiple input answers atomically. + + Args: + answers: Mapping of option_id to the caller-supplied value. + + Raises: + ValueError: When any pair references an unknown option. + No answers are recorded when the call raises. + """ + known_ids = {option.option_id for option in _OPTIONS_BY_PHASE[self.phase]} + unknown = set(answers) - known_ids + if unknown: + raise ValueError(f"Unknown input options for phase {self.phase!r}: {sorted(unknown)}") + self._answers.update(answers) + + def collected(self) -> dict[str, str]: + """Returns the collected answers merged with each option's default. + + Returns: + A dict keyed by option_id covering every option for the + phase: caller-supplied answers take precedence; otherwise + the option's ``default`` value (which may be the empty + string) is used. Required options whose answers are + missing are omitted so the caller can detect them. + """ + collected: dict[str, str] = {} + for option in _OPTIONS_BY_PHASE[self.phase]: + if option.option_id in self._answers: + collected[option.option_id] = self._answers[option.option_id] + elif option.default is not None: + collected[option.option_id] = option.default + return collected diff --git a/src/orchestra/bundler/__init__.py b/src/flowx/bundler/__init__.py similarity index 100% rename from src/orchestra/bundler/__init__.py rename to src/flowx/bundler/__init__.py diff --git a/src/orchestra/bundler/constants.py b/src/flowx/bundler/constants.py similarity index 100% rename from src/orchestra/bundler/constants.py rename to src/flowx/bundler/constants.py diff --git a/src/orchestra/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py similarity index 83% rename from src/orchestra/bundler/dab_writer.py rename to src/flowx/bundler/dab_writer.py index 660d020..bb99399 100644 --- a/src/orchestra/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -66,15 +66,13 @@ class _BundleYamlDumper(yaml.SafeDumper): # Module-level warnings collector — reset per write_bundle call. _bundle_warnings: list[str] = [] -# Cross-bundle ExecutePipeline refs seen while translating: variable_name → -# target pipeline name. Reset per write_bundle call and surfaced via the -# bundle's ``variables`` block + SETUP.md. +# Cross-bundle ExecutePipeline refs seen while translating (variable_name -> target pipeline). Reset per +# write_bundle call and surfaced via the bundle's ``variables`` block + SETUP.md. _cross_bundle_variables: dict[str, str] = {} -# C-43 (CF5-001 / CF5-002): condition_task operands the dangling-ref safety -# net had to blank. Each entry is {task_key, field, original_ref}. Reset -# per write_bundle call and surfaced as a SETUP.md section so a neutralised -# branch predicate (always-true) is never silent. +# C-43 (CF5-001 / CF5-002): condition_task operands the dangling-ref safety net blanked +# ({task_key, field, original_ref}). Reset per write_bundle call and surfaced in SETUP.md so a +# neutralised (always-true) branch predicate is never silent. _neutralized_conditions: list[dict[str, str]] = [] _WIDGET_REFERENCE = re.compile(r"""dbutils\.widgets\.get\(\s*["']([^"']+)["']\s*\)""") @@ -99,9 +97,8 @@ def write_bundle( Returns: List of absolute paths to all created files. """ - # Reset module-level accumulators so successive ``write_bundle`` calls - # (CLI loops, library users, integration tests) don't carry warnings or - # cross-bundle variables from one bundle into the next. + # Reset module-level accumulators so successive write_bundle calls (CLI loops, tests) don't carry + # warnings or cross-bundle variables from one bundle into the next. _bundle_warnings.clear() _cross_bundle_variables.clear() _neutralized_conditions.clear() @@ -113,11 +110,8 @@ def write_bundle( resource_key = normalize_task_key(workflow.name) effective_name = bundle_name or resource_key - # Bind clusters across the parent workflow and any inner workflows up - # front so we can decide whether the bundle needs cluster-related - # tunables in ``databricks.yml`` at all. Binding is idempotent, so the - # subsequent ``_build_job_resource`` calls re-checking the same tasks is - # harmless. + # Bind clusters across the parent and inner workflows up front to decide whether databricks.yml needs + # cluster tunables at all. Binding is idempotent, so _build_job_resource re-checking these is harmless. _bind_cluster_to_notebook_tasks(workflow.tasks) for inner in workflow.inner_workflows: _bind_cluster_to_notebook_tasks(inner.tasks) @@ -125,11 +119,8 @@ def write_bundle( _any_task_uses_classic_cluster(inner.tasks) for inner in workflow.inner_workflows ) - # 1. Write databricks.yml. When at least one task runs on classic - # compute, defaults for spark_version / node_type_id come from the - # ADF linked service configs on the tasks so the emitted cluster - # matches the source-of-truth runtime. When every task is - # serverless, those variables are omitted entirely. + # 1. Write databricks.yml. When any task runs on classic compute, spark_version / node_type_id + # defaults come from the ADF linked-service configs; when every task is serverless, they're omitted. databricks_yml_path = output_dir / "databricks.yml" inferred_spark_version, inferred_node_type_id = _infer_bundle_cluster_defaults(workflow) databricks_yml_dict = _build_databricks_yml( @@ -152,10 +143,8 @@ def write_bundle( ) created_files.append(databricks_yml_path.resolve()) - # 2. Write job resource YAML. Strip broken base_parameters from - # existing-notebook tasks before serialising — these are surfaced - # in SETUP.md (§Existing-notebook parameter handling) further down - # and shouldn't ship in the YAML as malformed widget values. + # 2. Write job resource YAML. Strip broken base_parameters from existing-notebook tasks first — + # they're surfaced in SETUP.md further down and shouldn't ship in the YAML as malformed values. manual_parameters: list[ManualParameter] = _extract_manual_parameters_from_existing_notebook_tasks(workflow.tasks) for inner in workflow.inner_workflows: manual_parameters.extend(_extract_manual_parameters_from_existing_notebook_tasks(inner.tasks)) @@ -172,9 +161,8 @@ def write_bundle( ) created_files.append(job_yml_path.resolve()) - # Write inner workflows as additional resource files. Inner tasks reuse - # notebooks that live in the parent workflow's notebooks list, so pass - # those in so the inner job's widget auto-augmentation can see them. + # Write inner workflows as additional resource files. Inner tasks reuse notebooks from the parent's + # list, so pass those in for the inner job's widget auto-augmentation. for inner in workflow.inner_workflows: inner_key = normalize_task_key(inner.name) inner_yml_path = resources_dir / f"{inner_key}.yml" @@ -191,10 +179,8 @@ def write_bundle( ) created_files.append(inner_yml_path.resolve()) - # 2b. Write Lakeflow pipeline resources (Lakeflow Connect ingestion - # definitions emitted by the Copy preparer's LFC branch). Each - # resource lives in its own YAML so the bundle parser merges them - # alongside the job resources via the ``include`` glob. + # 2b. Write Lakeflow pipeline resources (Lakeflow Connect ingestion defs from the Copy preparer's LFC + # branch). Each lives in its own YAML so the bundle parser merges them via the ``include`` glob. pipelines_dir = resources_dir / "pipelines" for resource in _collect_pipeline_resources(workflow): pipelines_dir.mkdir(parents=True, exist_ok=True) @@ -216,9 +202,8 @@ def write_bundle( if workflow.notebooks: created_files.extend(write_notebooks(workflow.notebooks, src_dir)) - # 4. Generate and write setup notebooks (create-scope, create-volume, etc.). - # These are the *executable* provisioning artifacts; SETUP.md (below) - # is the human-readable companion. + # 4. Generate and write setup notebooks (create-scope, create-volume, etc.) — the executable + # provisioning artifacts; SETUP.md (below) is the human-readable companion. setup_notebooks: list[DabNotebook] = generate_setup_tasks( secrets=workflow.secrets, setup_tasks=workflow.setup_tasks, @@ -241,10 +226,8 @@ def write_bundle( if inner_setup: created_files.extend(write_notebooks(inner_setup, src_dir)) - # 5. Build SETUP.md — a root-level, human-readable summary of every - # external step the user must take before ``bundle run``. This is - # additive to the setup/ notebooks above; the setup notebooks are - # the executable path, SETUP.md is the checklist. + # 5. Build SETUP.md — a root-level, human-readable summary of every external step needed before + # ``bundle run``. Additive to the setup/ notebooks above (those are the executable path). all_notebooks = list(workflow.notebooks) for inner in workflow.inner_workflows: all_notebooks.extend(inner.notebooks) @@ -255,32 +238,37 @@ def write_bundle( for inner in workflow.inner_workflows: parameter_approximations.extend(inner.parameter_approximations) known_bundle_jobs = {resource_key} | {normalize_task_key(inner.name) for inner in workflow.inner_workflows} - # ``manual_parameters`` was collected above (before YAML emission) so - # the broken values are also stripped from the on-disk YAML. - # VAREX3-003: manual_variable_rollup SetupTasks emitted by - # workflow_preparer surface in SETUP.md so the user knows where to add - # a roll-up notebook. - rollup_configs = [st.config for st in workflow.setup_tasks if st.type == "manual_variable_rollup"] + # manual_parameters was collected above (before YAML emission) so broken values are stripped on disk too. + # VAREX3-003: manual_variable_rollup SetupTasks from workflow_preparer surface in SETUP.md so the user + # knows where to add a roll-up notebook. + rollup_configs = [task.config for task in workflow.setup_tasks if task.type == "manual_variable_rollup"] for inner in workflow.inner_workflows: - rollup_configs.extend(st.config for st in inner.setup_tasks if st.type == "manual_variable_rollup") - dynamic_dispatch_configs = [st.config for st in workflow.setup_tasks if st.type == "dynamic_notebook_dispatch"] - unresolved_library_configs = [st.config for st in workflow.setup_tasks if st.type == "unresolved_library"] - manual_variable_init_configs = [st.config for st in workflow.setup_tasks if st.type == "manual_variable_init"] + rollup_configs.extend(task.config for task in inner.setup_tasks if task.type == "manual_variable_rollup") + dynamic_dispatch_configs = [ + task.config for task in workflow.setup_tasks if task.type == "dynamic_notebook_dispatch" + ] + unresolved_library_configs = [task.config for task in workflow.setup_tasks if task.type == "unresolved_library"] + manual_variable_init_configs = [task.config for task in workflow.setup_tasks if task.type == "manual_variable_init"] manual_schedule_time_of_day_configs = [ - st.config for st in workflow.setup_tasks if st.type == "manual_schedule_time_of_day" + task.config for task in workflow.setup_tasks if task.type == "manual_schedule_time_of_day" ] - manual_credential_configs = [st.config for st in workflow.setup_tasks if st.type == "manual_credential"] + manual_credential_configs = [task.config for task in workflow.setup_tasks if task.type == "manual_credential"] for inner in workflow.inner_workflows: - dynamic_dispatch_configs.extend(st.config for st in inner.setup_tasks if st.type == "dynamic_notebook_dispatch") - unresolved_library_configs.extend(st.config for st in inner.setup_tasks if st.type == "unresolved_library") - manual_variable_init_configs.extend(st.config for st in inner.setup_tasks if st.type == "manual_variable_init") + dynamic_dispatch_configs.extend( + task.config for task in inner.setup_tasks if task.type == "dynamic_notebook_dispatch" + ) + unresolved_library_configs.extend( + task.config for task in inner.setup_tasks if task.type == "unresolved_library" + ) + manual_variable_init_configs.extend( + task.config for task in inner.setup_tasks if task.type == "manual_variable_init" + ) manual_schedule_time_of_day_configs.extend( - st.config for st in inner.setup_tasks if st.type == "manual_schedule_time_of_day" + task.config for task in inner.setup_tasks if task.type == "manual_schedule_time_of_day" ) - manual_credential_configs.extend(st.config for st in inner.setup_tasks if st.type == "manual_credential") - # LSC3-006: union typed SecretInstructions from the workflow (and - # inner workflows) with the notebook-scanned scopes so SETUP.md and - # create_secrets.py reference the same set of (scope, key) pairs. + manual_credential_configs.extend(task.config for task in inner.setup_tasks if task.type == "manual_credential") + # LSC3-006: union typed SecretInstructions (workflow + inner) with notebook-scanned scopes so SETUP.md + # and create_secrets.py reference the same set of (scope, key) pairs. all_secret_instructions = list(workflow.secrets) for inner in workflow.inner_workflows: all_secret_instructions.extend(inner.secrets) @@ -321,22 +309,47 @@ def write_bundle( return created_files -def main() -> None: - """CLI entry point for DAB bundle generation.""" +def _default_report_path(output_dir: Path) -> Path: + """Returns the conventional report path under a migration dir's .work/ folder. + + Prefers the modify-stamped report; falls back to the raw translation report + when modify was not run (no configuration applied). + """ + work = Path(output_dir) / ".work" + stamped = work / "translation_report.stamped.json" + if stamped.exists(): + return stamped + return work / "translation_report.json" + + +def main(argv: list[str] | None = None) -> int: + """Package-phase entry point for DAB bundle generation. + + Returns a process exit code so the adapter can run this phase in-process (instead of spawning a + second interpreter) and still propagate failures. + """ parser = argparse.ArgumentParser( description="Generate a Databricks Declarative Automation Bundle from a translation report.", ) parser.add_argument( "--report", type=Path, - required=True, - help="Path to the translation report or pipeline IR JSON produced by the translate phase.", + default=None, + help=( + "Path to the (stamped) translation report produced by translate/modify. " + "Defaults to /.work/translation_report.stamped.json (falling back to " + "/.work/translation_report.json when modify was not run)." + ), ) parser.add_argument( "--output-dir", type=Path, - default=Path("./orchestra_output/bundle"), - help="Output directory for the DAB bundle (default: ./orchestra_output/bundle).", + default=Path("./flowx_output"), + help=( + "Migration output directory. The bundle (databricks.yml, resources/, src/) is " + "written here alongside the metadata/ folder; the transient .work/ folder is pruned " + "after a successful build." + ), ) parser.add_argument( "--catalog", @@ -363,23 +376,30 @@ def main() -> None: help="Databricks CLI profile to use when downloading workspace artifacts.", ) parser.add_argument( - "--no-vendor-workspace-files", + "--no-download-workspace-files", action="store_true", help=( "Skip downloading workspace-resident notebooks / Python files / JARs. " "Tasks keep their original workspace paths and the bundle is not self-contained." ), ) - args = parser.parse_args() + parser.add_argument( + "--keep-intermediates", + action="store_true", + help="Keep the transient .work/ folder (translation report + IR) instead of pruning it.", + ) + args = parser.parse_args(argv) + if args.report is None: + args.report = _default_report_path(args.output_dir) if not args.report.exists(): print(f"Error: Report file not found: {args.report}", file=sys.stderr) - sys.exit(1) + return 1 if args.profile: set_profile(args.profile) - if not args.no_vendor_workspace_files: + if not args.no_download_workspace_files: workspace_paths = collect_workspace_artifact_paths(args.report) if workspace_paths: if not prompt_for_auth_if_missing(workspace_paths): @@ -387,7 +407,7 @@ def main() -> None: "Aborted. Run `databricks auth login --host ` and retry.", file=sys.stderr, ) - sys.exit(2) + return 2 enable_workspace_downloads(True) print(f"Loading translation report: {args.report}") @@ -395,7 +415,7 @@ def main() -> None: if not workflows: print("No translated pipelines found in the report.", file=sys.stderr) - sys.exit(1) + return 1 all_created: list[Path] = [] for index, workflow in enumerate(workflows): @@ -415,12 +435,21 @@ def main() -> None: all_created.extend(created) print(f" [{index + 1}/{len(workflows)}] {workflow.name}: {len(created)} files") + if not args.keep_intermediates: + work_dir = args.output_dir / ".work" + if work_dir.is_dir(): + import shutil + + shutil.rmtree(work_dir, ignore_errors=True) + print(f"Pruned transient {work_dir}") + print(f"\nBundle generation complete: {len(all_created)} files written to {args.output_dir}") print("\nNext steps:") print(" 1. Review the generated notebooks in src/") print(" 2. Run the setup notebooks to create secrets and volumes") print(" 3. Validate the bundle: databricks bundle validate") print(" 4. Deploy: databricks bundle deploy -t dev") + return 0 def _warn(task_key: str, message: str) -> None: @@ -431,13 +460,9 @@ def _warn(task_key: str, message: str) -> None: _DEFAULT_SPARK_VERSION = "15.4.x-scala2.12" _DEFAULT_NODE_TYPE_ID = "Standard_DS3_v2" -# C-29 (NB-ITER4-002): a real DBR version string matches e.g. -# "15.4.x-scala2.12" / "15.4.x-photon-scala2.12". ADF expressions like -# ``@if(equals(item()?.photon,true),...)`` slip through unfiltered today -# and land in ``databricks.yml`` as the spark_version variable default, -# which bundle deploy rejects. The regex anchors on the canonical -# Databricks Runtime shape so unrecognised strings fall through to the -# safe default. +# C-29 (NB-ITER4-002): anchor on the canonical DBR version shape (e.g. "15.4.x-photon-scala2.12") so +# unresolved ADF expressions like @if(equals(item()?.photon,true),...) fall through to the safe default +# instead of landing in databricks.yml as a spark_version that bundle deploy rejects. _DBR_VERSION_RE = re.compile(r"^\d+\.\d+\.x(-[a-z0-9.]+)*$") @@ -459,7 +484,7 @@ def _is_valid_node_type_id(value: Any) -> bool: return False if value.startswith("@"): return False - if any(ch.isspace() for ch in value): + if any(char.isspace() for char in value): return False return True @@ -475,10 +500,8 @@ def _infer_bundle_cluster_defaults(workflow: PreparedWorkflow) -> tuple[str, str """ from collections import Counter - # C-29 (NB-ITER4-002): filter out unparseable spark_version / - # node_type_id hints before Counter so unresolved ADF expressions - # (e.g. ``@if(equals(item()?.photon,true),...)``) don't land as the - # bundle's default and break ``databricks bundle deploy``. + # C-29 (NB-ITER4-002): filter out unparseable spark_version / node_type_id hints before Counter so + # unresolved ADF expressions don't land as the bundle default and break ``databricks bundle deploy``. spark_versions = [ hint["spark_version"] for hint in workflow.cluster_hints if _is_valid_spark_version(hint.get("spark_version")) ] @@ -586,9 +609,8 @@ def _build_databricks_yml( "description": "Databricks Runtime for the default job_cluster.", "default": spark_version, } - # Declare a variable for each cross-bundle ExecutePipeline reference so - # `${var.X_job_id}` resolves and `bundle validate` passes. Users fill in - # the numeric job ID per SETUP.md. + # Declare a variable for each cross-bundle ExecutePipeline reference so `${var.X_job_id}` resolves and + # `bundle validate` passes. Users fill in the numeric job ID per SETUP.md. for variable_name, target_pipeline in sorted(_cross_bundle_variables.items()): variables[variable_name] = { "description": ( @@ -771,10 +793,9 @@ def _strip_compute_mode_markers(tasks: list[dict[str, Any]]) -> None: task.pop("_compute_mode", None) -# Patterns that signal a base_parameter value couldn't be evaluated cleanly. -# When any task references an *existing* notebook (absolute workspace path), -# flowx can't inject the runtime computation, so these end up as manual -# work for the user. +# Patterns that signal a base_parameter value couldn't be evaluated cleanly. When a task references an +# existing notebook (absolute workspace path), flowx can't inject the runtime computation, so these +# end up as manual work for the user. _HYBRID_ADF_FN_RE = re.compile(r"@[a-zA-Z][a-zA-Z0-9]*\(") _PYTHON_CODE_HINTS = ("dbutils.widgets.get(", "datetime.now(", "datetime.fromisoformat(") @@ -867,9 +888,8 @@ def _bind_cluster_to_notebook_tasks(tasks: list[dict[str, Any]]) -> None: continue compute_mode = task.get("_compute_mode") if compute_mode == "serverless": - # Serverless cannot host jar/whl libraries. When the task - # ships libraries we must still bind a classic cluster so the - # Jobs API accepts the libraries block. + # Serverless cannot host jar/whl libraries, so when the task ships libraries we still bind a + # classic cluster so the Jobs API accepts the libraries block. if _task_has_jar_or_whl_libraries(task): task["job_cluster_key"] = DEFAULT_JOB_CLUSTER_KEY continue @@ -878,10 +898,8 @@ def _bind_cluster_to_notebook_tasks(tasks: list[dict[str, Any]]) -> None: task["job_cluster_key"] = cluster_key continue notebook_path = notebook_task.get("notebook_path", "") - # Stub notebooks (../src/...) are normally left unbound for - # serverless compute. But when libraries are attached we must - # bind to a real cluster (NB-2) -- serverless cannot install - # jar / whl libraries. + # Stub notebooks (../src/...) are normally left unbound for serverless compute, but when libraries + # are attached we must bind a real cluster (NB-2) -- serverless cannot install jar/whl libraries. if notebook_path.startswith("../src/"): if _task_has_jar_or_whl_libraries(task): task["job_cluster_key"] = DEFAULT_JOB_CLUSTER_KEY @@ -1074,9 +1092,8 @@ def visit(task: dict[str, Any]) -> None: if _is_dangling(value): job_parameters[param_name] = "" - # C-12: condition_task operands can also carry dangling refs - # when an upstream renamed task disappeared between rewrite - # passes. C-43: record each neutralised operand for SETUP.md. + # C-12: condition_task operands can also carry dangling refs when an upstream renamed task + # disappeared between rewrite passes. C-43: record each neutralised operand for SETUP.md. condition_task = task.get("condition_task") or {} if condition_task: task_key = task.get("task_key", "") @@ -1163,15 +1180,12 @@ def _build_job_resource( Dict ready for YAML serialization. """ _rewrite_post_branch_dependencies(workflow.tasks) - # For inner jobs (invoked via run_job_task), notebooks live in the parent - # workflow's notebooks list — pass them in so widget auto-augment can - # still find the bound notebook and populate base_parameters. + # For inner jobs (run_job_task), notebooks live in the parent workflow's list — pass them in so widget + # auto-augment can still find the bound notebook and populate base_parameters. augment_scope = list(workflow.notebooks) + list(extra_notebooks_for_augment or []) _augment_base_parameters(workflow.tasks, augment_scope) - # Task values don't cross ``run_job_task`` boundaries; any such - # reference in this job resolves to an empty string at runtime. Emit - # the empty string now so SETUP.md §4 flags it. C-43: a blanked - # condition operand silently makes the predicate always-true, so record + # Task values don't cross run_job_task boundaries; such a reference resolves to an empty string at + # runtime, so emit it now for SETUP.md §4. C-43: a blanked condition operand is always-true, so record # each neutralised condition for the SETUP.md re-wiring section. _neutralized_conditions.extend( _strip_dangling_task_value_refs(workflow.tasks, _collect_all_task_keys(workflow.tasks)) @@ -1195,17 +1209,30 @@ def _build_job_resource( _strip_compute_mode_markers(workflow.tasks) if workflow.parameters: - job_def["parameters"] = workflow.parameters + # Emit each job parameter once in the DAB shape ({name, default}); dropping the internal ``type`` + # field keeps both bundle paths byte-identical and matches the Databricks job-parameter schema. + seen_param_names: set[str | None] = set() + normalized_parameters: list[dict[str, Any]] = [] + for parameter in workflow.parameters: + name = parameter.get("name") + if name in seen_param_names: + continue + seen_param_names.add(name) + entry: dict[str, Any] = {"name": name} + default = parameter.get("default") + if default is not None: + # Databricks job-parameter defaults are strings; JSON-encode + # Array / Object defaults so the YAML carries valid JSON. + entry["default"] = json.dumps(default) if isinstance(default, (list, dict)) else default + normalized_parameters.append(entry) + job_def["parameters"] = normalized_parameters # C-10 (SCHED-001): render the workflow schedule / trigger spec. schedule_spec = getattr(workflow, "schedule", None) if schedule_spec: _apply_schedule_to_job(job_def, schedule_spec) - # SCHED3-003: trigger-supplied per-pipeline parameter overrides - # update the matching job.parameter defaults so scheduled runs - # receive the trigger's pinned values instead of the bare pipeline - # default. Overrides only mutate existing declared parameters; - # unknown names are silently ignored to keep job_def well-formed. + # SCHED3-003: trigger-supplied parameter overrides update the matching job.parameter defaults so + # scheduled runs get the trigger's pinned values; unknown names are ignored to keep job_def valid. overrides = schedule_spec.get("parameter_overrides") or {} if overrides and job_def.get("parameters"): for entry in job_def["parameters"]: @@ -1270,12 +1297,9 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]: return workflows if "translations" in report: - # Aggregated translation_report.json format: ``translations`` is a - # flat list of ``{pipeline, ir, status, ...}`` entries. Group by - # pipeline name and route each group through the same - # ``_pipeline_dict_to_workflow`` machinery as the single-pipeline IR - # format, so secret discovery / setup tasks / control-flow handling - # all match. + # Aggregated translation_report.json: ``translations`` is a flat list of {pipeline, ir, status}. + # Group by pipeline and route each group through _pipeline_dict_to_workflow (same machinery as the + # single-pipeline IR format) so secret discovery / setup tasks / control-flow handling all match. pipelines: dict[str, list[dict]] = {} pipeline_params: dict[str, list[dict[str, Any]]] = {} pipeline_schedules: dict[str, dict[str, Any]] = {} @@ -1287,16 +1311,13 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]: if not ir: continue pipelines.setdefault(pipeline_name, []).append(ir) - # Round-trip pipeline-level parameters either supplied per- - # translation (newer report shape) or alongside the ir under - # an ``ir.parameters`` key (older single-pipeline serialisations - # roundtripped through this aggregator). + # Round-trip pipeline-level parameters supplied per-translation (newer shape) or under + # ir.parameters (older single-pipeline serialisations roundtripped through this aggregator). params = translation.get("parameters") or ir.get("parameters") if params and pipeline_name not in pipeline_params: pipeline_params[pipeline_name] = list(params) - # Likewise carry pipeline-level ``schedule`` through to the - # rehydrated pipeline_dict so trigger-derived schedule / trigger - # blocks survive the aggregated report shape. + # Likewise carry pipeline-level schedule through to the rehydrated pipeline_dict so + # trigger-derived schedule/trigger blocks survive the aggregated report shape. schedule = translation.get("schedule") or ir.get("schedule") if schedule and pipeline_name not in pipeline_schedules: pipeline_schedules[pipeline_name] = dict(schedule) @@ -1325,11 +1346,10 @@ def _pipeline_dict_to_workflow(pipeline_dict: dict[str, Any]) -> PreparedWorkflo expression resolution, and motif handling without duplicating the per-activity preparer logic. """ - pipeline, parameters = pipeline_dict_to_ir(pipeline_dict) - workflow = prepare_workflow(pipeline) - if parameters: - workflow.parameters.extend(parameters) - return workflow + pipeline, _parameters = pipeline_dict_to_ir(pipeline_dict) + # prepare_workflow already carries pipeline.parameters onto the workflow, so re-extending here would + # duplicate every job parameter (the same dict twice -> a duplicate ``region`` entry in YAML). + return prepare_workflow(pipeline) def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[dict[str, Any]]]: @@ -1351,14 +1371,16 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d entry: dict[str, Any] = {"name": param["name"]} if "default" in param and param["default"] is not None: default_value = param["default"] - # Bool / int / float defaults must survive the JSON round-trip - # as their declared type so the emitted YAML carries a real - # boolean / number, not a quoted string. String defaults go - # through normalize_value to resolve embedded ADF refs. + # Bool / int / float defaults must survive the JSON round-trip as their declared type so the + # YAML carries a real boolean/number; string defaults go through normalize_value for ADF refs. if isinstance(default_value, bool): entry["default"] = default_value elif isinstance(default_value, (int, float)): entry["default"] = default_value + elif isinstance(default_value, (list, dict)): + # Array / Object defaults are JSON-encoded to a string at emission time; keep the structure + # here so both bundle paths converge (a Python str() here would emit invalid JSON). + entry["default"] = default_value else: entry["default"] = normalize_value(str(default_value)) parameters.append(entry) @@ -1366,31 +1388,30 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d name=pipeline_dict.get("name", "unknown"), tasks=activities, parameters=parameters or None, - translation_preferences=_reconstruct_preferences(pipeline_dict.get("translation_preferences")), + translation_configuration=_reconstruct_configuration(pipeline_dict.get("translation_configuration")), schedule=pipeline_dict.get("schedule"), ) return pipeline, parameters -def _reconstruct_preferences(raw: dict[str, Any] | None) -> Any: - """Rebuilds a :class:`TranslationPreferences` from its serialised form. +def _reconstruct_configuration(raw: dict[str, Any] | None) -> Any: + """Rebuilds a :class:`TranslationConfiguration` from its serialised form. Args: - raw: Dict emitted by ``engine._preferences_to_dict``, or ``None`` - when the report carries no preferences. + raw: Dict emitted by ``engine._configuration_to_dict``, or ``None`` + when the report carries no configuration. Returns: - A :class:`TranslationPreferences` instance, or ``None`` when + A :class:`TranslationConfiguration` instance, or ``None`` when *raw* is falsy. """ if not raw: return None - from flowx.adapter.models import TranslationPreferences + from flowx.adapter.models import TranslationConfiguration - # Reports authored before the databricks_task_compute option was - # removed may still carry that key; drop it silently so old reports - # remain rehydratable. - return TranslationPreferences( + # Reports authored before the databricks_task_compute option was removed may still carry that key; + # drop it silently so old reports remain rehydratable. + return TranslationConfiguration( copy_activity_paradigm=raw.get("copy_activity_paradigm", "notebook"), non_databricks_task_compute=raw.get("non_databricks_task_compute", "serverless"), use_lakeflow_connectors=raw.get("use_lakeflow_connectors", "existing"), @@ -1441,6 +1462,11 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: body=task_ir.get("body"), headers=task_ir.get("headers"), authentication=task_ir.get("authentication"), + body_code=task_ir.get("body_code"), + body_imports=list(task_ir.get("body_imports") or []), + body_required_parameters=dict(task_ir.get("body_required_parameters") or {}), + disable_cert_validation=bool(task_ir.get("disable_cert_validation", False)), + http_request_timeout_seconds=task_ir.get("http_request_timeout_seconds"), ) if task_type == "SetVariableActivity": return SetVariableActivity( @@ -1536,9 +1562,8 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: right=task_ir.get("right", ""), if_true_activities=[_reconstruct_ir(child) for child in task_ir.get("if_true_activities") or []], if_false_activities=[_reconstruct_ir(child) for child in task_ir.get("if_false_activities") or []], - # C-14 (CF3-001 / VAREX3-001): preserve bridge fields so the - # preparer can re-synthesise the hidden _bridge SetVariable task - # after a JSON roundtrip. + # C-14 (CF3-001 / VAREX3-001): preserve bridge fields so the preparer can re-synthesise the + # hidden _bridge SetVariable task after a JSON roundtrip. bridge_notebook_code=task_ir.get("bridge_notebook_code"), bridge_notebook_imports=list(task_ir.get("bridge_notebook_imports") or []), bridge_required_parameters=dict(task_ir.get("bridge_required_parameters") or {}), @@ -1555,9 +1580,8 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: for case in task_ir.get("cases") or [] ], default_activities=[_reconstruct_ir(child) for child in task_ir.get("default_activities") or []], - # C-14 (CF3-001 / VAREX3-001): preserve bridge fields for Switch - # so the preparer can re-synthesise the bridge task after a - # JSON roundtrip. + # C-14 (CF3-001 / VAREX3-001): preserve bridge fields for Switch so the preparer can + # re-synthesise the bridge task after a JSON roundtrip. bridge_notebook_code=task_ir.get("bridge_notebook_code"), bridge_notebook_imports=list(task_ir.get("bridge_notebook_imports") or []), bridge_required_parameters=dict(task_ir.get("bridge_required_parameters") or {}), @@ -1614,6 +1638,7 @@ def _common_activity_kwargs(task_ir: dict[str, Any]) -> dict[str, Any]: "parameter_approximations": list(task_ir.get("parameter_approximations") or []), "required_parameters": dict(task_ir.get("required_parameters") or {}), "compute_mode": task_ir.get("compute_mode"), + "notifications": task_ir.get("notifications"), } @@ -1627,4 +1652,4 @@ def _reconstruct_dependencies(raw: list[dict[str, Any]] | None) -> list[Dependen if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/src/orchestra/bundler/inner_job_params.py b/src/flowx/bundler/inner_job_params.py similarity index 95% rename from src/orchestra/bundler/inner_job_params.py rename to src/flowx/bundler/inner_job_params.py index ab36e87..a42e1f7 100644 --- a/src/orchestra/bundler/inner_job_params.py +++ b/src/flowx/bundler/inner_job_params.py @@ -1,4 +1,4 @@ -"""Collects and normalize parameters for ForEach inner jobs.""" +"""Collects and normalizes parameters for ForEach inner jobs.""" from __future__ import annotations @@ -80,9 +80,8 @@ def collect_inner_job_params( parameters: list[dict[str, Any]] = [] for name in sorted(param_names): - # C-06: variables with a known setter task on the parent job route - # via {{tasks.X.values.Y}} -- they must NOT show up as inner-job - # parameter declarations. + # C-06: variables with a known setter task on the parent job route via {{tasks.X.values.Y}}, + # so they must NOT show up as inner-job parameter declarations. if name in variable_names and name in var_task_keys: continue param: dict[str, Any] = {"name": name} @@ -90,9 +89,8 @@ def collect_inner_job_params( param["default"] = "" parameters.append(param) - # "item" (bare @item()) maps to {{input}} (the full iteration value); - # item field names (@item().field) map to {{input.}}; - # pipeline params / variables map to {{job.parameters.}}. + # Map bare @item() -> {{input}}, @item().field -> {{input.}}, and pipeline params/variables + # -> {{job.parameters.}}. job_parameters: dict[str, str] = {} for name in sorted(param_names): if name == "item": @@ -152,28 +150,28 @@ def _scan_tasks( variable_names: Optional accumulator for names sourced from ``variables('X')`` references (separate from pipeline params). """ - kw: dict[str, Any] = {"item_field_names": item_field_names, "variable_names": variable_names} + field_name_kwargs: dict[str, Any] = {"item_field_names": item_field_names, "variable_names": variable_names} for task in tasks: notebook_task = task.get("notebook_task", {}) params = notebook_task.get("base_parameters", {}) for value in params.values(): - _extract_refs(value, param_names, **kw) + _extract_refs(value, param_names, **field_name_kwargs) run_job_task = task.get("run_job_task", {}) for value in run_job_task.get("job_parameters", {}).values(): - _extract_refs(value, param_names, **kw) + _extract_refs(value, param_names, **field_name_kwargs) condition_task = task.get("condition_task", {}) if condition_task: - _extract_refs(condition_task.get("left", ""), param_names, **kw) - _extract_refs(condition_task.get("right", ""), param_names, **kw) - _scan_tasks(condition_task.get("if_true", []), param_names, **kw) - _scan_tasks(condition_task.get("if_false", []), param_names, **kw) + _extract_refs(condition_task.get("left", ""), param_names, **field_name_kwargs) + _extract_refs(condition_task.get("right", ""), param_names, **field_name_kwargs) + _scan_tasks(condition_task.get("if_true", []), param_names, **field_name_kwargs) + _scan_tasks(condition_task.get("if_false", []), param_names, **field_name_kwargs) for_each_task = task.get("for_each_task", {}) body = for_each_task.get("task") if body: - _scan_tasks([body], param_names, **kw) + _scan_tasks([body], param_names, **field_name_kwargs) def _scan_ir_tasks( diff --git a/src/orchestra/bundler/notebook_writer.py b/src/flowx/bundler/notebook_writer.py similarity index 100% rename from src/orchestra/bundler/notebook_writer.py rename to src/flowx/bundler/notebook_writer.py diff --git a/src/orchestra/bundler/prereqs_writer.py b/src/flowx/bundler/prereqs_writer.py similarity index 94% rename from src/orchestra/bundler/prereqs_writer.py rename to src/flowx/bundler/prereqs_writer.py index 9408f5b..0149507 100644 --- a/src/orchestra/bundler/prereqs_writer.py +++ b/src/flowx/bundler/prereqs_writer.py @@ -111,32 +111,25 @@ class Prereqs: network_endpoints: list[NetworkEndpoint] = field(default_factory=list) manual_parameters: list[ManualParameter] = field(default_factory=list) parameter_approximations: list[ParameterApproximation] = field(default_factory=list) - # VAREX3-003: variables mutated inside a ForEach inner-job that a - # sibling task reads. Each entry is the SetupTask.config dict shape - # ({variable_name, parent_foreach, message}). + # VAREX3-003: variables mutated inside a ForEach inner-job that a sibling reads; each entry is the + # SetupTask.config dict ({variable_name, parent_foreach, message}). manual_variable_rollups: list[dict[str, Any]] = field(default_factory=list) - # C-28 (NB-ITER4-001): notebook activities whose ADF ``notebookPath`` is - # a runtime expression the translator couldn't resolve. Each entry is - # the SetupTask.config dict ({task_key, activity_name, expression, - # widget_name}). + # C-28 (NB-ITER4-001): notebook activities whose ADF notebookPath is an unresolved runtime expression; + # each entry is the SetupTask.config dict ({task_key, activity_name, expression, widget_name}). dynamic_notebook_dispatches: list[dict[str, Any]] = field(default_factory=list) - # C-30 (NB-ITER4-003): library descriptor jar/whl paths the translator - # couldn't resolve to a literal/dab_ref. Each entry is the SetupTask - # config dict ({task_key, library_type, expression, missing}). + # C-30 (NB-ITER4-003): library jar/whl paths unresolved to a literal/dab_ref; each entry is the + # SetupTask config dict ({task_key, library_type, expression, missing}). unresolved_libraries: list[dict[str, Any]] = field(default_factory=list) - # C-33 (VAREX4-001/CF4-003): SetVariable activities whose ADF - # expression couldn't be lowered. Each entry is the SetupTask config - # dict ({task_key, variable_name, expression}). + # C-33 (VAREX4-001/CF4-003): SetVariable activities whose ADF expression couldn't be lowered; each + # entry is the SetupTask config dict ({task_key, variable_name, expression}). manual_variable_inits: list[dict[str, Any]] = field(default_factory=list) # C-36 (SCHED4-001): scheduled jobs whose recurrence carried # hours/minutes/weekDays the cron emitter could not encode. manual_schedule_time_of_day: list[dict[str, Any]] = field(default_factory=list) # C-39 (LSC4-004): MSI / CredentialReference cluster substitutions. manual_credentials: list[dict[str, Any]] = field(default_factory=list) - # C-43 (CF5-001 / CF5-002): condition_task operands the bundler had to - # blank because they referenced a task in another job. Each entry is - # {task_key, field, original_ref}. A blanked operand makes the - # predicate always-true, so the user must re-wire the condition. + # C-43 (CF5-001 / CF5-002): condition_task operands blanked because they referenced a task in another + # job ({task_key, field, original_ref}); a blanked operand is always-true, so the user must re-wire it. neutralized_conditions: list[dict[str, str]] = field(default_factory=list) def is_empty(self) -> bool: @@ -393,11 +386,9 @@ def build_prereqs( # the tasks (in case upstream still emits them). cross_bundle.extend(collect_cross_bundle_refs(tasks, known_bundle_jobs)) - # LSC3-006: union notebook-scanned secrets with the workflow's typed - # SecretInstruction list so SETUP.md Option A (scope/key checklist) and - # Option B (create_secrets.py from workflow.secrets) reference the same - # set of (scope, key) pairs. De-dupe by hash; later additions don't - # overwrite earlier values. + # LSC3-006: union notebook-scanned secrets with the workflow's typed SecretInstruction list so + # SETUP.md Option A (scope/key checklist) and Option B (create_secrets.py) reference the same + # (scope, key) set. De-dupe by scope; later additions don't overwrite earlier values. secrets = scan_notebooks_for_secrets(notebooks) for instruction in secret_instructions or []: secrets.setdefault(instruction.scope, set()).add(instruction.key) @@ -490,7 +481,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("") lines.append( "The following notebooks are stubs that raise `NotImplementedError`. " - "Flowx could not download the source (either no workspace path was " + "flowx could not download the source (either no workspace path was " "supplied in ADF, or the path did not resolve against the " "authenticated workspace). Replace each stub with the real logic." ) @@ -510,7 +501,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("") lines.append( "If the workspace path exists in a reachable Databricks workspace, you can " - "have Flowx re-ingest it by running `databricks workspace export` and " + "have flowx re-ingest it by running `databricks workspace export` and " "placing the result at the indicated bundle path." ) lines.append("") @@ -520,7 +511,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("") lines.append( "Each row below describes a `run_job_task` that invokes a job **not** " - "defined in this bundle. Flowx emitted a bundle variable for each " + "defined in this bundle. flowx emitted a bundle variable for each " "one (`${var.}`) so `databricks bundle validate` passes. " "Before running, populate the variable with the numeric job ID the " "target pipeline was deployed under — either set a `default:` in " @@ -583,7 +574,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("## Parameter substitutions") lines.append("") lines.append( - "Flowx mapped the ADF expressions below to Databricks dynamic value " + "flowx mapped the ADF expressions below to Databricks dynamic value " "references so they land directly in the bundle YAML. The substitutions are " "semantically *close* but not identical to the originals; review the listed " "caveats and decide whether each replacement is acceptable for your workload." @@ -604,7 +595,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("") lines.append( "The ADF activities below carried a runtime expression for " - "`notebookPath`. Flowx emitted a dispatch-stub notebook for " + "`notebookPath`. flowx emitted a dispatch-stub notebook for " "each one that reads the resolved path from the listed widget and " "calls `dbutils.notebook.run()`. Supply the widget value at job " "runtime (via `--params`, a parent task value, or job parameter " @@ -650,7 +641,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("") lines.append( "The ADF SetVariable activities below carried expressions the " - "translator couldn't lower. Flowx blanked the variable's " + "translator couldn't lower. flowx blanked the variable's " "initial value to keep the bundle YAML valid. Compute the real " "value yourself (e.g. via a parent task value or runtime widget) " "before downstream tasks read the variable." @@ -691,7 +682,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append( "The cluster compute backing the tasks below was authenticated in " "ADF via a managed identity / CredentialReference that has no " - "direct Databricks equivalent. Flowx defaulted the bundle's " + "direct Databricks equivalent. flowx defaulted the bundle's " "default_cluster to `single_user_name: ${workspace.current_user.userName}` " "so deployment works for the deploying user, but production runs " "should swap that for a service principal." @@ -717,7 +708,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: "The IfCondition tasks below referenced a task value that lives only " "in another job (typically a parent-job init task hoisted out of a " "split-out ForEach inner job). Databricks task values cannot cross " - "`run_job_task` boundaries, so Flowx blanked the operand. A blanked " + "`run_job_task` boundaries, so flowx blanked the operand. A blanked " "operand makes the predicate `NOT_EQUAL('', '0')` **always true**, so the " "branch now runs unconditionally. Re-wire each condition below — either " "recompute the operand inside this job or pass it as a job parameter." diff --git a/src/orchestra/bundler/setup_generator.py b/src/flowx/bundler/setup_generator.py similarity index 91% rename from src/orchestra/bundler/setup_generator.py rename to src/flowx/bundler/setup_generator.py index 76059d0..6aa8e7e 100644 --- a/src/orchestra/bundler/setup_generator.py +++ b/src/flowx/bundler/setup_generator.py @@ -30,12 +30,12 @@ def generate_setup_tasks( notebook = _generate_secrets_setup_notebook(secrets) notebooks.append(notebook) - volume_tasks = [t for t in setup_tasks if t.type == "volume"] + volume_tasks = [task for task in setup_tasks if task.type == "volume"] if volume_tasks: notebook = _generate_volume_setup_notebook(volume_tasks, catalog, schema) notebooks.append(notebook) - connection_tasks = [t for t in setup_tasks if t.type == "connection"] + connection_tasks = [task for task in setup_tasks if task.type == "connection"] if connection_tasks: notebook = _generate_connection_setup_notebook(connection_tasks, catalog) notebooks.append(notebook) @@ -57,7 +57,7 @@ def _generate_secrets_setup_notebook(secrets: list[SecretInstruction]) -> DabNot # MAGIC %md # MAGIC # Setup: Create Secret Scopes and Secrets # MAGIC - # MAGIC *Auto-generated by Flowx. Run this notebook once before deploying the job.* + # MAGIC *Auto-generated by flowx. Run this notebook once before deploying the job.* # MAGIC # MAGIC This notebook creates the Databricks secret scopes and placeholder secrets # MAGIC required by the translated pipelines. After running, update each secret @@ -71,13 +71,11 @@ def _generate_secrets_setup_notebook(secrets: list[SecretInstruction]) -> DabNot separator = "\n# COMMAND ----------\n\n" scopes: dict[str, list[SecretInstruction]] = {} - for s in secrets: - scopes.setdefault(s.scope, []).append(s) + for secret in secrets: + scopes.setdefault(secret.scope, []).append(secret) - # C-46 (LSC5-002): the ``dbutils.secrets`` submodule is read-only - # (get / getBytes / list / listScopes) — ``createScope`` and ``put`` do - # not exist and raise AttributeError on the first cell. Provision via - # the Databricks SDK ``WorkspaceClient`` instead. + # C-46 (LSC5-002): dbutils.secrets is read-only (get/getBytes/list/listScopes) — createScope/put + # don't exist and raise AttributeError, so provision via the Databricks SDK WorkspaceClient instead. init_cell = textwrap.dedent("""\ from databricks.sdk import WorkspaceClient @@ -150,7 +148,7 @@ def _generate_volume_setup_notebook( # MAGIC %md # MAGIC # Setup: Create Unity Catalog Volumes # MAGIC - # MAGIC *Auto-generated by Flowx. Run this notebook once before deploying the job.* + # MAGIC *Auto-generated by flowx. Run this notebook once before deploying the job.* # MAGIC # MAGIC For each external volume below, this notebook attempts to create the # MAGIC underlying Storage Credential and External Location first. Both require @@ -212,11 +210,11 @@ def _generate_volume_setup_notebook( def _credential_name_for(volume_name: str) -> str: - return f"orchestra_{volume_name}_credential" + return f"flowx_{volume_name}_credential" def _external_location_name_for(volume_name: str) -> str: - return f"orchestra_{volume_name}_location" + return f"flowx_{volume_name}_location" def _render_storage_credential_ddl(credential_name: str, location_type: str) -> str: @@ -230,7 +228,7 @@ def _render_storage_credential_ddl(credential_name: str, location_type: str) -> " WITH AZURE_MANAGED_IDENTITY (\n" " 'PLACEHOLDER_ACCESS_CONNECTOR_RESOURCE_ID'\n" " )\n" - f" COMMENT 'Auto-generated by Flowx for volume {credential_name}'\n" + f" COMMENT 'Auto-generated by flowx for volume {credential_name}'\n" '""")' ) if location_type == "AmazonS3Location": @@ -239,7 +237,7 @@ def _render_storage_credential_ddl(credential_name: str, location_type: str) -> 'spark.sql("""\n' f" CREATE STORAGE CREDENTIAL IF NOT EXISTS {credential_name}\n" " WITH IAM_ROLE 'arn:aws:iam::PLACEHOLDER_ACCOUNT_ID:role/PLACEHOLDER_ROLE'\n" - f" COMMENT 'Auto-generated by Flowx for volume {credential_name}'\n" + f" COMMENT 'Auto-generated by flowx for volume {credential_name}'\n" '""")' ) if location_type == "GoogleCloudStorageLocation": @@ -248,7 +246,7 @@ def _render_storage_credential_ddl(credential_name: str, location_type: str) -> 'spark.sql("""\n' f" CREATE STORAGE CREDENTIAL IF NOT EXISTS {credential_name}\n" " WITH GCP_SERVICE_ACCOUNT 'PLACEHOLDER_SERVICE_ACCOUNT_EMAIL'\n" - f" COMMENT 'Auto-generated by Flowx for volume {credential_name}'\n" + f" COMMENT 'Auto-generated by flowx for volume {credential_name}'\n" '""")' ) return ( @@ -264,7 +262,7 @@ def _render_external_location_ddl(name: str, url: str, credential: str) -> str: f" CREATE EXTERNAL LOCATION IF NOT EXISTS {name}\n" f" URL '{url}'\n" f" WITH (STORAGE CREDENTIAL {credential})\n" - f" COMMENT 'Auto-generated by Flowx'\n" + f" COMMENT 'Auto-generated by flowx'\n" '""")' ) @@ -287,7 +285,7 @@ def _generate_connection_setup_notebook( # MAGIC %md # MAGIC # Setup: Create External Connections # MAGIC - # MAGIC *Auto-generated by Flowx. Run this notebook once before deploying the job.* + # MAGIC *Auto-generated by flowx. Run this notebook once before deploying the job.* # MAGIC # MAGIC Update the placeholder connection details below with real values before running. """) diff --git a/src/flowx/mcp/__init__.py b/src/flowx/mcp/__init__.py new file mode 100644 index 0000000..924b4f4 --- /dev/null +++ b/src/flowx/mcp/__init__.py @@ -0,0 +1,18 @@ +"""MCP packaging for flowx: exposes the migration phases and adapter operations as MCP tools. + +``build_server`` / ``build_http_app`` require the ``mcp`` extra (``pip install -e .[mcp]``) and are +imported lazily so :mod:`flowx.mcp.runner` stays usable without it. +""" + +from typing import Any + +__all__ = ["build_server", "build_http_app"] + + +def __getattr__(name: str) -> Any: + # Lazy re-export: import server (and the `mcp` extra) only when these names are accessed. + if name in ("build_server", "build_http_app"): + from flowx.mcp import server + + return getattr(server, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/flowx/mcp/__main__.py b/src/flowx/mcp/__main__.py new file mode 100644 index 0000000..e97f770 --- /dev/null +++ b/src/flowx/mcp/__main__.py @@ -0,0 +1,6 @@ +"""``python -m flowx.mcp`` entry point.""" + +from flowx.mcp.server import serve + +if __name__ == "__main__": + serve() diff --git a/src/flowx/mcp/runner.py b/src/flowx/mcp/runner.py new file mode 100644 index 0000000..bc6986e --- /dev/null +++ b/src/flowx/mcp/runner.py @@ -0,0 +1,365 @@ +"""Subprocess bridge between MCP tools and the flowx adapter CLI. + +Every MCP tool shells out to ``python -m flowx.adapter`` — the same unified +entry point the agent skills already use — then reads back the JSON/CSV +artifacts each phase writes. This reuses the tested phase contracts instead of +re-implementing their logic, so the MCP surface stays in lockstep with the CLI. +""" + +from __future__ import annotations + +import csv +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +# Phases can take a while on large factories; allow generous default headroom. +DEFAULT_TIMEOUT = int(os.environ.get("FLOWX_MCP_TIMEOUT", "1800")) + +# Cap on inline `adf_definitions` payloads (they pass through the agent context); above this, stage +# to a UC Volume and pass a path reference instead. Configurable via FLOWX_MAX_INLINE_BYTES. +MAX_INLINE_BYTES = int(os.environ.get("FLOWX_MAX_INLINE_BYTES", str(5_000_000))) + + +@dataclass +class AdapterResult: + """Outcome of a single ``flowx.adapter`` invocation.""" + + command: list[str] + returncode: int + stdout: str + stderr: str + + @property + def ok(self) -> bool: + return self.returncode == 0 + + def as_dict(self) -> dict[str, Any]: + """Serialise the raw process outcome for inclusion in a tool result.""" + return { + "command": " ".join(self.command), + "ok": self.ok, + "returncode": self.returncode, + "stdout": self.stdout.strip(), + "stderr": self.stderr.strip(), + } + + +def run_adapter(args: list[Any], *, cwd: str | Path | None = None, timeout: int = DEFAULT_TIMEOUT) -> AdapterResult: + """Invoke ``python -m flowx.adapter`` with the supplied arguments. + + Args: + args: Adapter subcommand and flags (each item is stringified). + cwd: Working directory for the subprocess. Defaults to the current one. + timeout: Seconds before the subprocess is killed. + + Returns: + The captured :class:`AdapterResult`. + """ + command = [sys.executable, "-m", "flowx.adapter", *[str(arg) for arg in args]] + proc = subprocess.run( + command, + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + timeout=timeout, + ) + return AdapterResult(command=command, returncode=proc.returncode, stdout=proc.stdout, stderr=proc.stderr) + + +def read_json(path: Path) -> Any | None: + """Return parsed JSON at *path*, or ``None`` when the file is absent/invalid.""" + try: + return json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + + +def read_text(path: Path) -> str | None: + """Return text at *path*, or ``None`` when it cannot be read.""" + try: + return path.read_text() + except OSError: + return None + + +def parse_stdout_json(result: AdapterResult) -> Any | None: + """Parse the adapter's stdout as JSON (used by inspect/inputs/workspace-paths).""" + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + return None + + +def list_tree(root: Path, *, max_entries: int = 250) -> list[str]: + """Return repo-relative paths of files under *root* (sorted, capped).""" + if not root.exists(): + return [] + files = sorted(str(p.relative_to(root)) for p in root.rglob("*") if p.is_file()) + return files[:max_entries] + + +def summarize_inventory(output_dir: Path) -> dict[str, Any] | None: + """Summarise ``metadata/inventory.json`` produced by the discover phase. + + The discover phase writes a ready-made ``summary`` block (pipeline/activity + counts by strategy plus coverage); surface it directly when present. + """ + inventory = read_json(output_dir / "metadata" / "inventory.json") + if not isinstance(inventory, dict): + return None + + summary = inventory.get("summary") + if isinstance(summary, dict): + return summary + + pipelines = inventory.get("pipelines") or [] + return {"pipeline_count": len(pipelines)} + + +def summarize_translation(output_dir: Path) -> dict[str, Any] | None: + """Summarise the translation report (transient under ``.work/``).""" + for candidate in (".work/translation_report.json", "translation_report.json"): + report = read_json(output_dir / candidate) + if isinstance(report, dict): + break + else: + return None + + pipelines = report.get("pipelines") or [] + statuses: dict[str, int] = {} + for pipeline in pipelines: + for task in pipeline.get("tasks", []): + status = str(task.get("status", "translated")).lower() + statuses[status] = statuses.get(status, 0) + 1 + return {"pipelines": len(pipelines), "task_status_counts": statuses} + + +def materialize_lookup_rows(source: str) -> list[dict[str, str]]: + """Parse a CSV file path or literal CSV string into a list of row dicts. + + Mirrors the adapter's ``materialize-lookup`` parsing so the MCP tool can + validate input without a second subprocess hop. + """ + text = Path(source).read_text() if Path(source).exists() else source + reader = csv.DictReader(io.StringIO(text)) + return [dict(row) for row in reader] + + +def materialize_adf_definitions(definitions: dict[str, Any]) -> str: + """Write an inline ADF-definitions payload to a temp dir and return a source path. + + A hosted MCP server (Databricks App) cannot read the user's workspace / UC Volume files, so + the caller (which can) passes the ADF JSON inline and the server materializes it locally. + + ``definitions`` maps relative file paths — mirroring the ADF Git-export layout, e.g. + ``"pipeline/Foo.json"``, ``"dataset/Bar.json"``, ``"linkedService/Baz.json"``, + ``"trigger/Qux.json"`` — to JSON content (a dict, or a JSON string). The files are written + under a fresh temp directory whose path is returned (the loader reads it as a tree). + + Special case: a single entry whose content is an ARM template (a dict with a top-level + ``resources`` list) is written as one file and that file path is returned, so the loader + parses it in ARM-template mode. + + Raises: + ValueError: if the payload is empty or a key escapes the temp directory. + """ + if not definitions: + raise ValueError("adf_definitions is empty") + + total_bytes = sum(len(v if isinstance(v, str) else json.dumps(v)) for v in definitions.values()) + if total_bytes > MAX_INLINE_BYTES: + raise ValueError( + f"adf_definitions is ~{total_bytes} bytes (limit {MAX_INLINE_BYTES}); inline payloads pass " + "through the agent's context and do not scale. Stage the ADF export to a UC Volume and pass " + "'adf_volume_path' instead (the server reads it directly via the SDK Files API)." + ) + + base = Path(tempfile.mkdtemp(prefix="flowx-adf-")) + + if len(definitions) == 1: + (only_value,) = definitions.values() + content = json.loads(only_value) if isinstance(only_value, str) else only_value + if isinstance(content, dict) and isinstance(content.get("resources"), list): + file_path = base / "arm_template.json" + file_path.write_text(json.dumps(content), encoding="utf-8") + return str(file_path) + + base_resolved = base.resolve() + for rel_path, content in definitions.items(): + dest = (base / rel_path).resolve() + if base_resolved not in dest.parents and dest != base_resolved: + shutil.rmtree(base, ignore_errors=True) + raise ValueError(f"unsafe path in adf_definitions: {rel_path!r}") + dest.parent.mkdir(parents=True, exist_ok=True) + text = content if isinstance(content, str) else json.dumps(content) + dest.write_text(text, encoding="utf-8") + return str(base) + + +def cleanup_materialized(source: str) -> None: + """Remove a temp tree created by :func:`materialize_adf_definitions`. + + Accepts either the returned directory or the single-file path (whose parent temp dir is + removed). Only paths under the system temp dir are deleted, as a safety guard. + """ + path = Path(source) + target = path if path.is_dir() else path.parent + if str(target.resolve()).startswith(str(Path(tempfile.gettempdir()).resolve())): + shutil.rmtree(target, ignore_errors=True) + + +def read_tree(root: Path, *, max_total_bytes: int = 2_000_000) -> dict[str, Any]: + """Return the text contents of files under *root* so a caller can persist them. + + The hosted app writes the generated bundle to ephemeral local disk that the user cannot + reach, so the bundle contents are returned inline. Binary/unreadable files are skipped, and + once the cumulative size passes *max_total_bytes* further files are listed under ``truncated`` + instead of being included. + + Returns: + ``{"files": {relpath: text, ...}, "truncated": [relpath, ...]}`` ("truncated" omitted when empty). + """ + files: dict[str, str] = {} + truncated: list[str] = [] + total = 0 + if root.exists(): + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + rel = str(path.relative_to(root)) + try: + data = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + if total + len(data) > max_total_bytes: + truncated.append(rel) + continue + files[rel] = data + total += len(data) + result: dict[str, Any] = {"files": files} + if truncated: + result["truncated"] = truncated + return result + + +def download_volume_dir(volume_path: str) -> str: + """Download a Unity Catalog Volume directory tree to a local temp dir via the SDK Files API. + + This is the scalable input path for large factories: the bytes are pulled by the server (which + can read the volume as its service principal) and never pass through the calling agent. Returns + the local temp-dir path (clean up with :func:`cleanup_materialized`). + """ + from databricks.sdk import WorkspaceClient + + client = WorkspaceClient() + base = Path(tempfile.mkdtemp(prefix="flowx-vol-")) + root = volume_path.rstrip("/") + + def _recurse(directory: str) -> None: + for entry in client.files.list_directory_contents(directory): + entry_path = entry.path or "" + if entry.is_directory: + _recurse(entry_path) + continue + rel = entry_path[len(root) :].lstrip("/") + dest = base / rel + dest.parent.mkdir(parents=True, exist_ok=True) + contents = client.files.download(entry_path).contents + dest.write_bytes(contents.read() if contents is not None else b"") + + _recurse(root) + return str(base) + + +def download_workspace_dir(workspace_path: str) -> str: + """Download a ``/Workspace`` directory tree to a local temp dir via the SDK Workspace API. + + Workspace files (e.g. an ADF Git folder cloned under ``/Workspace``) use the Workspace API + (``w.workspace.list`` / ``w.workspace.download``), which is distinct from the Files API used for + UC Volumes. Like the volume path, the bytes are pulled by the server and bypass the agent. + Returns the local temp-dir path (clean up with :func:`cleanup_materialized`). + """ + from databricks.sdk import WorkspaceClient + from databricks.sdk.service.workspace import ObjectType + + client = WorkspaceClient() + base = Path(tempfile.mkdtemp(prefix="flowx-ws-")) + root = workspace_path.rstrip("/") + + def _recurse(directory: str) -> None: + for entry in client.workspace.list(directory): + entry_path = entry.path or "" + if entry.object_type in (ObjectType.DIRECTORY, ObjectType.REPO): + _recurse(entry_path) + elif entry.object_type == ObjectType.FILE: + rel = entry_path[len(root) :].lstrip("/") + dest = base / rel + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(client.workspace.download(entry_path).read()) + + _recurse(root) + return str(base) + + +def upload_tree_to_volume(local_root: Path, volume_path: str) -> dict[str, Any]: + """Upload a local directory tree to a Unity Catalog Volume via the SDK Files API. + + This is the scalable output path: the generated bundle is written to a location the user can + reach without the (potentially large) file contents passing back through the agent. + + Returns: + ``{"output_volume_path": , "files": [relpath, ...], "count": n}``. + """ + from databricks.sdk import WorkspaceClient + + client = WorkspaceClient() + root = volume_path.rstrip("/") + uploaded: list[str] = [] + for path in sorted(local_root.rglob("*")): + if not path.is_file(): + continue + rel = str(path.relative_to(local_root)) + client.files.upload(f"{root}/{rel}", io.BytesIO(path.read_bytes()), overwrite=True) + uploaded.append(rel) + return {"output_volume_path": root, "files": uploaded, "count": len(uploaded)} + + +def upload_tree_to_workspace(local_root: Path, workspace_path: str) -> dict[str, Any]: + """Upload a local directory tree to a ``/Workspace`` directory via the SDK Workspace API. + + The output counterpart to :func:`download_workspace_dir`: the generated DAB lands in a workspace + folder the user can reach without the contents passing back through the agent. Files are imported + with ``ImportFormat.RAW`` so each one (``databricks.yml``, ``*.py``, ``*.yml``, ``SETUP.md``, …) is + stored verbatim as a workspace **file** rather than being interpreted as a notebook (which + ``AUTO`` would do to ``.py`` files, corrupting the bundle source tree). + + Returns: + ``{"output_workspace_path": , "files": [relpath, ...], "count": n}``. + """ + from databricks.sdk import WorkspaceClient + from databricks.sdk.service.workspace import ImportFormat + + client = WorkspaceClient() + root = workspace_path.rstrip("/") + uploaded: list[str] = [] + made_dirs: set[str] = set() + for path in sorted(local_root.rglob("*")): + if not path.is_file(): + continue + rel = str(path.relative_to(local_root)) + dest = f"{root}/{rel}" + parent = dest.rsplit("/", 1)[0] + if parent not in made_dirs: + client.workspace.mkdirs(parent) + made_dirs.add(parent) + client.workspace.upload(dest, path.read_bytes(), format=ImportFormat.RAW, overwrite=True) + uploaded.append(rel) + return {"output_workspace_path": root, "files": uploaded, "count": len(uploaded)} diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py new file mode 100644 index 0000000..f21e8c9 --- /dev/null +++ b/src/flowx/mcp/server.py @@ -0,0 +1,554 @@ +"""MCP server exposing flowx as a single dispatcher tool, ``flowx(command, parameters)``. + +Each command is a thin wrapper over ``python -m flowx.adapter`` (see :mod:`flowx.mcp.runner`); +keeping flowx to one tool stays under host tool-count caps such as Genie Code's 20-tool limit. +""" + +from __future__ import annotations + +import os +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from mcp.server.fastmcp import FastMCP +from mcp.server.transport_security import TransportSecuritySettings + +from flowx.mcp import runner + + +def _allowed_origins() -> list[str]: + """Allowed browser/MCP origins from ``FLOWX_ALLOWED_ORIGINS`` (comma-separated, default ``*``).""" + raw = os.environ.get("FLOWX_ALLOWED_ORIGINS", "*") + return [origin.strip() for origin in raw.split(",") if origin.strip()] + + +def _transport_security() -> TransportSecuritySettings: + """Builds transport-security settings with the SDK's DNS-rebinding check disabled. + + Returns: + Settings that skip the Host/Origin allowlist check. + + Notes: + Behind the Databricks Apps OAuth proxy the check misfires (403/421) and adds nothing on top + of the proxy. Browser CORS is handled separately in :func:`build_http_app`. See the "MCP + server design notes" in AGENTS.md. + """ + return TransportSecuritySettings(enable_dns_rebinding_protection=False) + + +_INSTRUCTIONS = """\ +flowx translates Azure Data Factory (ADF) pipelines into Databricks Lakeflow Jobs packaged as +Declarative Automation Bundles (DABs). Everything is driven through the single `flowx` tool: +`flowx(command="", parameters={...})`. + +Typical flow: + flowx("inputs", {"phase": "discover"}) # learn a phase's inputs + flowx("discover", {"adf_source_path": "...", "output_dir": "..."}) + flowx("convert", {"output_dir": "..."}) + flowx("inspect", {"report_path": "/.work/translation_report.json"}) + flowx("apply_answers", {"report_path": "...", "answers": ["id=value"], "output_dir": "..."}) + flowx("package", {"output_dir": "...", "catalog": "main", "schema": "default"}) +Or run it all at once: + flowx("migrate", {"adf_source_path": "...", "output_dir": "...", "catalog": "...", "schema": "..."}) + +All phases share one output_dir. Provide ADF source paths and output_dir as locations the server can +read/write (a local path, or a Unity Catalog Volume path when the host has volume access). +""" + + +def _phase_result(result: runner.AdapterResult, output_dir: Path, **extra: Any) -> dict[str, Any]: + """Assemble a structured tool result from an adapter run plus artifacts.""" + payload: dict[str, Any] = {"ok": result.ok, "process": result.as_dict(), "output_dir": str(output_dir)} + payload.update({k: v for k, v in extra.items() if v is not None}) + return payload + + +def _resolve_source(p: dict[str, Any], path_key: str = "adf_source_path") -> tuple[str | None, Callable[[], None]]: + """Resolve the ADF source for a command into a local path the adapter can read. + + Input modes, in priority order — a hosted app can't read the user's files directly, so it relies + on the first three: + + 1. ``adf_volume_path`` — a UC Volume directory; the server downloads it via the SDK Files API. + 2. ``adf_workspace_path`` — a ``/Workspace`` directory (e.g. an ADF Git folder); the server + downloads it via the SDK Workspace API. + Both (1) and (2) scale to large factories — the bytes bypass the agent. Each returns a temp + dir + cleanup. + 3. ``adf_definitions`` — an inline ARM-JSON payload (small jobs); materialized to a temp dir. + 4. ``path_key`` (``adf_source_path`` / ``source_dir``) — a path the server itself can read + (local hosting or a mounted volume). + """ + if p.get("adf_volume_path"): + src = runner.download_volume_dir(p["adf_volume_path"]) + return src, lambda: runner.cleanup_materialized(src) + if p.get("adf_workspace_path"): + src = runner.download_workspace_dir(p["adf_workspace_path"]) + return src, lambda: runner.cleanup_materialized(src) + definitions = p.get("adf_definitions") + if definitions: + src = runner.materialize_adf_definitions(definitions) + return src, lambda: runner.cleanup_materialized(src) + return p.get(path_key), (lambda: None) + + +def _bundle_output(p: dict[str, Any], out: Path) -> dict[str, Any]: + """Deliver the generated bundle to a location the user can reach. + + The server's ``output_dir`` is local/ephemeral, so the bundle is written to the target via the + SDK (contents bypass the agent and it scales), in priority order: + + 1. ``output_volume_path`` — upload to a UC Volume via the SDK Files API. + 2. ``output_workspace_path`` — upload to a ``/Workspace`` directory via the SDK Workspace API. + 3. neither — return the contents inline as ``bundle`` for the agent to persist (small bundles). + """ + if p.get("output_volume_path"): + return {"bundle_uploaded": runner.upload_tree_to_volume(out, p["output_volume_path"])} + if p.get("output_workspace_path"): + return {"bundle_uploaded": runner.upload_tree_to_workspace(out, p["output_workspace_path"])} + return {"bundle": runner.read_tree(out)} + + +def _noop() -> None: + """Cleanup placeholder used when there is no materialized source to remove.""" + + +def _pending_options(inspect_result: dict[str, Any]) -> list[dict[str, Any]]: + """Extract the per-pipeline configuration options still awaiting an answer. + + Reads the payload :func:`_cmd_inspect` returns (``{"questions": {"pipelines": [...]}}``) + and keeps only pipelines that still have unanswered ``options`` — empty when nothing needs + input, which is the signal for ``migrate`` to package without pausing. + """ + questions = inspect_result.get("questions") or {} + pipelines = questions.get("pipelines") or [] + return [pipeline for pipeline in pipelines if pipeline.get("options")] + + +# Command handlers: map a `parameters` dict to a structured result; required keys via p[...] so a +# missing one raises KeyError, which the dispatcher converts into a clear error. + + +def _cmd_inputs(p: dict[str, Any]) -> dict[str, Any]: + result = runner.run_adapter(["inputs", p["phase"]]) + return {"ok": result.ok, "inputs": runner.parse_stdout_json(result), "process": result.as_dict()} + + +def _cmd_discover(p: dict[str, Any]) -> dict[str, Any]: + output_dir = p.get("output_dir", "./flowx_output") + source, cleanup = _resolve_source(p) + if not source: + return {"ok": False, "error": "Provide 'adf_definitions' (inline ARM JSON) or 'adf_source_path'."} + try: + args = ["discover", "--adf-source-path", source, "--output-dir", output_dir] + if p.get("pipeline"): + args += ["--pipeline", p["pipeline"]] + result = runner.run_adapter(args) + out = Path(output_dir) + return _phase_result(result, out, inventory=runner.summarize_inventory(out)) + finally: + cleanup() + + +def _cmd_convert(p: dict[str, Any]) -> dict[str, Any]: + output_dir = p.get("output_dir", "./flowx_output") + source, cleanup = _resolve_source(p) + try: + args = ["convert", "--output-dir", output_dir] + if source: + args += ["--adf-source-path", source] + if p.get("pipeline"): + args += ["--pipeline", p["pipeline"]] + result = runner.run_adapter(args) + out = Path(output_dir) + return _phase_result(result, out, translation=runner.summarize_translation(out)) + finally: + cleanup() + + +def _cmd_merge_agentic(p: dict[str, Any]) -> dict[str, Any]: + args = ["convert", "--merge-agentic", "--report", p["report_path"], "--agentic-results", p["agentic_results_dir"]] + if p.get("output_path"): + args += ["--output", p["output_path"]] + result = runner.run_adapter(args) + return {"ok": result.ok, "process": result.as_dict()} + + +def _cmd_inspect(p: dict[str, Any]) -> dict[str, Any]: + args: list[Any] = ["inspect", p["report_path"]] + for answer in p.get("answers") or []: + args += ["--answer", answer] + result = runner.run_adapter(args) + return {"ok": result.ok, "questions": runner.parse_stdout_json(result), "process": result.as_dict()} + + +def _cmd_apply_answers(p: dict[str, Any]) -> dict[str, Any]: + args: list[Any] = ["modify", p["report_path"]] + for answer in p["answers"]: + args += ["--answer", answer] + if p.get("output_dir"): + args += ["--output-dir", p["output_dir"]] + if p.get("lookup_csv"): + args += ["--lookup-csv", p["lookup_csv"]] + result = runner.run_adapter(args) + return {"ok": result.ok, "process": result.as_dict()} + + +def _cmd_materialize_lookup(p: dict[str, Any]) -> dict[str, Any]: + result = runner.run_adapter(["materialize-lookup", p["source"], "--out", p["out"]]) + return {"ok": result.ok, "out": p["out"], "process": result.as_dict()} + + +def _cmd_workspace_paths(p: dict[str, Any]) -> dict[str, Any]: + args: list[Any] = ["workspace-paths", p["report_path"]] + source, cleanup = _resolve_source(p, path_key="source_dir") + try: + if source: + args += ["--source-dir", source] + result = runner.run_adapter(args) + return {"ok": result.ok, "result": runner.parse_stdout_json(result), "process": result.as_dict()} + finally: + cleanup() + + +def _cmd_package(p: dict[str, Any]) -> dict[str, Any]: + output_dir = p.get("output_dir", "./flowx_output") + args: list[Any] = [ + "package", + "--output-dir", + output_dir, + "--catalog", + p.get("catalog", "main"), + "--schema", + p.get("schema", "default"), + ] + if p.get("report_path"): + args += ["--report", p["report_path"]] + if p.get("bundle_name"): + args += ["--bundle-name", p["bundle_name"]] + if p.get("profile"): + args += ["--profile", p["profile"]] + if p.get("download_workspace_files") is False: + args += ["--no-download-workspace-files"] + if p.get("keep_intermediates"): + args += ["--keep-intermediates"] + result = runner.run_adapter(args) + out = Path(output_dir) + setup_md = runner.read_text(out / "SETUP.md") or runner.read_text(out / "setup" / "SETUP.md") + extra = _bundle_output(p, out) if result.ok else {} + return _phase_result(result, out, bundle_files=runner.list_tree(out), setup_md=setup_md, **extra) + + +def _cmd_migrate(p: dict[str, Any]) -> dict[str, Any]: + """Run discover→convert→package, pausing for configuration when options are available. + + Because an MCP call can't prompt mid-flight, ``migrate`` is interactive by *handing the questions + back to the agent*: after ``convert`` it returns the **full option schema** once + (``status="needs_input"`` with ``pending_options`` -- every option annotated with a ``show_when`` + condition). The agent drives the whole chain locally (asking only the options whose ``show_when`` + is satisfied, performing any data lookups), then re-calls ``migrate`` a single time with the + complete ``answers`` (``["option_id=value", ...]``), which applies them and packages + (``status="completed"``). No per-follow-up round trip. Pass ``interactive=False`` to skip the + prompt and package with defaults. + """ + output_dir = p.get("output_dir", "./flowx_output") + catalog = p.get("catalog", "main") + schema = p.get("schema", "default") + pipeline = p.get("pipeline") + answers = p.get("answers") or [] + interactive = p.get("interactive", True) + out = Path(output_dir) + report_path = str(out / ".work" / "translation_report.json") + steps: dict[str, Any] = {} + + # Resume: with answers in hand and a prior report present, skip re-running discover/convert. + resume = bool(answers) and (out / ".work" / "translation_report.json").is_file() + + cleanup = _noop + try: + if not resume: + source, cleanup = _resolve_source(p) + if not source: + return { + "ok": False, + "error": ( + "Provide 'adf_volume_path' / 'adf_workspace_path' / 'adf_definitions' / 'adf_source_path'." + ), + } + discover_args = ["discover", "--adf-source-path", source, "--output-dir", output_dir] + if pipeline: + discover_args += ["--pipeline", pipeline] + discover_res = runner.run_adapter(discover_args) + steps["discover"] = _phase_result(discover_res, out, inventory=runner.summarize_inventory(out)) + if not discover_res.ok: + return {"ok": False, "status": "failed", "failed_phase": "discover", "steps": steps} + + convert_args = ["convert", "--output-dir", output_dir, "--adf-source-path", source] + if pipeline: + convert_args += ["--pipeline", pipeline] + convert_res = runner.run_adapter(convert_args) + steps["convert"] = _phase_result(convert_res, out, translation=runner.summarize_translation(out)) + if not convert_res.ok: + return {"ok": False, "status": "failed", "failed_phase": "convert", "steps": steps} + + # Interactive gate: on the first (answerless) call, hand the full option schema to the agent. + if interactive and not answers: + options_schema = _pending_options(_cmd_inspect({"report_path": report_path})) + if options_schema: + return { + "ok": True, + "status": "needs_input", + "pending_options": options_schema, + "report_path": report_path, + "output_dir": output_dir, + "steps": steps, + "message": ( + "Configuration options are available. Each option carries a `show_when` " + "condition (a list of {option_id, in:[values]} clauses; empty = always). Ask " + "only the options whose `show_when` clauses are all satisfied by the answers " + "collected so far, validating each answer against its `choices`. When the user " + "has answered every applicable option, call migrate again with the full " + "`answers` list (['option_id=value', ...]) to apply and package in one shot. To " + "accept defaults and skip prompting, call migrate with interactive=false." + ), + } + + # No (more) pending options: stamp the collected answers (if any) then package. + if answers: + apply_res = _cmd_apply_answers( + { + "report_path": report_path, + "answers": answers, + "output_dir": output_dir, + "lookup_csv": p.get("lookup_csv"), + } + ) + steps["apply_answers"] = apply_res + if not apply_res.get("ok"): + return {"ok": False, "status": "failed", "failed_phase": "apply_answers", "steps": steps} + + package_res = runner.run_adapter( + ["package", "--output-dir", output_dir, "--catalog", catalog, "--schema", schema] + ) + extra = _bundle_output(p, out) if package_res.ok else {} + steps["package"] = _phase_result(package_res, out, bundle_files=runner.list_tree(out), **extra) + return { + "ok": package_res.ok, + "status": "completed" if package_res.ok else "failed", + "failed_phase": None if package_res.ok else "package", + "steps": steps, + } + finally: + cleanup() + + +def _cmd_record_results(p: dict[str, Any]) -> dict[str, Any]: + args: list[Any] = ["record-results", "--output-dir", p["output_dir"], "--results-table", p["results_table"]] + if p.get("warehouse_id"): + args += ["--warehouse-id", p["warehouse_id"]] + result = runner.run_adapter(args) + return {"ok": result.ok, "process": result.as_dict()} + + +def _cmd_install_dashboard(p: dict[str, Any]) -> dict[str, Any]: + args: list[Any] = ["install-dashboard", "--results-table", p["results_table"]] + if p.get("warehouse_id"): + args += ["--warehouse-id", p["warehouse_id"]] + if p.get("dashboard_name"): + args += ["--dashboard-name", p["dashboard_name"]] + if p.get("parent_path"): + args += ["--parent-path", p["parent_path"]] + result = runner.run_adapter(args) + return {"ok": result.ok, "result": runner.parse_stdout_json(result), "process": result.as_dict()} + + +_COMMANDS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = { + "inputs": _cmd_inputs, + "discover": _cmd_discover, + "convert": _cmd_convert, + "merge_agentic": _cmd_merge_agentic, + "inspect": _cmd_inspect, + "apply_answers": _cmd_apply_answers, + "materialize_lookup": _cmd_materialize_lookup, + "workspace_paths": _cmd_workspace_paths, + "package": _cmd_package, + "migrate": _cmd_migrate, + "record_results": _cmd_record_results, + "install_dashboard": _cmd_install_dashboard, +} + + +def build_server() -> FastMCP: + """Construct and return the flowx :class:`FastMCP` server with the single dispatcher tool. + + ``stateless_http=True`` is required by Databricks Genie Code (no persistent ``Mcp-Session-Id`` + round-trip). ``streamable_http_path="/mcp"`` pins the transport to ``/mcp`` (Genie expects the + server at ``/mcp``). ``transport_security`` disables the SDK's DNS-rebinding Origin/Host + check (see :func:`_transport_security`). + """ + mcp = FastMCP( + "flowx", + instructions=_INSTRUCTIONS, + stateless_http=True, + streamable_http_path="/mcp", + transport_security=_transport_security(), + ) + + # structured_output=False: suppress the auto-derived outputSchema (Genie Code rejects tools that + # declare one); the dict is still returned as JSON text. See "MCP server design notes" in AGENTS.md. + @mcp.tool(structured_output=False) + def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, Any]: + """Run an flowx ADF→Databricks migration command. + + Call as ``flowx(command="", parameters={...})``. Commands and their + ``parameters`` keys (req = required; phases share ``output_dir``, default "./flowx_output"): + + - "inputs": phase(req: "discover"|"convert"|"package") — list a phase's input prompts. + - "discover": one of adf_volume_path | adf_workspace_path | adf_definitions | adf_source_path + (req), output_dir, pipeline — parse ADF JSON, classify activities. + - "convert": output_dir, (adf_volume_path | adf_workspace_path | adf_definitions | + adf_source_path), pipeline. + - "merge_agentic": report_path(req), agentic_results_dir(req), output_path — merge agent results. + - "inspect": report_path(req) — return the full translation-option schema (every option with + a `show_when` condition) for the agent to walk locally. See "Collecting options" below. + - "apply_answers": report_path(req), answers(req, list of "ID=VALUE"), output_dir, lookup_csv. + - "materialize_lookup": source(req: CSV path or literal CSV), out(req: destination JSON path). + - "workspace_paths": report_path(req), (adf_volume_path | adf_workspace_path | adf_definitions + | source_dir). + - "package": output_dir, output_volume_path, output_workspace_path, report_path, + catalog(default "main"), schema(default "default"), bundle_name, profile, + download_workspace_files(bool), keep_intermediates(bool). + - "migrate": one of adf_volume_path | adf_workspace_path | adf_definitions | adf_source_path + (req), output_dir, output_volume_path, output_workspace_path, catalog, schema, pipeline, + answers(list of "ID=VALUE"), interactive(bool, default true), lookup_csv — runs + discover→convert→package, returning the full option schema once (status "needs_input") when + configuration is available; re-call once with the complete answers to apply (see below). + - "record_results": output_dir(req), results_table(req: catalog.schema.table), warehouse_id. + - "install_dashboard": results_table(req), warehouse_id, dashboard_name, parent_path. + + Providing the ADF source (a hosted app can't read the user's workspace/volume files directly): + - ``adf_volume_path``: a UC Volume directory the server reads via the SDK Files API. **Preferred + for large factories** — the bytes never pass through the agent. Requires the app's service + principal to have read on the volume. + - ``adf_workspace_path``: a ``/Workspace`` directory (e.g. an ADF Git folder) the server reads + via the SDK Workspace API. Also scales (bytes bypass the agent); needs SP read on that path. + - ``adf_definitions``: an inline mapping of relative path → JSON content mirroring the ADF + Git-export layout, e.g. {"pipeline/Foo.json": {...}, "linkedService/Bar.json": {...}} (a single + ARM-template object is also accepted). Convenient for small jobs; capped (~5 MB) since it flows + through the agent's context — over the cap, switch to ``adf_volume_path``. + - ``adf_source_path`` / ``source_dir``: a path the server itself can read (local hosting / mounted volume). + + Delivering the generated DAB (the server's output_dir is local/ephemeral, so "package"/"migrate" + write it to the target via the SDK — the contents bypass the agent): + - Set ``output_volume_path`` to upload the bundle to a UC Volume (SDK Files API), or + ``output_workspace_path`` to upload it to a ``/Workspace`` directory (SDK Workspace API, files + written verbatim). Either returns ``bundle_uploaded`` = {"output_volume_path" or + "output_workspace_path", "files":[...], "count"}. **Preferred** — required for large bundles. + - With neither set, they return ``bundle`` = {"files": {relpath: text, ...}, "truncated": [...]} + inline for the agent to persist (small bundles only; capped). + + Collecting options (agent-driven chain): + - The server returns the **full option schema** in one shot — it never runs a multi-step + prompt loop itself. "inspect" (and "migrate" on its first, answerless call via + ``status="needs_input"``) returns ``pending_options`` = + ``[{"pipeline_name", "options":[{option_id, prompt, rationale, choices, free_text, default, + show_when}, ...]}, ...]``, where ``show_when`` is a list of ``{option_id, in:[values]}`` + clauses (empty = always shown). + - The **agent** drives the conversation locally: ask an option only when every ``show_when`` + clause is satisfied by the answers gathered so far (e.g. ``notify_slack_url`` shows once + ``notify_destination=slack``); validate each answer against ``choices`` (``free_text`` options + accept any value); perform any data action (e.g. run the lookup query when + ``metadata_driven_lookup_tool=have``). No round trip per follow-up. + - When every applicable option is answered, submit **once**: "migrate" re-called with the full + ``answers`` (applies + packages), or standalone "apply_answers" → "package". The server still + validates every answer at apply time. ``interactive=false`` on "migrate" skips prompting. + + Returns a dict ``{"ok": bool, ...}`` with per-command summaries (inventory / translation / + bundle_files / questions / result) and a "process" block (stdout/stderr/returncode). An unknown + command, missing required parameter, or an oversized inline payload returns + ``{"ok": false, "error": ...}``. + + Args: + command: The operation to run (see the list above). + parameters: Operation-specific keyword arguments. + """ + handler = _COMMANDS.get(command) + if handler is None: + return {"ok": False, "error": f"Unknown command {command!r}. Valid commands: {', '.join(_COMMANDS)}."} + try: + return handler(parameters or {}) + except KeyError as missing: + return {"ok": False, "error": f"Missing required parameter {missing} for command {command!r}."} + except ValueError as error: + return {"ok": False, "error": str(error)} + + return mcp + + +def build_http_app() -> Any: + """Builds the streamable-HTTP ASGI app for hosting (Databricks Apps / Genie Code). + + Returns: + FastMCP's own streamable-HTTP app, serving ``/mcp`` plus ``/`` and ``/health`` routes, with + CORS attached (origins from ``FLOWX_ALLOWED_ORIGINS``, default ``*``). + + Notes: + Returns FastMCP's *own* app rather than mounting it inside another Starlette app: mounting + drops the sub-app's lifespan, leaving the StreamableHTTP session manager uninitialized so + every ``/mcp`` request 500s. See the "MCP server design notes" in AGENTS.md. + """ + from starlette.middleware.cors import CORSMiddleware + from starlette.responses import JSONResponse + + mcp = build_server() + + @mcp.custom_route("/", methods=["GET"]) + async def health(_request: Any) -> JSONResponse: + return JSONResponse({"status": "ok", "service": "mcp-flowx"}) + + @mcp.custom_route("/health", methods=["GET"]) + async def health_alias(_request: Any) -> JSONResponse: + return JSONResponse({"status": "ok", "service": "mcp-flowx"}) + + # FastMCP's own app — its lifespan starts the StreamableHTTP session manager. + app = mcp.streamable_http_app() + + allow_origins = _allowed_origins() + # Credentialed requests cannot use the "*" wildcard per the CORS spec. + allow_credentials = allow_origins != ["*"] + app.add_middleware( + CORSMiddleware, + allow_origins=allow_origins, + allow_credentials=allow_credentials, + allow_methods=["*"], + allow_headers=["*"], + expose_headers=["Mcp-Session-Id"], + ) + return app + + +def serve() -> None: + """Entry point used by ``python -m flowx.mcp``. + + Defaults to stdio (local agents). With ``--http`` (or FLOWX_MCP_HTTP=1) it serves + the streamable-HTTP app via uvicorn on ``--port`` / ``$DATABRICKS_APP_PORT`` / 8000. + """ + import argparse + + parser = argparse.ArgumentParser(prog="python -m flowx.mcp", description="Run the flowx MCP server.") + parser.add_argument("--http", action="store_true", help="Serve over streamable HTTP instead of stdio.") + parser.add_argument("--host", default="0.0.0.0", help="Bind host for --http mode.") + parser.add_argument( + "--port", + type=int, + default=int(os.environ.get("DATABRICKS_APP_PORT", "8000")), + help="Bind port for --http mode (defaults to $DATABRICKS_APP_PORT or 8000).", + ) + args = parser.parse_args() + + if args.http or os.environ.get("FLOWX_MCP_HTTP") == "1": + import uvicorn + + uvicorn.run(build_http_app(), host=args.host, port=args.port) + else: + build_server().run(transport="stdio") diff --git a/src/orchestra/models/__init__.py b/src/flowx/models/__init__.py similarity index 100% rename from src/orchestra/models/__init__.py rename to src/flowx/models/__init__.py diff --git a/src/orchestra/models/adf_ast.py b/src/flowx/models/adf_ast.py similarity index 96% rename from src/orchestra/models/adf_ast.py rename to src/flowx/models/adf_ast.py index 9e2d2f9..3fa8d90 100644 --- a/src/orchestra/models/adf_ast.py +++ b/src/flowx/models/adf_ast.py @@ -149,6 +149,8 @@ class AdfActivity: if_true_activities: list[AdfActivity] | None = None if_false_activities: list[AdfActivity] | None = None activities: list[AdfActivity] | None = None # ForEach, Until + # Original ADF/ARM activity JSON, retained so agentic handlers can translate from the source. + raw: dict[str, Any] | None = None # --------------------------------------------------------------------------- @@ -167,6 +169,9 @@ class AdfPipeline: variables: Pipeline variable declarations, keyed by name. annotations: Free-form annotation strings attached to the pipeline. folder: Organisational folder path within the ADF workspace. + raw: Original ADF/ARM pipeline JSON as loaded from source, retained so + the discover phase can emit a verbatim ``.arm.json`` into the + bundle's metadata folder for provenance. """ name: str @@ -175,6 +180,7 @@ class AdfPipeline: variables: dict[str, AdfVariable] | None = None annotations: list[str] | None = None folder: str | None = None + raw: dict[str, Any] | None = None # --------------------------------------------------------------------------- diff --git a/src/orchestra/models/dab.py b/src/flowx/models/dab.py similarity index 100% rename from src/orchestra/models/dab.py rename to src/flowx/models/dab.py diff --git a/src/orchestra/models/ir.py b/src/flowx/models/ir.py similarity index 93% rename from src/orchestra/models/ir.py rename to src/flowx/models/ir.py index 2fd263c..1b272ba 100644 --- a/src/orchestra/models/ir.py +++ b/src/flowx/models/ir.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, TypeAlias if TYPE_CHECKING: - from flowx.adapter.models import TranslationPreferences + from flowx.adapter.models import TranslationConfiguration @dataclass(slots=True, kw_only=True) @@ -86,16 +86,14 @@ class Activity: cluster: dict[str, Any] | None = None existing_cluster_id: str | None = None libraries: list[dict[str, Any]] | None = None - # Approximate parameter substitutions made at translation time (e.g. - # ``utcnow()`` mapped to ``{{job.start_time.iso_datetime}}``). Each - # entry has keys ``widget_name``, ``raw_expression``, ``replacement``, - # and ``note``; the bundler surfaces these in SETUP.md. + # Parameter substitutions approximated at translation time (e.g. utcnow()); the bundler lists each + # entry (widget_name/raw_expression/replacement/note) in SETUP.md. parameter_approximations: list[dict[str, str]] = field(default_factory=list) required_parameters: dict[str, str] = field(default_factory=dict) - # Compute mode stamped by the pipeline modifier in response to user - # preferences. One of "serverless", "classic_single_node", - # "classic_multi_node", "inherit", or None when no preferences were applied. + # Compute mode stamped by the modifier: serverless | classic_single_node | classic_multi_node | inherit | None. compute_mode: str | None = None + # Collapsed activity_and_notify spec set by the adapter: {destination, events, args, destination_name}. + notifications: dict[str, Any] | None = None @dataclass(slots=True, kw_only=True) @@ -158,15 +156,11 @@ class CopyActivity(Activity): sink_format: str | None = None sink_resolved_path: str | None = None column_mapping: list[dict[str, str]] | None = None - # Code paradigm chosen by the pipeline modifier: "notebook" (default - # PySpark output) or "sdp" (Lakeflow Spark Declarative Pipeline). + # Code paradigm chosen by the modifier: "notebook" (PySpark) or "sdp" (Lakeflow SDP). target_format: str | None = None - # True when the modifier selected Lakeflow Connect for an eligible - # database-source Copy → Delta ingestion. + # True when the modifier selected Lakeflow Connect for an eligible database-source Copy -> Delta. use_lakeflow_connector: bool = False - # Lakeflow Connect connector flavour resolved by the modifier when - # use_lakeflow_connector is True: "query_based" or "cdc". None when - # the modifier did not stamp a connector type. + # LFC connector flavour when use_lakeflow_connector is True: "query_based" | "cdc" (None if unstamped). lakeflow_connector_type: str | None = None @@ -294,6 +288,11 @@ class WebActivity(Activity): authentication: dict[str, Any] | None = None disable_cert_validation: bool = False http_request_timeout_seconds: int | None = None + # Request body pre-lowered to a Python expression at translate time when it contained + # @-expressions; the code generator emits it verbatim instead of re-resolving. + body_code: str | None = None + body_imports: list[str] = field(default_factory=list) + body_required_parameters: dict[str, str] = field(default_factory=dict) @dataclass(slots=True, kw_only=True) @@ -483,6 +482,9 @@ class PlaceholderActivity(Activity): original_type: str notebook_path: str = "/UNSUPPORTED_ADF_ACTIVITY" comment: str | None = None + # For an agentic gap (e.g. Until): the recommended skill the agent should translate from. + agentic_skill: str | None = None + raw_definition: dict[str, Any] | None = None @dataclass(slots=True, kw_only=True) @@ -511,22 +513,13 @@ class MotifActivity(Activity): confidence_notes: list[str] = field(default_factory=list) original_activities: list[Activity] = field(default_factory=list) notebook_template: str | None = None - # Set by the pipeline modifier when the user opts into metadata-driven - # consolidation, has access to query the lookup source, and the - # configuration size is S or M. When True the preparer should emit - # a single consolidated pipeline whose objects come from lookup_values. + # Set by the modifier when the user opts into metadata-driven consolidation (access granted, size S/M). consolidate_metadata_driven: bool = False - # Concrete lookup rows materialised at translation time (CLI - # ``materialize-lookup`` subcommand or agent-supplied JSON). Each - # element is a dict mirroring a row from the original ADF Lookup - # query. Empty when consolidation is requested but values have not - # been resolved yet. + # Concrete lookup rows materialised at translation time; empty when consolidation was requested but + # the values have not been resolved yet. lookup_values: list[dict[str, Any]] = field(default_factory=list) - # Small dict of motif-specific settings extracted from the collapsed - # activities — e.g. ``{"lookup_query": ..., "lookup_scope": ...}`` for - # ``for_each_ingestion``. Used by the notebook generator so the motif - # can fetch its input list itself instead of requiring an ``items`` - # widget that has no upstream writer. + # Motif-specific settings from the collapsed activities (e.g. lookup_query/lookup_scope for + # for_each_ingestion) so the notebook generator can fetch its own input list. motif_config: dict[str, Any] = field(default_factory=dict) @@ -549,7 +542,7 @@ class Pipeline: tasks: list[Activity] = field(default_factory=list) tags: dict[str, str] = field(default_factory=dict) not_translatable: list[dict[str, Any]] = field(default_factory=list) - translation_preferences: TranslationPreferences | None = None + translation_configuration: TranslationConfiguration | None = None @dataclass(frozen=True, slots=True) diff --git a/src/orchestra/models/motifs.py b/src/flowx/models/motifs.py similarity index 93% rename from src/orchestra/models/motifs.py rename to src/flowx/models/motifs.py index 1bc85fa..d962947 100644 --- a/src/orchestra/models/motifs.py +++ b/src/flowx/models/motifs.py @@ -169,17 +169,17 @@ class DetectedMotif: notebook_template="staged_load.py", ) -MOTIF_COPY_AND_NOTIFY = MotifDefinition( - motif_id="copy_and_notify", - display_name="Copy and Notify", +MOTIF_ACTIVITY_AND_NOTIFY = MotifDefinition( + motif_id="activity_and_notify", + display_name="Activity and Notify", description=( - "A Copy activity followed by WebActivity calls for success/failure " - "notifications (Logic Apps, Slack, email). Translates to a notebook " - "task with built-in notification via job email/webhook settings." + "Any activity (Copy, Notebook, Lookup, stored procedure, …) followed by WebActivity " + "calls for success/failure notifications (Logic Apps, Slack, email). Collapses to the " + "upstream task with built-in notification via job email/webhook settings." ), - expected_activity_types=("Copy", "WebActivity"), - databricks_replacement="notebook_with_notification", - notebook_template="copy_and_notify.py", + expected_activity_types=("*", "WebActivity"), + databricks_replacement="task_with_notification", + notebook_template=None, ) MOTIF_LAKEFLOW_CONNECT_DATABASE = MotifDefinition( @@ -207,6 +207,6 @@ class DetectedMotif: MOTIF_FILE_EXISTENCE_VALIDATION, MOTIF_SCD_TYPE_2, MOTIF_STAGED_LOAD_SYNAPSE, - MOTIF_COPY_AND_NOTIFY, + MOTIF_ACTIVITY_AND_NOTIFY, MOTIF_LAKEFLOW_CONNECT_DATABASE, ) diff --git a/src/orchestra/models/source_types.py b/src/flowx/models/source_types.py similarity index 56% rename from src/orchestra/models/source_types.py rename to src/flowx/models/source_types.py index 8d815c1..383743c 100644 --- a/src/orchestra/models/source_types.py +++ b/src/flowx/models/source_types.py @@ -2,10 +2,7 @@ from __future__ import annotations -# Database-style sources reachable via JDBC. Every entry here implies the -# generated notebook will read with ``spark.read.format("jdbc")`` and -# require ``jdbc-url`` / ``jdbc-password`` (and optionally ``jdbc-user``) -# secrets. +# Database sources read via spark.read.format("jdbc") (need jdbc-url/jdbc-password/jdbc-user secrets). JDBC_SOURCE_TYPES: frozenset[str] = frozenset( { "AzureSqlSource", @@ -21,9 +18,7 @@ ) -# File-based sources that resolve to an object store location. These -# trigger UC volume / external-location provisioning and use Auto Loader -# (``cloudFiles``) for ingestion. +# Object-store file sources: trigger UC volume / external-location provisioning and Auto Loader ingestion. FILE_SOURCE_TYPES: frozenset[str] = frozenset( { "BlobSource", @@ -43,8 +38,6 @@ ) -# Paginated REST API sources -- handled by a generic ``requests``-based -# pagination loop in the generated copy notebook. ADF ``HttpSource`` -# is *not* in this set: it downloads a single file (CSV / JSON / -# Parquet) over HTTP and is handled as a FILE source via Auto Loader. +# Paginated REST API sources -- a generic requests-based pagination loop in the copy notebook. ADF +# HttpSource is NOT here: it downloads a single file over HTTP and is treated as a FILE source. REST_SOURCE_TYPES: frozenset[str] = frozenset({"RestSource"}) diff --git a/src/orchestra/motifs/__init__.py b/src/flowx/motifs/__init__.py similarity index 100% rename from src/orchestra/motifs/__init__.py rename to src/flowx/motifs/__init__.py diff --git a/src/orchestra/motifs/collapser.py b/src/flowx/motifs/collapser.py similarity index 91% rename from src/orchestra/motifs/collapser.py rename to src/flowx/motifs/collapser.py index e532e01..8dede0a 100644 --- a/src/orchestra/motifs/collapser.py +++ b/src/flowx/motifs/collapser.py @@ -34,11 +34,8 @@ def collapse_motifs( tasks_by_name: dict[str, Activity] = {task.name: task for task in pipeline.tasks} new_tasks: list[Activity] = [] - # Maps a *sanitised* task_key of a collapsed activity to the - # MotifActivity's task_key so ``_rewire_dependencies`` can match - # against ``Dependency.task_key`` (which is also sanitised). Keying - # by raw activity name here would silently fail to rewire any edge - # whose source had spaces or other characters in its name. + # Maps a collapsed activity's sanitised task_key to the MotifActivity's task_key so + # _rewire_dependencies can match Dependency.task_key (also sanitised); raw names would miss edges. motif_task_keys: dict[str, str] = {} inserted_motifs: set[str] = set() @@ -99,10 +96,7 @@ def _build_motif_activity( original_activities = [tasks_by_name[name] for name in motif.matched_activities if name in tasks_by_name] - # Use the sanitised task_keys (not raw activity names) for the - # internal-dependency check; ``Dependency.task_key`` is sanitised by - # the translator, so comparing against raw names would mis-classify - # any internal dep whose source name contained spaces / hyphens. + # Compare sanitised task_keys (Dependency.task_key is sanitised); raw names would mis-classify deps. matched_task_keys = {activity.task_key for activity in original_activities} external_deps = _collect_external_dependencies(original_activities, matched_task_keys) diff --git a/src/orchestra/motifs/detector.py b/src/flowx/motifs/detector.py similarity index 95% rename from src/orchestra/motifs/detector.py rename to src/flowx/motifs/detector.py index 2579501..76180a5 100644 --- a/src/orchestra/motifs/detector.py +++ b/src/flowx/motifs/detector.py @@ -11,8 +11,8 @@ AdfPipeline, ) from flowx.models.motifs import ( + MOTIF_ACTIVITY_AND_NOTIFY, MOTIF_CDC_CHANGE_TRACKING, - MOTIF_COPY_AND_NOTIFY, MOTIF_FILE_EXISTENCE_VALIDATION, MOTIF_FILE_LANDING_ZONE_PROCESSING, MOTIF_INCREMENTAL_LOAD_WATERMARK, @@ -105,7 +105,7 @@ def detect_motifs( (MOTIF_FILE_EXISTENCE_VALIDATION, _detect_file_existence_validation), (MOTIF_SCD_TYPE_2, _detect_scd_type_2), (MOTIF_STAGED_LOAD_SYNAPSE, _detect_staged_load_synapse), - (MOTIF_COPY_AND_NOTIFY, _detect_copy_and_notify), + (MOTIF_ACTIVITY_AND_NOTIFY, _detect_activity_and_notify), ] for motif_def, detector_fn in _detectors: @@ -358,13 +358,8 @@ def _detect_metadata_driven_bulk_copy( for_each_activities = _activities_of_type(activities, "ForEach", claimed) for for_each_activity in for_each_activities: - # Bulk-copy motif requires the inner body to *be* the Copy: a single - # Copy child, with no other transform / orchestration activity in the - # loop body. Patterns like Notebook -> Copy or BuildReport -> Export - # are not bulk-copy motifs even when an upstream Lookup is present; - # they are generic "build then archive" pipelines and the user almost - # never wants the Copy collapsed into a metadata-driven ingestion - # template that ignores the upstream notebook work. + # Bulk-copy requires the ForEach body to *be* a single Copy with no other activity; a + # Notebook->Copy or build-then-archive loop is not this motif even with an upstream Lookup. inner_activities = list(for_each_activity.activities or []) if len(inner_activities) != 1 or inner_activities[0].type != "Copy": continue @@ -476,22 +471,27 @@ def _detect_file_landing_zone( return results -def _detect_copy_and_notify( +def _detect_activity_and_notify( activities: list[AdfActivity], by_name: dict[str, AdfActivity], definitions: AdfDefinitions, claimed: set[str], ) -> list[DetectedMotif]: - """Detects copy-and-notify pattern.""" + """Detects the activity-and-notify pattern: any activity followed by notification Web calls. + + Generalised beyond Copy -- any upstream activity (Copy, Notebook, Lookup, stored procedure, …) + that is directly followed by a WebActivity which looks like a notification (Logic Apps / email / + Slack / Teams / webhook keywords, or a success/failure-conditioned dependency) is reported. + """ results: list[DetectedMotif] = [] - copies = _activities_of_type(activities, "Copy", claimed) + upstream_acts = [a for a in activities if a.type != "WebActivity" and a.name not in claimed] - for copy_act in copies: + for upstream_act in upstream_acts: downstream_webs: list[AdfActivity] = [] for activity in activities: if activity.name in claimed: continue - if activity.type == "WebActivity" and _depends_on(activity, copy_act.name): + if activity.type == "WebActivity" and _depends_on(activity, upstream_act.name): downstream_webs.append(activity) if not downstream_webs: @@ -509,27 +509,28 @@ def _detect_copy_and_notify( if not notification_found: # If there is no notification hint, we still accept if the Web - # activity depends on Copy with success/failure conditions + # activity depends on the upstream with success/failure conditions for web in downstream_webs: if web.depends_on: for dep in web.depends_on: - if dep.activity == copy_act.name and dep.dependency_conditions: + if dep.activity == upstream_act.name and dep.dependency_conditions: conds = [cond.lower() for cond in dep.dependency_conditions] if "failed" in conds or "completed" in conds: notification_found = True notes.append( - f"WebActivity '{web.name}' triggers on {dep.dependency_conditions} of Copy" + f"WebActivity '{web.name}' triggers on " + f"{dep.dependency_conditions} of '{upstream_act.name}'" ) if not notification_found: continue - matched = [copy_act.name] + [web.name for web in downstream_webs] - source_hint = _infer_source_type(copy_act, definitions) + matched = [upstream_act.name] + [web.name for web in downstream_webs] + source_hint = _infer_source_type(upstream_act, definitions) _record_motif( results, - definition=MOTIF_COPY_AND_NOTIFY, + definition=MOTIF_ACTIVITY_AND_NOTIFY, matched_activities=matched, source_type_hint=source_hint, confidence_notes=notes, diff --git a/src/orchestra/parser/__init__.py b/src/flowx/parser/__init__.py similarity index 100% rename from src/orchestra/parser/__init__.py rename to src/flowx/parser/__init__.py diff --git a/src/orchestra/parser/adf_loader.py b/src/flowx/parser/adf_loader.py similarity index 66% rename from src/orchestra/parser/adf_loader.py rename to src/flowx/parser/adf_loader.py index acc3ed9..30c7c6e 100644 --- a/src/orchestra/parser/adf_loader.py +++ b/src/flowx/parser/adf_loader.py @@ -1,10 +1,13 @@ -"""Loads ADF JSON files from a directory structure and produce typed AST objects.""" +"""Loads ADF JSON exports from a directory tree and produces typed AST objects (``AdfDefinitions``).""" from __future__ import annotations import argparse +import csv import json import logging +import re +import shutil from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -67,6 +70,20 @@ "Script": "adf-to-databricks:adf-pipeline-converter", } +# Activity complexity weights (easiest first): Databricks-native ~1:1 tasks, then control-flow, then +# everything else (Copy/Web/Lookup/data movement/agentic) -- summed into each pipeline's complexity score. +_DATABRICKS_NATIVE_TYPES: frozenset[str] = frozenset( + {"DatabricksNotebook", "DatabricksSparkJar", "DatabricksSparkPython", "DatabricksJob"} +) +_CONTROL_FLOW_TYPES: frozenset[str] = frozenset( + {"ForEach", "IfCondition", "Switch", "SetVariable", "AppendVariable", "Filter", "Wait", "Until"} +) +_ACTIVITY_WEIGHT: dict[str, int] = {"databricks": 1, "control": 2, "other": 3} + +# Complexity-score -> T-shirt size cutoffs (inclusive upper bounds). Score is +# sum(activity weights) + #datasets + #linked_services + #collapsible_patterns. +_TSHIRT_CUTOFFS: tuple[tuple[int, str], ...] = ((5, "S"), (15, "M"), (30, "L")) + # --------------------------------------------------------------------------- # Public API @@ -249,7 +266,7 @@ def _find_json_dir(source_dir: Path, *candidate_names: str) -> Path | None: if candidate.is_dir(): return candidate # Case-insensitive fallback - lower_candidates = {n.lower() for n in candidate_names} + lower_candidates = {candidate_name.lower() for candidate_name in candidate_names} for child in source_dir.iterdir(): if child.is_dir() and child.name.lower() in lower_candidates: return child @@ -266,12 +283,13 @@ def _parse_pipeline_json(data: dict[str, Any], *, fallback_name: str = "unknown" Returns: Parsed :class:`AdfPipeline`. """ + raw_source = data data = _normalize_arm(data) props = data.get("properties", data) name = data.get("name") or props.get("name") or fallback_name activities_raw: list[dict[str, Any]] = props.get("activities", []) - activities = [parse_activity(a) for a in activities_raw] + activities = [parse_activity(raw_activity) for raw_activity in activities_raw] parameters: dict[str, AdfParameter] | None = None raw_params = props.get("parameters") @@ -310,6 +328,7 @@ def _parse_pipeline_json(data: dict[str, Any], *, fallback_name: str = "unknown" variables=variables, annotations=annotations, folder=folder, + raw=raw_source, ) @@ -372,13 +391,13 @@ def parse_activity(data: dict[str, Any]) -> AdfActivity: if type_properties: raw_if_true = type_properties.get("ifTrueActivities") if raw_if_true: - if_true_activities = [parse_activity(a) for a in raw_if_true] + if_true_activities = [parse_activity(raw_activity) for raw_activity in raw_if_true] raw_if_false = type_properties.get("ifFalseActivities") if raw_if_false: - if_false_activities = [parse_activity(a) for a in raw_if_false] + if_false_activities = [parse_activity(raw_activity) for raw_activity in raw_if_false] raw_children = type_properties.get("activities") if raw_children: - child_activities = [parse_activity(a) for a in raw_children] + child_activities = [parse_activity(raw_activity) for raw_activity in raw_children] return AdfActivity( name=name, @@ -392,6 +411,7 @@ def parse_activity(data: dict[str, Any]) -> AdfActivity: if_true_activities=if_true_activities, if_false_activities=if_false_activities, activities=child_activities, + raw=data, ) @@ -635,7 +655,7 @@ def _classify_activities( """ for activity in activities: strategy, skill = classify_activity(activity.type) - dep_names = [d.activity for d in activity.depends_on] if activity.depends_on else None + dep_names = [dependency.activity for dependency in activity.depends_on] if activity.depends_on else None items.append( InventoryItem( @@ -702,39 +722,286 @@ def _inventory_to_dict(inventory: Inventory, source_dir: str) -> dict[str, Any]: } +# --------------------------------------------------------------------------- +# Profile complexity report (CSV) +# --------------------------------------------------------------------------- + + +def _walk_activities(activities: list[AdfActivity]): + """Yields every activity in *activities*, descending into container children.""" + for activity in activities: + yield activity + for child in (activity.if_true_activities, activity.if_false_activities, activity.activities): + if child: + yield from _walk_activities(child) + + +def _activity_category(activity_type: str) -> str: + """Returns the complexity category of *activity_type*. + + One of ``"databricks"`` (native, simplest), ``"control"`` (control-flow / + parameter-setting), or ``"other"`` (data movement, web, agentic, ...). + """ + if activity_type in _DATABRICKS_NATIVE_TYPES: + return "databricks" + if activity_type in _CONTROL_FLOW_TYPES: + return "control" + return "other" + + +def _dataset_refs_for_activity(activity: AdfActivity) -> set[str]: + """Collects dataset names referenced by a single activity. + + Looks at the activity's ``inputs``/``outputs`` plus the ``dataset`` / + ``datasets`` references some activity types (Lookup, Delete, GetMetadata) + carry inside ``typeProperties``. + """ + names: set[str] = set() + for ref in (activity.inputs or []) + (activity.outputs or []): + if ref.reference_name: + names.add(ref.reference_name) + props = activity.type_properties or {} + for key in ("dataset", "source", "sink"): + candidate = props.get(key) + if isinstance(candidate, dict) and candidate.get("referenceName"): + names.add(candidate["referenceName"]) + return names + + +def _pipeline_reference_counts( + pipeline: AdfPipeline, definitions: AdfDefinitions +) -> tuple[int, set[str], set[str], dict[str, int]]: + """Returns ``(activity_count, dataset_names, linked_service_names, category_counts)``. + + Linked services are attributed both from activity-level references (e.g. + DatabricksNotebook compute) and transitively via the datasets a pipeline + touches (each dataset names its backing linked service). + """ + dataset_names: set[str] = set() + linked_service_names: set[str] = set() + category_counts = {"databricks": 0, "control": 0, "other": 0} + activity_count = 0 + for activity in _walk_activities(pipeline.activities): + activity_count += 1 + category_counts[_activity_category(activity.type)] += 1 + dataset_names |= _dataset_refs_for_activity(activity) + if activity.linked_service_name and activity.linked_service_name.reference_name: + linked_service_names.add(activity.linked_service_name.reference_name) + for dataset_name in dataset_names: + dataset = definitions.datasets.get(dataset_name) + if dataset and dataset.linked_service_name: + linked_service_names.add(dataset.linked_service_name) + return activity_count, dataset_names, linked_service_names, category_counts + + +def _complexity_score(category_counts: dict[str, int], n_datasets: int, n_linked: int, n_patterns: int) -> int: + """Weighted complexity score: activity weights + datasets + linked services + patterns.""" + weighted = sum(category_counts[cat] * _ACTIVITY_WEIGHT[cat] for cat in category_counts) + return weighted + n_datasets + n_linked + n_patterns + + +def _tshirt_size(score: int) -> str: + """Maps a complexity score to a T-shirt size (S / M / L / XL).""" + for cutoff, size in _TSHIRT_CUTOFFS: + if score <= cutoff: + return size + return "XL" + + +def build_profile_rows(definitions: AdfDefinitions) -> list[dict[str, Any]]: + """Builds one profile-report row per pipeline. + + Each row carries the source activity / dataset / linked-service counts, the + number of collapsible motif patterns detected, and a weighted complexity + score plus its T-shirt size. + + Args: + definitions: Parsed ADF definitions. + + Returns: + List of row dicts ordered by pipeline name. + """ + from flowx.motifs.detector import detect_motifs + + rows: list[dict[str, Any]] = [] + for pipeline in sorted(definitions.pipelines, key=lambda p: p.name): + activity_count, datasets, linked_services, category_counts = _pipeline_reference_counts(pipeline, definitions) + try: + n_patterns = len(detect_motifs(pipeline, definitions)) + except Exception as exc: # noqa: BLE001 - profiling must never hard-fail on motif detection + logger.warning("Motif detection failed for pipeline %r: %s", pipeline.name, exc) + n_patterns = 0 + score = _complexity_score(category_counts, len(datasets), len(linked_services), n_patterns) + rows.append( + { + "pipeline": pipeline.name, + "activities": activity_count, + "datasets": len(datasets), + "linked_services": len(linked_services), + "collapsible_patterns": n_patterns, + "databricks_native_activities": category_counts["databricks"], + "control_flow_activities": category_counts["control"], + "other_activities": category_counts["other"], + "complexity_score": score, + "complexity_size": _tshirt_size(score), + } + ) + return rows + + +_PROFILE_CSV_COLUMNS: tuple[str, ...] = ( + "pipeline", + "activities", + "datasets", + "linked_services", + "collapsible_patterns", + "databricks_native_activities", + "control_flow_activities", + "other_activities", + "complexity_score", + "complexity_size", +) + + +def write_profile_csv(rows: list[dict[str, Any]], path: Path) -> None: + """Writes the per-pipeline profile rows to *path* as CSV.""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=list(_PROFILE_CSV_COLUMNS)) + writer.writeheader() + for row in rows: + writer.writerow(row) + + +def _sanitize_filename(name: str) -> str: + """Slugifies a pipeline name into a safe filename stem.""" + slug = re.sub(r"[^0-9A-Za-z._-]+", "_", name).strip("_") + return slug or "pipeline" + + +def write_pipeline_arm(definitions: AdfDefinitions, metadata_dir: Path) -> list[Path]: + """Writes each pipeline's original ARM JSON to ``metadata_dir/.arm.json``. + + Returns the list of written paths. Pipelines whose source JSON was not + retained are skipped (should not happen for parsed sources). + """ + metadata_dir.mkdir(parents=True, exist_ok=True) + written: list[Path] = [] + for pipeline in definitions.pipelines: + if pipeline.raw is None: + logger.warning("No source ARM JSON retained for pipeline %r; skipping arm export.", pipeline.name) + continue + arm_path = metadata_dir / f"{_sanitize_filename(pipeline.name)}.arm.json" + arm_path.write_text(json.dumps(pipeline.raw, indent=2), encoding="utf-8") + written.append(arm_path) + return written + + # --------------------------------------------------------------------------- # CLI entry point # --------------------------------------------------------------------------- +# Flowx-managed entries under the shared output_dir, cleared at the start of each fresh run. +_MANAGED_OUTPUT_DIRS: tuple[str, ...] = ("metadata", ".work", "resources", "src", "setup") +_MANAGED_OUTPUT_FILES: tuple[str, ...] = ("databricks.yml", "SETUP.md", "WARNINGS.md") -if __name__ == "__main__": + +def clear_stale_outputs(output_dir: Path) -> None: + """Removes a prior run's artifacts from a reused ``output_dir``. + + Discover begins a fresh migration, so a previous run's per-pipeline metadata, transient + intermediates, and generated bundle must not survive into this run. Without this, a + single-pipeline migration into a reused output directory ships the earlier run's other + pipelines' source ARM and generated notebooks in the packaged bundle. + + Args: + output_dir: Migration output directory shared by all three phases. + + Notes: + Only flowx-managed entries are removed (never the directory itself or unrelated + files), so pointing ``output_dir`` at a populated directory stays safe. + """ + for directory in _MANAGED_OUTPUT_DIRS: + shutil.rmtree(output_dir / directory, ignore_errors=True) + for filename in _MANAGED_OUTPUT_FILES: + (output_dir / filename).unlink(missing_ok=True) + + +def main(argv: list[str] | None = None) -> int: + """Discover-phase entry point: load ADF, build the inventory + profile report. + + Exposed as a callable (not just an ``if __name__`` block) so the adapter can run the phase + in-process instead of spawning a second interpreter. Clears any prior run's artifacts from the + shared output directory first, so a reused ``output_dir`` never leaks stale pipelines into the + bundle this run packages. + """ parser = argparse.ArgumentParser(description="Load ADF definitions and build a translation inventory.") parser.add_argument("--source-dir", required=True, type=Path, help="Root directory containing ADF JSON exports.") parser.add_argument( "--output-dir", type=Path, - default=Path("./orchestra_output/ingest"), - help="Directory to write inventory.json into.", + default=Path("./flowx_output"), + help=( + "Migration output directory. Profile artifacts are written into its " + "metadata/ subfolder (inventory.json, profile_report.csv, .arm.json)." + ), + ) + parser.add_argument( + "--pipeline", + type=str, + default=None, + help="Filter to a single pipeline by name. When omitted, all pipelines are included.", ) - args = parser.parse_args() + args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") definitions = load_adf_definitions(args.source_dir) logger.info("Loaded %d pipeline(s) from %s", len(definitions.pipelines), args.source_dir) + # Filter to a single pipeline when --pipeline is specified + if args.pipeline: + matched = [pipeline for pipeline in definitions.pipelines if pipeline.name == args.pipeline] + if not matched: + available = [pipeline.name for pipeline in definitions.pipelines] + logger.error( + "Pipeline %r not found. Available pipelines: %s", + args.pipeline, + ", ".join(available) or "(none)", + ) + return 1 + definitions = AdfDefinitions( + pipelines=matched, + datasets=definitions.datasets, + linked_services=definitions.linked_services, + triggers=definitions.triggers, + global_parameters=definitions.global_parameters, + ) + logger.info("Filtered to pipeline: %s", args.pipeline) + inventory = build_inventory(definitions) output_dir: Path = args.output_dir.resolve() - output_dir.mkdir(parents=True, exist_ok=True) - inventory_path = output_dir / "inventory.json" + clear_stale_outputs(output_dir) + metadata_dir = output_dir / "metadata" + metadata_dir.mkdir(parents=True, exist_ok=True) + + inventory_path = metadata_dir / "inventory.json" inventory_dict = _inventory_to_dict(inventory, str(args.source_dir)) inventory_path.write_text(json.dumps(inventory_dict, indent=2), encoding="utf-8") logger.info("Wrote inventory to %s", inventory_path) + profile_rows = build_profile_rows(definitions) + csv_path = metadata_dir / "profile_report.csv" + write_profile_csv(profile_rows, csv_path) + logger.info("Wrote profile report to %s", csv_path) + + arm_paths = write_pipeline_arm(definitions, metadata_dir) + logger.info("Wrote %d pipeline ARM JSON file(s) to %s", len(arm_paths), metadata_dir) + summary = inventory_dict["summary"] - print("\nADF Ingestion Summary") - print("=====================") + print("\nADF Profile Summary") + print("===================") print(f"Pipelines parsed: {summary['pipeline_count']}") print(f"Total activities: {summary['activity_count']}") print("\nStrategy Breakdown:") @@ -742,3 +1009,16 @@ def _inventory_to_dict(inventory: Inventory, source_dir: str) -> dict[str, Any]: print(f" Agentic: {summary['agentic_count']}") print(f" Unsupported: {summary['unsupported_count']}") print(f"\nCoverage: {summary['coverage_pct']}%") + print("\nComplexity by pipeline (metadata/profile_report.csv):") + print(f" {'pipeline':<32} {'acts':>4} {'ds':>3} {'ls':>3} {'patt':>4} {'score':>5} size") + for row in profile_rows: + print( + f" {row['pipeline'][:32]:<32} {row['activities']:>4} {row['datasets']:>3} " + f"{row['linked_services']:>3} {row['collapsible_patterns']:>4} " + f"{row['complexity_score']:>5} {row['complexity_size']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/orchestra/parser/expression_parser.py b/src/flowx/parser/expression_parser.py similarity index 93% rename from src/orchestra/parser/expression_parser.py rename to src/flowx/parser/expression_parser.py index 7ca884b..2bec906 100644 --- a/src/orchestra/parser/expression_parser.py +++ b/src/flowx/parser/expression_parser.py @@ -10,9 +10,8 @@ _ITEM_RE = re.compile(r"item\(\s*\)$", re.IGNORECASE) -# C-35 (CF4-004): anchor the end-of-string so multi-segment chains like -# ``item().condition.name`` don't match here and silently drop the trailing -# ``.name`` (the previous behaviour mapped to ``{{input.condition}}``). +# C-35 (CF4-004): anchored to end-of-string so multi-segment chains like item().condition.name don't +# match here and silently drop the trailing .name. _ITEM_FIELD_RE = re.compile(r"item\(\s*\)\.(\w+)\s*$", re.IGNORECASE) _ACTIVITY_OUTPUT_RE = re.compile( @@ -89,11 +88,8 @@ re.IGNORECASE | re.DOTALL, ) -# Function names that are no-op wrappers when they appear at the outermost -# position around a single deterministic parameter / variable reference. -# Stripping these lets resolve_expression reach the underlying ref instead -# of falling through to notebook_code for trivial @json(pipeline().parameters.X) -# style wrappers commonly used in ADF for type coercion. +# No-op wrapper function names: stripping them at the outermost position lets resolve_expression reach +# the underlying parameter/variable ref instead of falling through to notebook_code (e.g. @json(...) coercion). _NOOP_WRAPPER_NAMES: frozenset[str] = frozenset({"json", "string", "array"}) _DATETIME_IMPORTS = ["from datetime import datetime, timezone, timedelta"] @@ -134,9 +130,7 @@ def resolve_expression( return None if isinstance(value, bool): - # VAREX3-002: render Python bool as lowercase 'true'/'false' so - # downstream ADF comparisons like @equals(variables('X'), true) - # match ADF's lowercase boolean tokens. + # VAREX3-002: render Python bool as lowercase true/false so @equals(variables('X'), true) matches ADF. return ExpressionResult(kind="literal", value="true" if value else "false") if isinstance(value, (int, float)): return ExpressionResult(kind="literal", value=str(value)) @@ -149,10 +143,8 @@ def resolve_expression( expr = value[1:].rstrip() # strip leading @ and trailing whitespace/newlines - # Strip no-op wrappers like @json(pipeline().parameters.X) so the inner - # ref resolves to its DAB dynamic value. We only unwrap when the inner - # expression itself resolves cleanly (literal / dab_ref) so we don't - # eat the wrapper's semantics where it actually matters. + # Strip no-op wrappers like @json(pipeline().parameters.X) only when the inner ref resolves cleanly + # (literal/dab_ref), so we don't eat the wrapper's semantics where it matters. unwrapped = _unwrap_noop_call(expr, context, variable_task_keys=variable_task_keys) if unwrapped is not None: return unwrapped @@ -205,17 +197,13 @@ def resolve_expression( if result is not None: return result - # CF3-004 / fix-attribute-access-on-function-results: handle - # ``....`` chains like - # ``json(pipeline().parameters.items).type`` by resolving the function - # call first then chaining `.get('attr')` onto the resulting code. + # CF3-004: handle . chains (e.g. json(pipeline().parameters.items).type) + # by resolving the call first, then chaining .get('attr') onto the result. result = _resolve_function_call_with_attribute(expr, context, variable_task_keys=variable_task_keys) if result is not None: return result - # C-33 (VAREX4-001): handle ``[N]`` chains so e.g. - # ``@split(pipeline().parameters.referenceDate,'/')[0]`` lowers to - # notebook_code. + # C-33 (VAREX4-001): handle [N] chains, e.g. @split(...,'/')[0], lowering to notebook_code. result = _resolve_function_call_with_index(expr, context, variable_task_keys=variable_task_keys) if result is not None: return result @@ -413,12 +401,8 @@ def _resolve_item_safe_nav(expr: str) -> ExpressionResult | None: segments: list[tuple[str, str]] = re.findall(r"(\??\.)(\w+)", chain) if not segments: return None - # If the chain has no safe-nav operator at all (purely ``item().a.b``) - # AND only one segment, defer to _ITEM_FIELD_RE's dab_ref path. - # C-35 (CF4-004): multi-segment pure-dotted chains like - # ``item().condition.name`` must lower to notebook_code so the - # downstream consumers can walk both segments instead of mapping to - # ``{{input.condition}}`` and silently dropping ``.name``. + # Single-segment pure-dotted item() chains defer to _ITEM_FIELD_RE's dab_ref path; multi-segment + # ones (C-35: item().condition.name) must lower to notebook_code so both segments survive. has_safe_nav = any(op == "?." for op, _ in segments) if not has_safe_nav and len(segments) < 2: return None @@ -537,9 +521,8 @@ def _resolve_variable( "between job start and the moment the activity actually runs." ) -# ADF .NET-style format strings that map cleanly onto a Databricks dynamic -# value reference. Anything not in this table falls back to a notebook_code -# strftime call. +# ADF .NET-style format strings that map cleanly onto a DAB dynamic value; anything else falls back to +# a notebook_code strftime call. _UTCNOW_FORMAT_TO_DAB_REF: dict[str, str] = { "yyyy-MM-dd": "{{job.start_time.iso_date}}", "yyyy-MM-ddTHH:mm:ss": "{{job.start_time.iso_datetime}}", @@ -648,9 +631,8 @@ def _resolve_concat( if not code_parts: return None - # If every part collapsed to a literal value, fold the whole concat into - # a single literal so downstream consumers (notebook library install, - # cluster fields, etc.) get a plain string instead of Python source. + # If every part collapsed to a literal, fold the concat into one literal so downstream consumers + # get a plain string instead of Python source. if all_literal: return ExpressionResult(kind="literal", value="".join(literal_parts)) @@ -727,23 +709,15 @@ def _split_args(inner: str) -> list[str]: _FUNCTION_CALL_WITH_ATTRIBUTE_RE = re.compile( - # Captures `funcName(args).attr.attr...` -- the trailing attribute chain - # must end with a word character so we don't accidentally swallow other - # closing parens / spaces. Used to lower - # ``json(pipeline().parameters.items).type`` to a notebook_code expression - # since the bare function dispatcher requires the function call to be the - # outermost token. + # Captures funcName(args).attr.attr... (trailing chain ends with a word char) to lower e.g. + # json(pipeline().parameters.items).type to notebook_code, since the bare dispatcher needs the call outermost. r"^([a-zA-Z_]\w*)\((.*)\)((?:\.\w+)+)\s*$", re.IGNORECASE | re.DOTALL, ) _FUNCTION_CALL_WITH_INDEX_RE = re.compile( - # C-33 (VAREX4-001): ``funcName(args)[N]`` — captures a trailing - # integer subscript so ``split(...)[0]`` and similar ADF expressions - # lower to notebook_code (the bare dispatcher only matched when the - # function call was the outermost token). We support a single - # numeric subscript for now; nested chains (``...[0][1]``) fall - # through to the legacy unsupported path. + # C-33 (VAREX4-001): captures funcName(args)[N] so split(...)[0] lowers to notebook_code (single + # numeric subscript only; nested ...[0][1] falls through to the unsupported path). r"^([a-zA-Z_]\w*)\((.*)\)\[\s*(-?\d+)\s*\]\s*$", re.IGNORECASE | re.DOTALL, ) @@ -865,20 +839,14 @@ def _resolve_function_call( continue if (raw_arg.startswith("'") and raw_arg.endswith("'")) or (raw_arg.startswith('"') and raw_arg.endswith('"')): - # C-34 (VAREX4-002): preserve the quotedness so the codegen - # downstream emits ``repr(value)`` rather than a bare token — - # otherwise quoted ``'09'`` / ``'12'`` collapse to a bare - # numeric and either raise a SyntaxError (leading zero) or - # silently compare against the wrong value. + # C-34 (VAREX4-002): preserve quotedness so codegen emits repr(value); else quoted '09'/'12' + # collapse to a bare numeric and raise (leading zero) or compare against the wrong value. resolved_args.append(ExpressionResult(kind="literal", value=raw_arg[1:-1], was_string_literal=True)) elif _is_numeric(raw_arg): resolved_args.append(ExpressionResult(kind="literal", value=raw_arg)) elif raw_arg.lower() in ("true", "false"): - # C-34 (VAREX4-003): ADF Booleans (``true`` / ``false``) match - # lowercase strings on the SetVariable consumer side (C-21). - # Mark the literal so ``_arg_to_code`` emits ``'true'`` / - # ``'false'`` strings rather than the bare Python ``True`` / - # ``False`` (whose ``str()`` is title-case and never matches). + # C-34 (VAREX4-003): mark ADF booleans so _arg_to_code emits 'true'/'false' strings (matching + # the SetVariable consumer, C-21) rather than Python True/False whose str() is title-case. resolved_args.append( ExpressionResult( kind="literal", @@ -896,9 +864,8 @@ def _resolve_function_call( resolved_args.append(sub_result) handler_result = handler(resolved_args) - # Auto-propagate required_parameters from args onto notebook_code results - # so preparers can thread DAB refs into base_parameters even for handlers - # that pre-date the required_parameters contract. + # Auto-propagate required_parameters from args onto notebook_code results so preparers can thread + # DAB refs into base_parameters even for handlers predating the required_parameters contract. if handler_result is not None and handler_result.kind == "notebook_code": extra_parameters = _collect_required_parameters(*resolved_args) if extra_parameters: @@ -1022,9 +989,9 @@ def _handle_concat(args: list[ExpressionResult]) -> ExpressionResult | None: """ if not args: return None - if all(a.kind == "literal" for a in args): - return ExpressionResult(kind="literal", value="".join(a.value for a in args)) - parts = [f"str({_arg_to_code(a)})" for a in args] + if all(arg.kind == "literal" for arg in args): + return ExpressionResult(kind="literal", value="".join(arg.value for arg in args)) + parts = [f"str({_arg_to_code(arg)})" for arg in args] return _result_from_args(" + ".join(parts), args) @@ -1158,7 +1125,7 @@ def _handle_intersection(args: list[ExpressionResult]) -> ExpressionResult | Non """intersection(c1, c2, ...) -> list(set(c1) & set(c2) & ...)""" if len(args) < 2: return None - parts = " & ".join(f"set({_arg_to_code(a)})" for a in args) + parts = " & ".join(f"set({_arg_to_code(arg)})" for arg in args) return _result_from_args(f"list({parts})", args) @@ -1201,7 +1168,7 @@ def _handle_union(args: list[ExpressionResult]) -> ExpressionResult | None: """union(c1, c2, ...) -> list(set(c1) | set(c2) | ...)""" if len(args) < 2: return None - parts = " | ".join(f"set({_arg_to_code(a)})" for a in args) + parts = " | ".join(f"set({_arg_to_code(arg)})" for arg in args) return _result_from_args(f"list({parts})", args) @@ -1314,13 +1281,13 @@ def _handle_coalesce(args: list[ExpressionResult]) -> ExpressionResult | None: """coalesce(a, b, ...) -> next((x for x in [a, b, ...] if x is not None), None)""" if not args: return None - items = ", ".join(_arg_to_code(a) for a in args) + items = ", ".join(_arg_to_code(arg) for arg in args) return _result_from_args(f"next((x for x in [{items}] if x is not None), None)", args) def _handle_create_array(args: list[ExpressionResult]) -> ExpressionResult | None: """createArray(a, b, ...) -> [a, b, ...]""" - items = ", ".join(_arg_to_code(a) for a in args) + items = ", ".join(_arg_to_code(arg) for arg in args) return _result_from_args(f"[{items}]", args) @@ -1394,7 +1361,7 @@ def _handle_max(args: list[ExpressionResult]) -> ExpressionResult | None: """max(a, b, ...) -> max(a, b, ...)""" if not args: return None - items = ", ".join(_arg_to_code(a) for a in args) + items = ", ".join(_arg_to_code(arg) for arg in args) return _result_from_args(f"max({items})", args) @@ -1402,7 +1369,7 @@ def _handle_min(args: list[ExpressionResult]) -> ExpressionResult | None: """min(a, b, ...) -> min(a, b, ...)""" if not args: return None - items = ", ".join(_arg_to_code(a) for a in args) + items = ", ".join(_arg_to_code(arg) for arg in args) return _result_from_args(f"min({items})", args) @@ -1783,5 +1750,5 @@ def _handle_ticks(args: list[ExpressionResult]) -> ExpressionResult | None: } _FUNCTION_HANDLERS_CI: dict[str, Callable[[list[ExpressionResult]], ExpressionResult | None]] = { - k.lower(): v for k, v in _FUNCTION_HANDLERS.items() if v is not None + name.lower(): handler for name, handler in _FUNCTION_HANDLERS.items() if handler is not None } diff --git a/src/orchestra/parser/ir_rewriter.py b/src/flowx/parser/ir_rewriter.py similarity index 96% rename from src/orchestra/parser/ir_rewriter.py rename to src/flowx/parser/ir_rewriter.py index b3cc0a5..0fc536f 100644 --- a/src/orchestra/parser/ir_rewriter.py +++ b/src/flowx/parser/ir_rewriter.py @@ -166,18 +166,18 @@ def _rewrite_activity(activity: Activity, context: TranslationContext, warnings: flow activities have their inner branches recursed into. """ field_overrides: dict[str, Any] = {} - for f in dataclasses.fields(activity): - if f.name in _FIELDS_TO_SKIP: + for field_info in dataclasses.fields(activity): + if field_info.name in _FIELDS_TO_SKIP: continue - original = getattr(activity, f.name) + original = getattr(activity, field_info.name) rewritten = _rewrite_value( original, context, warnings, - field_path=f"{type(activity).__name__}.{activity.task_key}.{f.name}", + field_path=f"{type(activity).__name__}.{activity.task_key}.{field_info.name}", ) if rewritten is not original: - field_overrides[f.name] = rewritten + field_overrides[field_info.name] = rewritten if not field_overrides: return activity @@ -206,7 +206,7 @@ def _rewrite_value(value: Any, context: TranslationContext, warnings: list[str], return _rewrite_activity(value, context, warnings) if isinstance(value, SwitchCase): new_value = _rewrite_value(value.value, context, warnings, field_path=f"{field_path}.value") - new_activities = [_rewrite_activity(a, context, warnings) for a in value.activities] + new_activities = [_rewrite_activity(inner_activity, context, warnings) for inner_activity in value.activities] if new_value is value.value and all(n is o for n, o in zip(new_activities, value.activities)): return value return SwitchCase(value=new_value, activities=new_activities) diff --git a/src/orchestra/preparer/__init__.py b/src/flowx/preparer/__init__.py similarity index 100% rename from src/orchestra/preparer/__init__.py rename to src/flowx/preparer/__init__.py diff --git a/src/orchestra/preparer/activity_preparers/__init__.py b/src/flowx/preparer/activity_preparers/__init__.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/__init__.py rename to src/flowx/preparer/activity_preparers/__init__.py diff --git a/src/orchestra/preparer/activity_preparers/append_variable.py b/src/flowx/preparer/activity_preparers/append_variable.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/append_variable.py rename to src/flowx/preparer/activity_preparers/append_variable.py diff --git a/src/orchestra/preparer/activity_preparers/copy.py b/src/flowx/preparer/activity_preparers/copy.py similarity index 97% rename from src/orchestra/preparer/activity_preparers/copy.py rename to src/flowx/preparer/activity_preparers/copy.py index c21cf58..4165f23 100644 --- a/src/orchestra/preparer/activity_preparers/copy.py +++ b/src/flowx/preparer/activity_preparers/copy.py @@ -121,6 +121,9 @@ def prepare(activity: CopyActivity, *, scope: str = "") -> PreparedActivity: secrets = _build_secrets(activity, source_type, scope_name) setup_tasks = _build_setup_tasks(activity, source_type, volume_binding) + # Note: a collapsed activity_and_notify notification spec (activity.notifications) is wired onto + # the task generically in workflow_preparer.prepare_activity, for every task type -- not here. + return PreparedActivity(task=task, notebooks=notebooks, secrets=secrets, setup_tasks=setup_tasks) @@ -224,7 +227,7 @@ def _lakeflow_connection_name_for_activity(activity: CopyActivity) -> str: activity: Source Copy activity carrying source-side metadata. Returns: - A connection name namespaced under ``orchestra_`` and derived + A connection name namespaced under ``flowx_`` and derived from the source linked service name when available, so multiple Copies that share a source linked service emit one connection. Falls back to the activity task key when the IR does not record @@ -232,7 +235,7 @@ def _lakeflow_connection_name_for_activity(activity: CopyActivity) -> str: """ source_properties = activity.source_properties or {} linked_service_name = source_properties.get("linked_service_name") or activity.task_key - return f"orchestra_{_sanitize_identifier(linked_service_name)}_connection" + return f"flowx_{_sanitize_identifier(linked_service_name)}_connection" def _sanitize_identifier(value: str) -> str: @@ -464,9 +467,8 @@ def _build_sink_volume_setup_task(activity: CopyActivity) -> SetupTask | None: "volume_name": volume_name, "volume_type": "EXTERNAL", "location": external_location, - # ``location_type`` drives the storage-credential DDL the setup - # notebook emits (Azure managed identity vs S3 IAM vs GCS service - # account); omitting it leaves the user a manual TODO. + # location_type drives the storage-credential DDL the setup notebook emits (Azure MI / + # S3 IAM / GCS service account); omitting it leaves the user a manual TODO. "location_type": sink_properties.get("volume_location_type", ""), "storage_account": sink_properties.get("volume_storage_account", ""), }, diff --git a/src/orchestra/preparer/activity_preparers/databricks_job.py b/src/flowx/preparer/activity_preparers/databricks_job.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/databricks_job.py rename to src/flowx/preparer/activity_preparers/databricks_job.py diff --git a/src/orchestra/preparer/activity_preparers/delete.py b/src/flowx/preparer/activity_preparers/delete.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/delete.py rename to src/flowx/preparer/activity_preparers/delete.py diff --git a/src/orchestra/preparer/activity_preparers/execute_pipeline.py b/src/flowx/preparer/activity_preparers/execute_pipeline.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/execute_pipeline.py rename to src/flowx/preparer/activity_preparers/execute_pipeline.py diff --git a/src/orchestra/preparer/activity_preparers/filter.py b/src/flowx/preparer/activity_preparers/filter.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/filter.py rename to src/flowx/preparer/activity_preparers/filter.py diff --git a/src/orchestra/preparer/activity_preparers/for_each.py b/src/flowx/preparer/activity_preparers/for_each.py similarity index 95% rename from src/orchestra/preparer/activity_preparers/for_each.py rename to src/flowx/preparer/activity_preparers/for_each.py index 7be09d1..fdbe6a8 100644 --- a/src/orchestra/preparer/activity_preparers/for_each.py +++ b/src/flowx/preparer/activity_preparers/for_each.py @@ -203,10 +203,8 @@ def prepare( all_setup_tasks.extend(inner_prepared.setup_tasks) inner_workflows.extend(inner_prepared.inner_workflows) - # If the single child contributed extra_tasks (e.g. IfCondition or - # Switch branch bodies) we cannot inline as for_each_task.task — - # for_each only accepts a single task. Escalate to the sub-job - # path so the entire branch body survives (CF-001). + # CF-001: a single child with extra_tasks (IfCondition/Switch branch bodies) can't inline as + # for_each_task.task (one task only); escalate to the sub-job path so the whole body survives. if inner_prepared.extra_tasks: inner_job_name = f"{activity.task_key}_inner_tasks" inner_tasks: list[dict[str, Any]] = [ @@ -216,9 +214,8 @@ def prepare( normalize_inner_task_params(inner_tasks) parameters, job_parameters = collect_inner_job_params(inner_tasks, variable_task_keys=variable_task_keys) - # LSC3-001: gather cluster hints from inner activities so the - # inner-job default cluster lifts spark_env_vars / custom_tags / - # driver_node_type_id etc. from the LS-derived cluster spec. + # LSC3-001: gather cluster hints from inner activities so the inner-job default cluster lifts + # spark_env_vars / custom_tags / driver_node_type_id from the LS-derived cluster spec. inner_cluster_hints: list[dict[str, Any]] = [] for nested_activity in _iter_activity_with_descendants(inner_activities[0]): if nested_activity.cluster: @@ -276,9 +273,8 @@ def prepare( parameters, job_parameters = collect_inner_job_params(inner_tasks, variable_task_keys=variable_task_keys) - # LSC3-001: gather cluster hints from every nested inner activity - # so the inner-job default cluster picks up LS-derived - # spark_env_vars / custom_tags / driver_node_type_id. + # LSC3-001: gather cluster hints from every nested inner activity so the inner-job default + # cluster picks up LS-derived spark_env_vars / custom_tags / driver_node_type_id. inner_cluster_hints = [] for child in inner_activities: for nested_activity in _iter_activity_with_descendants(child): diff --git a/src/orchestra/preparer/activity_preparers/helpers.py b/src/flowx/preparer/activity_preparers/helpers.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/helpers.py rename to src/flowx/preparer/activity_preparers/helpers.py diff --git a/src/orchestra/preparer/activity_preparers/if_condition.py b/src/flowx/preparer/activity_preparers/if_condition.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/if_condition.py rename to src/flowx/preparer/activity_preparers/if_condition.py diff --git a/src/orchestra/preparer/activity_preparers/lookup.py b/src/flowx/preparer/activity_preparers/lookup.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/lookup.py rename to src/flowx/preparer/activity_preparers/lookup.py diff --git a/src/orchestra/preparer/activity_preparers/motif.py b/src/flowx/preparer/activity_preparers/motif.py similarity index 56% rename from src/orchestra/preparer/activity_preparers/motif.py rename to src/flowx/preparer/activity_preparers/motif.py index cf25589..dcf2963 100644 --- a/src/orchestra/preparer/activity_preparers/motif.py +++ b/src/flowx/preparer/activity_preparers/motif.py @@ -1,17 +1,27 @@ -"""Preparer for MotifActivity -> notebook_task or consolidated pipeline_task.""" +"""Preparer for MotifActivity -> notebook_task, for_each_task, or consolidated pipeline_task.""" from __future__ import annotations +import json from typing import TYPE_CHECKING, Any -from flowx.models.dab import SetupTask +from flowx.models.dab import DabNotebook, SetupTask from flowx.preparer.activity_preparers.helpers import build_notebook_activity_task -from flowx.preparer.code_generator import generate_motif_notebook +from flowx.preparer.code_generator import ( + generate_metadata_driven_control_lookup_notebook, + generate_metadata_driven_item_notebook, + generate_motif_notebook, +) from flowx.preparer.workflow_preparer import PreparedActivity, build_common_task_fields if TYPE_CHECKING: from flowx.models.ir import MotifActivity +# Default destination-table pattern when the collapsed Copy did not name a concrete sink. +_DEFAULT_SINK_TABLE_PATTERN = "raw.{schema_name}_{table_name}" +# Mirrors the ForEach preparer's default fan-out. +_FOR_EACH_CONCURRENCY = 20 + def prepare(activity: MotifActivity, *, scope: str = "") -> PreparedActivity: """Converts a MotifActivity into a DAB task. @@ -22,13 +32,18 @@ def prepare(activity: MotifActivity, *, scope: str = "") -> PreparedActivity: Returns: A :class:`PreparedActivity` whose shape depends on the motif: - metadata-driven motifs marked for consolidation emit a - consolidated Lakeflow Connect pipeline resource and a - ``pipeline_task``; every other motif keeps the legacy scaffold - notebook task. + + * Metadata-driven bulk copy the user **consolidated into a managed pipeline** (Lakeflow + Connect) becomes a single ``pipeline_task``. + * Metadata-driven bulk copy on the **default** path (not consolidated; + ``databricks_replacement == "for_each_ingestion"``) becomes a ``for_each_task`` that runs + one Spark JDBC read per source table -- instead of a single notebook looping internally. + * Every other motif keeps the scaffold notebook task. """ if activity.consolidate_metadata_driven and activity.lookup_values: return _prepare_consolidated_metadata_driven(activity) + if activity.databricks_replacement == "for_each_ingestion": + return _prepare_metadata_driven_for_each(activity) task, notebooks = build_notebook_activity_task( activity, notebook_relative_path=f"notebooks/{activity.task_key}.py", @@ -37,6 +52,69 @@ def prepare(activity: MotifActivity, *, scope: str = "") -> PreparedActivity: return PreparedActivity(task=task, notebooks=notebooks) +def _prepare_metadata_driven_for_each(activity: MotifActivity) -> PreparedActivity: + """Returns a ``for_each_task`` that ingests each source table via its own Spark JDBC read. + + This is the default (non-Lakeflow-Connect) translation of the metadata-driven bulk-copy motif. + Each iteration runs :func:`generate_metadata_driven_item_notebook` for one control-table row + (passed as ``{{input}}``), replacing the former single-notebook Python ``for`` loop with a + native Databricks for-each fan-out. + + The iteration ``inputs`` come from one of two sources: + + * **Static** -- when the control rows were materialised (``activity.lookup_values``), they are + inlined as a literal JSON array. + * **Runtime** -- otherwise a control-table lookup notebook task is emitted (queries the metadata + table and publishes the rows as the ``items`` task value); ``inputs`` references + ``{{tasks..values.items}}`` and the for-each task depends on it. + """ + config = activity.motif_config or {} + task_key = activity.task_key + copy_scope = config.get("copy_scope") or task_key + lookup_scope = config.get("lookup_scope") or task_key + sink_table_pattern = config.get("sink_table") or _DEFAULT_SINK_TABLE_PATTERN + + item_notebook_path = f"notebooks/{task_key}_ingest.py" + notebooks = [ + DabNotebook( + relative_path=item_notebook_path, + content=generate_metadata_driven_item_notebook(scope=copy_scope, sink_table_pattern=sink_table_pattern), + ) + ] + inner_task: dict[str, Any] = { + "task_key": f"{task_key}_ingest", + "notebook_task": { + "notebook_path": f"../src/{item_notebook_path}", + "base_parameters": {"item": "{{input}}"}, + }, + } + + task = build_common_task_fields(activity) + extra_tasks: list[dict[str, Any]] = [] + + if activity.lookup_values: + inputs = json.dumps(activity.lookup_values) + else: + lookup_key = f"{task_key}_control_lookup" + lookup_notebook_path = f"notebooks/{lookup_key}.py" + notebooks.append( + DabNotebook( + relative_path=lookup_notebook_path, + content=generate_metadata_driven_control_lookup_notebook( + scope=lookup_scope, lookup_query=config.get("lookup_query", "") + ), + ) + ) + extra_tasks.append( + {"task_key": lookup_key, "notebook_task": {"notebook_path": f"../src/{lookup_notebook_path}"}} + ) + task["depends_on"] = [*(task.get("depends_on") or []), {"task_key": lookup_key}] + inputs = f"{{{{tasks.{lookup_key}.values.items}}}}" + + task["for_each_task"] = {"inputs": inputs, "task": inner_task, "concurrency": _FOR_EACH_CONCURRENCY} + return PreparedActivity(task=task, notebooks=notebooks, extra_tasks=extra_tasks) + + def _prepare_consolidated_metadata_driven(activity: MotifActivity) -> PreparedActivity: """Returns a PreparedActivity that materialises a consolidated ingestion pipeline. @@ -92,10 +170,10 @@ def _consolidated_connection_name(task_key: str) -> str: task_key: Sanitised task key of the source motif activity. Returns: - A connection name namespaced under ``orchestra_`` so the setup + A connection name namespaced under ``flowx_`` so the setup notebook can recreate it idempotently. """ - return f"orchestra_{task_key}_connection" + return f"flowx_{task_key}_connection" def _build_consolidated_pipeline_definition( diff --git a/src/orchestra/preparer/activity_preparers/naming.py b/src/flowx/preparer/activity_preparers/naming.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/naming.py rename to src/flowx/preparer/activity_preparers/naming.py diff --git a/src/orchestra/preparer/activity_preparers/notebook.py b/src/flowx/preparer/activity_preparers/notebook.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/notebook.py rename to src/flowx/preparer/activity_preparers/notebook.py diff --git a/src/orchestra/preparer/activity_preparers/set_variable.py b/src/flowx/preparer/activity_preparers/set_variable.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/set_variable.py rename to src/flowx/preparer/activity_preparers/set_variable.py diff --git a/src/orchestra/preparer/activity_preparers/spark_jar.py b/src/flowx/preparer/activity_preparers/spark_jar.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/spark_jar.py rename to src/flowx/preparer/activity_preparers/spark_jar.py diff --git a/src/orchestra/preparer/activity_preparers/spark_python.py b/src/flowx/preparer/activity_preparers/spark_python.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/spark_python.py rename to src/flowx/preparer/activity_preparers/spark_python.py diff --git a/src/orchestra/preparer/activity_preparers/switch.py b/src/flowx/preparer/activity_preparers/switch.py similarity index 96% rename from src/orchestra/preparer/activity_preparers/switch.py rename to src/flowx/preparer/activity_preparers/switch.py index bfec286..a9eb353 100644 --- a/src/orchestra/preparer/activity_preparers/switch.py +++ b/src/flowx/preparer/activity_preparers/switch.py @@ -128,12 +128,9 @@ def prepare(activity: SwitchActivity, *, scope: str = "") -> PreparedActivity: inner_workflows=list(artifacts.inner_workflows), ) - # Build one condition task per case, chained via outcome="false" deps. - # Every case (including the first) is named ``_case_`` - # for clarity in the rendered job graph. The first case carries the - # original Switch's depends_on edges; ``prepare_workflow`` rewrites any - # downstream task that referenced the bare ```` key to point - # at the renamed first case. + # One condition task per case, chained via outcome="false" deps. Every case is named + # _case_; the first carries the Switch's depends_on edges and prepare_workflow + # rewrites downstream refs from the bare key to the renamed first case. case_keys: list[str] = [] for index, case in enumerate(activity.cases): is_first = index == 0 diff --git a/src/orchestra/preparer/activity_preparers/wait.py b/src/flowx/preparer/activity_preparers/wait.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/wait.py rename to src/flowx/preparer/activity_preparers/wait.py diff --git a/src/orchestra/preparer/activity_preparers/web_activity.py b/src/flowx/preparer/activity_preparers/web_activity.py similarity index 93% rename from src/orchestra/preparer/activity_preparers/web_activity.py rename to src/flowx/preparer/activity_preparers/web_activity.py index bc6245d..257d30b 100644 --- a/src/orchestra/preparer/activity_preparers/web_activity.py +++ b/src/flowx/preparer/activity_preparers/web_activity.py @@ -21,11 +21,8 @@ def prepare(activity: WebActivity, *, scope: str = "") -> PreparedActivity: """Converts a WebActivity into a notebook_task with a generated HTTP notebook.""" secrets, setup_tasks = _extract_secrets_and_setup(activity, scope=scope) - # C-38 (LSC4-002): when the preparer resolved an AzureKeyVaultSecret - # payload to a real (scope, key) pair, thread it into the notebook - # generator so the rendered ``dbutils.secrets.get`` references the - # real values rather than the hard-coded ``scope=task_key, - # key='auth-credential'`` fallback. + # C-38 (LSC4-002): thread any resolved AzureKeyVaultSecret (scope, key) into the generator so the + # rendered dbutils.secrets.get uses real values, not the scope=task_key, key='auth-credential' fallback. credential_scope: str | None = None credential_key: str | None = None if secrets: @@ -45,6 +42,9 @@ def prepare(activity: WebActivity, *, scope: str = "") -> PreparedActivity: base_parameters={ "url": resolve_param_value(activity.url), "method": resolve_param_value(activity.method), + # Bind variable/task-value widgets referenced by the resolved body + # (e.g. {{tasks._init_batchId.values.batchId}}). + **dict(activity.body_required_parameters or {}), }, ) diff --git a/src/orchestra/preparer/code_generator.py b/src/flowx/preparer/code_generator.py similarity index 90% rename from src/orchestra/preparer/code_generator.py rename to src/flowx/preparer/code_generator.py index 2f2e43a..6fa73e8 100644 --- a/src/orchestra/preparer/code_generator.py +++ b/src/flowx/preparer/code_generator.py @@ -203,11 +203,9 @@ def _assemble_file_lookup_source_path(props: dict[str, Any]) -> str: folder = _coerce_to_str(props.get("folder_path")) filename = _coerce_to_str(props.get("file_name")) container = _coerce_to_str(props.get("container")) - # C-47 (LSC5-001): never join a raw ``dataset()`` reference into the - # baked default path. lookup.translate substitutes these from the - # dataset reference's parameter bindings; if one still leaks through - # (e.g. an unbound dataset parameter) drop it so spark.read does not get - # a literal broken ``abfss://.../@dataset().fileName`` path. + # C-47 (LSC5-001): never bake a raw dataset() reference into the default path. lookup.translate + # substitutes these; drop any that still leak through so spark.read doesn't get a broken + # abfss://.../@dataset().fileName path. if "dataset(" in folder: folder = "" if "dataset(" in filename: @@ -223,7 +221,7 @@ def _assemble_file_lookup_source_path(props: dict[str, Any]) -> str: parts.append(folder.strip("/")) if filename: parts.append(filename.strip("/")) - return "/".join(p for p in parts if p) + return "/".join(part for part in parts if part) def _file_lookup_body(activity: LookupActivity) -> str: @@ -316,19 +314,14 @@ def generate_web_activity_notebook( if auth: scope = scope or activity.task_key auth_type = auth.get("type", "") - # C-38 (LSC4-002): prefer the resolved (scope, key) tuple from the - # preparer when supplied. Fall back to the legacy - # ``(task_key, 'auth-credential')`` shape only when the preparer - # didn't (or couldn't) compute one. + # C-38 (LSC4-002): prefer the preparer's resolved (scope, key) tuple; fall back to the legacy + # (task_key, 'auth-credential') shape only when the preparer didn't compute one. resolved_scope = credential_scope or scope resolved_key = credential_key or "auth-credential" if auth_type in ("MSI", "ManagedServiceIdentity"): - # LSC3-002: MSI / Managed Identity auth carries no static secret, - # so reading ``auth-credential`` from a secret scope is a fake - # placeholder that fails at runtime. Surface a NotImplementedError - # so the user can implement the credential exchange manually -- - # the manual_credential SetupTask emitted by web_activity preparer - # already flags this in SETUP.md. + # LSC3-002: MSI/Managed Identity has no static secret, so reading auth-credential would be a + # placeholder that fails at runtime; raise NotImplementedError (web_activity's manual_credential + # SetupTask already flags this in SETUP.md) so the user wires the exchange manually. auth_block = textwrap.dedent(f"""\ # Authentication ({auth_type}) - manual implementation required raise NotImplementedError( @@ -361,19 +354,22 @@ def generate_web_activity_notebook( """) body_block = "" + extra_imports: list[str] = [] request_call = "" if activity.method in ("POST", "PUT", "PATCH"): raw_body = activity.body - # If the body was pre-resolved to Python code by the translator - # (contains function calls like __import__ or json.loads), embed directly. - if isinstance(raw_body, str) and ("__import__" in raw_body or "json.loads" in raw_body): + # Prefer the body the translator pre-resolved to Python code (the real TranslationContext lowered + # @concat / @variables / @{...} that the empty context here could not). + if activity.body_code is not None: + extra_imports = list(activity.body_imports) + body_block = f"body = {activity.body_code}\n" + # Legacy: top-level Expression bodies stored directly on ``body``. + elif isinstance(raw_body, str) and ("__import__" in raw_body or "json.loads" in raw_body): body_block = f"body = {raw_body}\n" else: body_str = _resolve_body(raw_body) - # ``_resolve_body`` may return either a JSON literal, a Python - # dict literal, or a ``repr()``'d string containing Python-like - # concat syntax. Parse strings as JSON when possible so the - # downstream ``requests.request(json=...)`` gets a real object. + # _resolve_body may return a JSON literal, a Python dict literal, or a repr()'d concat string; + # parse strings as JSON when possible so requests.request(json=...) gets a real object. body_block = textwrap.dedent(f"""\ body_raw = dbutils.widgets.get("body") or {body_str} if isinstance(body_raw, str): @@ -409,9 +405,13 @@ def generate_web_activity_notebook( else: method_line = f'method = dbutils.widgets.get("method") or "{activity.method}"' - body = textwrap.dedent(f"""\ + body = textwrap.dedent("""\ import json import requests + """) + for imp in dict.fromkeys(extra_imports): + body += f"{imp}\n" + body += textwrap.dedent(f"""\ # Parameters {url_line} @@ -485,9 +485,8 @@ def generate_set_variable_notebook(activity: SetVariableActivity) -> str: else: import_block = "" - # Build body lines list to avoid textwrap.dedent issues when - # import_block starts at column 0 (which would prevent dedent - # from stripping the common leading whitespace). + # Build the body as a lines list to avoid textwrap.dedent issues when import_block starts at + # column 0 (which would stop dedent stripping the common leading whitespace). lines = ["import json"] if import_block: lines.append(import_block.rstrip("\n")) @@ -581,10 +580,8 @@ def generate_copy_notebook(activity: CopyActivity, *, scope: str = "") -> str: else: body = _generate_generic_copy_body(activity) - # Hoist any imports the body needs into a single cell at the top of - # the notebook. ``_render_sink_write`` and a few other helpers used - # to inline ``from datetime import ...`` next to the call site, which - # produced an awkward block sandwiched between two comment groups. + # Hoist imports the body needs into a single cell at the top of the notebook, since some helpers + # inline ``from datetime import ...`` at the call site (an awkward block between comment groups). imports = _detect_imports(body) if imports: body = _strip_inline_imports(body, imports) @@ -681,7 +678,7 @@ def _safe_identifier(value: str) -> str: The input with non-identifier characters replaced by underscores, falling back to ``ingest`` when the result would be empty. """ - cleaned = "".join(ch if ch.isalnum() or ch == "_" else "_" for ch in value) + cleaned = "".join(char if char.isalnum() or char == "_" else "_" for char in value) cleaned = cleaned.strip("_") if not cleaned or cleaned[0].isdigit(): cleaned = f"ingest_{cleaned}" if cleaned else "ingest" @@ -896,7 +893,7 @@ def _notebook_header(title: str) -> str: # MAGIC %md # MAGIC # {title} # MAGIC - # MAGIC *Auto-generated by Flowx. Do not edit manually unless necessary.* + # MAGIC *Auto-generated by flowx. Do not edit manually unless necessary.* """) @@ -1103,9 +1100,8 @@ def _render_sink_write( fmt = activity.sink_format sink_props = activity.sink_properties or {} - # File-format sink — write the actual format declared by the ADF - # output dataset. Delta files written with ``.save(path)`` skip the - # metastore, which matches the ADF semantic of a path-based dataset. + # File-format sink: write the format declared by the ADF output dataset. .save(path) skips the + # metastore, matching the ADF semantic of a path-based dataset. if fmt and fmt != "delta": opts: list[str] = [] format_settings = sink_props.get("formatSettings") or {} @@ -1119,17 +1115,13 @@ def _render_sink_write( volume_relative = sink_props.get("volume_relative_path") if volume_relative is not None: - # Volume-rooted sink: ``output_path_root`` is set by the bundler - # as a base_parameter (with DAB-substituted ``${var.catalog}`` - # / ``${var.schema}``). Any ``@{...}`` expressions in the - # ADF dataset's folderPath / fileName have already been - # rewritten to Python f-string fragments, so we wrap the - # relative path in an f-string and join. + # Volume-rooted sink: the bundler sets output_path_root as a base_parameter (DAB-substituted + # ${var.catalog}/${var.schema}). The dataset's folderPath/fileName @{...} expressions are + # already rewritten to f-string fragments, so wrap the relative path in an f-string and join. rel_literal = volume_relative.replace('"', '\\"') preamble = "" - # Pull in any modules the rewritten f-string fragments reference - # so the notebook is runnable as-is. Today the only one is - # ``datetime`` (from ``@{formatDateTime(...)}`` rewrites). + # Pull in any modules the rewritten f-string fragments reference so the notebook runs as-is + # (today only datetime, from @{formatDateTime(...)} rewrites). if "datetime." in rel_literal: preamble = f"{indent}from datetime import datetime\n" return ( @@ -1141,9 +1133,8 @@ def _render_sink_write( f'{indent}{df_var}.write.format("{fmt}"){opts_str}.mode("{mode}").save(output_path)\n' ) - # No structured sink volume — fall back to a single ``output_path`` - # widget the user fills in. Common when the linked service uses - # a masked connection string and we can't reconstruct any path. + # No structured sink volume: fall back to a single output_path widget the user fills in. Common + # when the linked service uses a masked connection string and no path can be reconstructed. return ( f"{indent}# The ADF output dataset path could not be resolved at translation time.\n" f"{indent}# Set ``output_path`` on this task to the destination URI.\n" @@ -1215,9 +1206,8 @@ def _generate_autoloader_body(activity: CopyActivity) -> str: source_properties = activity.source_properties or {} sink_properties = activity.sink_properties or {} - # Prefer the UC volume path when available (set by the copy preparer - # when an external volume setup task is created); otherwise fall back - # to the resolved abfss:// path or raw dataset path. + # Prefer the UC volume path when available (set by the copy preparer for external-volume setup); + # otherwise fall back to the resolved abfss:// path or raw dataset path. source_path = source_properties.get( "volume_path", source_properties.get( @@ -1228,9 +1218,8 @@ def _generate_autoloader_body(activity: CopyActivity) -> str: sink_table = sink_properties.get("table", sink_properties.get("tableName", f"{activity.task_key}_raw")) file_format = _infer_file_format(activity.source_type, source_properties) - # Use the volume for checkpoints and schema evolution storage instead of - # /tmp. This ensures state persists across cluster restarts and is - # visible in Unity Catalog. + # Use the volume (not /tmp) for checkpoints and schema-evolution storage so state persists across + # cluster restarts and is visible in Unity Catalog. volume_base = source_properties.get("volume_base", "") if volume_base: checkpoint = f"{volume_base}/_checkpoints/{activity.task_key}" @@ -1313,10 +1302,8 @@ def _generate_jdbc_body(activity: CopyActivity, *, scope: str = "") -> str: is_expression = True if is_expression: - # The query is an ADF expression (e.g. @concat('SELECT * FROM ', item().schema_name, ...)). - # Generate a notebook that reads the current ForEach item from the - # "item" widget (set to {{input}} by the for_each_task) and builds - # the SQL query dynamically. + # The query is an ADF expression (e.g. @concat('SELECT * FROM ', item().schema_name, ...)); generate + # a notebook that reads the current ForEach item from the "item" widget and builds the SQL dynamically. return ( textwrap.dedent(f"""\ import json @@ -1478,6 +1465,78 @@ def _generate_generic_copy_body(activity: CopyActivity) -> str: ) +def generate_metadata_driven_item_notebook(*, scope: str, sink_table_pattern: str) -> str: + """Generates the per-iteration notebook for a metadata-driven for_each_task. + + Each ``for_each_task`` iteration runs this notebook with the current control-table row passed as + the ``item`` widget (set to ``{{input}}``). It reads that one source table over Spark JDBC and + writes it to Delta -- the per-row body of the former in-notebook loop, now one task per table. + + Args: + scope: Secret scope holding ``jdbc-url`` / ``jdbc-user`` / ``jdbc-password`` for the source. + sink_table_pattern: ``str.format`` pattern for the destination table, e.g. + ``"raw.{schema_name}_{table_name}"``. + """ + return "# Databricks notebook source\n" + textwrap.dedent(f"""\ + import json + + # The for_each_task passes the current control-table row as the "item" widget via {{{{input}}}}. + item_raw = dbutils.widgets.get("item") + item = json.loads(item_raw) if item_raw else {{}} + schema_name = item.get("schema_name", "dbo") + table_name = item.get("table_name") or item.get("name") or "UNKNOWN_TABLE" + target = {sink_table_pattern!r}.format(schema_name=schema_name, table_name=table_name) + query = f"SELECT * FROM {{schema_name}}.{{table_name}}" + + jdbc_url = dbutils.secrets.get(scope="{scope}", key="jdbc-url") + jdbc_user = dbutils.secrets.get(scope="{scope}", key="jdbc-user") + jdbc_password = dbutils.secrets.get(scope="{scope}", key="jdbc-password") + + ( + spark.read.format("jdbc") + .option("url", jdbc_url) + .option("user", jdbc_user) + .option("password", jdbc_password) + .option("query", query) + .load() + .write.format("delta") + .mode("overwrite") + .option("overwriteSchema", "true") + .saveAsTable(target) + ) + """) + + +def generate_metadata_driven_control_lookup_notebook(*, scope: str, lookup_query: str) -> str: + """Generates the control-table lookup notebook that seeds a metadata-driven for_each_task. + + Used when the control rows were not materialised at translation time: it queries the metadata + table over Spark JDBC and publishes the rows as the ``items`` task value, which the downstream + ``for_each_task`` consumes via ``{{tasks..values.items}}``. + + Args: + scope: Secret scope holding the control DB's ``jdbc-*`` credentials. + lookup_query: SQL that returns one row per source table to ingest. + """ + return "# Databricks notebook source\n" + textwrap.dedent(f"""\ + jdbc_url = dbutils.secrets.get(scope="{scope}", key="jdbc-url") + jdbc_user = dbutils.secrets.get(scope="{scope}", key="jdbc-user") + jdbc_password = dbutils.secrets.get(scope="{scope}", key="jdbc-password") + + control_query = {lookup_query!r} + control_df = ( + spark.read.format("jdbc") + .option("url", jdbc_url) + .option("user", jdbc_user) + .option("password", jdbc_password) + .option("query", control_query) + .load() + ) + items = [row.asDict() for row in control_df.collect()] + dbutils.jobs.taskValues.set(key="items", value=items) + """) + + def generate_motif_notebook(activity: MotifActivity) -> str: """Generates a notebook scaffold for a collapsed motif activity.""" return _build_motif_notebook( @@ -1530,7 +1589,7 @@ def _build_motif_notebook( "# MAGIC", notes_list, "# MAGIC", - "# MAGIC *Auto-generated by Flowx motif collapser.*", + "# MAGIC *Auto-generated by flowx motif collapser.*", "", "# COMMAND ----------", "", diff --git a/src/flowx/preparer/notifications.py b/src/flowx/preparer/notifications.py new file mode 100644 index 0000000..18e8401 --- /dev/null +++ b/src/flowx/preparer/notifications.py @@ -0,0 +1,158 @@ +"""Resolves collapsed ``activity_and_notify`` specs into DAB task notifications. + +Email specs become ``email_notifications`` from raw addresses; Slack/Teams/PagerDuty/Generic Webhook +specs reuse the destination id provisioned at modify time (``provision_destination``), creating one +via the SDK only as a fallback when the spec carries no pre-resolved id. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from flowx.models.dab import SetupTask + +logger = logging.getLogger(__name__) + +_WEBHOOK_DESTINATIONS = frozenset({"slack", "teams", "pagerduty", "webhook"}) + +# Optional SDK config kwargs per destination (beyond the primary field); only passed when the +# user supplied a value, so the SDK defaults anything left blank. +_DESTINATION_CONFIG_FIELDS: dict[str, tuple[str, ...]] = { + "slack": ("url", "channel_id", "oauth_token"), + "teams": ("url",), + "webhook": ("url", "username", "password"), + "pagerduty": ("integration_key",), +} + + +def resolve_task_notifications(spec: dict[str, Any]) -> tuple[dict[str, Any], list[SetupTask]]: + """Return ``(task_notification_keys, setup_tasks)`` for a notification spec. + + ``task_notification_keys`` is merged into the DAB task dict (it carries an + ``email_notifications`` or ``webhook_notifications`` entry). ``setup_tasks`` + is non-empty only when a webhook-style destination could not be created + (e.g. no workspace auth at prepare time), in which case a documentation + SetupTask is emitted instead and the task ships without notifications. + """ + destination = spec.get("destination", "") + events: list[str] = spec.get("events") or ["on_failure"] + args: dict[str, Any] = spec.get("args") or {} + + if destination == "email": + recipients = [recipient for recipient in (args.get("addresses") or []) if recipient] + if not recipients: + logger.warning("activity_and_notify email destination has no recipients; skipping notification wiring.") + return {}, [] + return {"email_notifications": {event: list(recipients) for event in events}}, [] + + if destination not in _WEBHOOK_DESTINATIONS: + return {}, [] + + display_name = spec.get("destination_name") or f"flowx-{destination}" + # Prefer a destination id provisioned at modify time; create one here only when the report + # lacks a pre-resolved id (e.g. no workspace auth was available then). + destination_id = spec.get("destination_id") or _ensure_destination(destination, display_name, args) + if destination_id is None: + setup = SetupTask( + type="notification_destination", + config={ + "destination": destination, + "display_name": display_name, + **{key: value for key, value in args.items() if value}, + "note": ( + "Could not create this notification destination during prepare. Create it " + "(Settings > Notifications, or w.notification_destinations.create), then add " + '{"id": ""} to the task\'s webhook_notifications.' + ), + }, + ) + return {}, [setup] + + return {"webhook_notifications": {event: [{"id": destination_id}] for event in events}}, [] + + +def provision_destination(spec: dict[str, Any]) -> tuple[dict[str, Any], str]: + """Create (or reuse) the SDK notification destination for *spec* at prompt time. + + Called from the adapter ``modify`` phase right after the user answers the + notification follow-ups. For non-email destinations it creates (or reuses by + display name) the Databricks notification destination via the SDK and returns a + copy of *spec* augmented with the resolved ``destination_id`` so prepare can wire + it without another SDK call. + + Email specs (and anything that is not a webhook-style destination) pass through + unchanged -- email uses raw ``email_notifications`` and needs no destination. + + On failure the spec is returned unchanged (its ``args`` retained), so prepare can + retry the create or fall back to a ``notification_destination`` setup task. + + Returns: + ``(spec, status_message)`` where ``status_message`` is a human-readable line + describing the outcome (empty for email / no-op). + """ + destination = spec.get("destination", "") + if destination == "email" or destination not in _WEBHOOK_DESTINATIONS: + return spec, "" + display_name = spec.get("destination_name") or f"flowx-{destination}" + if spec.get("destination_id"): + return ( + spec, + f"Notification destination '{display_name}' ({destination}) already resolved -> {spec['destination_id']}.", + ) + destination_id = _ensure_destination(destination, display_name, spec.get("args") or {}) + if destination_id is None: + return spec, ( + f"WARNING: could not create notification destination '{display_name}' ({destination}) now; " + "prepare will retry or emit a setup task." + ) + return {**spec, "destination_id": destination_id}, ( + f"Created/reused notification destination '{display_name}' ({destination}) -> {destination_id}." + ) + + +def _build_destination_config(sdk_settings: Any, destination: str, args: dict[str, Any]) -> Any | None: + """Build the SDK ``settings.Config`` for *destination* from the resolved *args*. + + Only kwargs the user supplied are passed; blank optional fields are omitted so + the SDK applies its own defaults. + """ + fields = _DESTINATION_CONFIG_FIELDS.get(destination) + if fields is None: + return None + kwargs = {field: args[field] for field in fields if args.get(field)} + if destination == "slack": + return sdk_settings.Config(slack=sdk_settings.SlackConfig(**kwargs)) + if destination == "teams": + return sdk_settings.Config(microsoft_teams=sdk_settings.MicrosoftTeamsConfig(**kwargs)) + if destination == "webhook": + return sdk_settings.Config(generic_webhook=sdk_settings.GenericWebhookConfig(**kwargs)) + if destination == "pagerduty": + return sdk_settings.Config(pagerduty=sdk_settings.PagerdutyConfig(**kwargs)) + return None + + +def _ensure_destination(destination: str, display_name: str, args: dict[str, Any]) -> str | None: + """Create (or reuse) a notification destination via the SDK; return its id or None.""" + try: + from databricks.sdk.service import settings as sdk_settings + + from flowx.preparer.workspace_downloader import _get_workspace_client + + client = _get_workspace_client() + # Reuse an existing destination with the same display name so prepare is idempotent. + for existing in client.notification_destinations.list(): + if getattr(existing, "display_name", None) == display_name and getattr(existing, "id", None): + logger.info("Reusing existing notification destination '%s' (%s).", display_name, existing.id) + return existing.id + + config = _build_destination_config(sdk_settings, destination, args) + if config is None: + return None + + created = client.notification_destinations.create(display_name=display_name, config=config) + logger.info("Created notification destination '%s' (%s).", display_name, getattr(created, "id", None)) + return getattr(created, "id", None) + except Exception as exc: # noqa: BLE001 - degrade gracefully to a setup task + logger.warning("Notification destination create failed for '%s' (%s): %s", display_name, destination, exc) + return None diff --git a/src/orchestra/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py similarity index 86% rename from src/orchestra/preparer/workflow_preparer.py rename to src/flowx/preparer/workflow_preparer.py index 57a178f..9edc4b0 100644 --- a/src/orchestra/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -42,14 +42,9 @@ class PreparedActivity: secrets: list[SecretInstruction] = field(default_factory=list) setup_tasks: list[SetupTask] = field(default_factory=list) inner_workflows: list[PreparedWorkflow] = field(default_factory=list) - # Switch renames its first case from ```` to - # ``_case_``; ``prepare_workflow`` reads this map to - # rewrite ``depends_on`` edges that referenced the original key. + # Switch renames its first case key; prepare_workflow reads this map to rewrite depends_on edges. task_key_remap: dict[str, str] = field(default_factory=dict) - # Lakeflow pipeline resources (e.g. Lakeflow Connect managed - # ingestion pipelines) the bundle writer emits under - # ``resources/pipelines/.yml``. Each entry is a dict - # with ``resource_key`` and ``definition`` keys. + # Lakeflow pipeline resources emitted under resources/pipelines/.yml ({resource_key, definition}). pipeline_resources: list[dict[str, Any]] = field(default_factory=list) parameter_approximations: list[ParameterApproximation] = field(default_factory=list) @@ -178,15 +173,22 @@ def prepare_activity( elif type(activity) is AppendVariableActivity: prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) elif type(activity) is ForEachActivity: - # C-06 (VAREX-004): inner-job parameter collector needs the parent's - # variable -> setter mapping so @variables('X') references inside the - # ForEach body route through {{tasks.X.values.Y}} rather than an - # undeclared {{job.parameters.X}}. + # C-06 (VAREX-004): pass the parent's variable->setter map so @variables('X') inside the ForEach + # body routes through {{tasks.X.values.Y}} instead of an undeclared {{job.parameters.X}}. prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) else: prepared = preparer_fn(activity, scope=scope) prepared.task = _stamp_compute_mode(prepared.task, activity.compute_mode) + + # Wire any notification spec the adapter stamped onto this task (generic across task types, not just Copy). + if activity.notifications: + from flowx.preparer.notifications import resolve_task_notifications + + notification_keys, notification_setup = resolve_task_notifications(activity.notifications) + prepared.task = {**prepared.task, **notification_keys} + prepared.setup_tasks = prepared.setup_tasks + notification_setup + return prepared @@ -212,9 +214,13 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: """Returns a PreparedActivity with a stub notebook for an unsupported activity.""" task = build_common_task_fields(activity) + agentic_skill: str | None = None + raw_definition: dict[str, Any] | None = None if isinstance(activity, PlaceholderActivity): comment = activity.comment or "This activity requires manual implementation." original_type = activity.original_type + agentic_skill = activity.agentic_skill + raw_definition = activity.raw_definition elif isinstance(activity, UnsupportedActivity): comment = activity.reason or "This activity type is not supported." original_type = activity.original_type @@ -225,6 +231,22 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: notebook_name = f"{activity.task_key}.py" notebook_path = f"notebooks/{notebook_name}" + # When the activity is an agentic gap (e.g. Until), embed its full ADF/ARM + # JSON so the agentic handler can translate it directly from source. + arm_block = "" + if raw_definition is not None: + import json as _json + + skill_hint = f" using `{agentic_skill}`" if agentic_skill else "" + arm_lines = _json.dumps(raw_definition, indent=2).splitlines() + arm_block = ( + "# MAGIC\n" + f"# MAGIC An agent should translate this activity{skill_hint} from the ADF/ARM JSON below,\n" + "# MAGIC then replace the `raise NotImplementedError` cell with the generated code.\n" + "# MAGIC\n" + "# MAGIC ```json\n" + "".join(f"# MAGIC {line}\n" for line in arm_lines) + "# MAGIC ```\n" + ) + content = ( "# Databricks notebook source\n" "# MAGIC %md\n" @@ -232,9 +254,8 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: "# MAGIC\n" f"# MAGIC Original ADF activity type: **{original_type}**\n" "# MAGIC\n" - f"# MAGIC {comment}\n" - "\n# COMMAND ----------\n\n" - f"raise NotImplementedError(\"Activity '{activity.name}' ({original_type}) requires manual implementation.\")\n" + f"# MAGIC {comment}\n" + arm_block + "\n# COMMAND ----------\n\n" + f"raise NotImplementedError(\"Activity '{activity.name}' ({original_type}) needs agentic translation.\")\n" ) task["notebook_task"] = { @@ -294,11 +315,8 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: all_tasks.extend(prepared.extra_tasks) artifacts = merge_prepared_artifacts(artifacts, prepared) task_key_remap.update(prepared.task_key_remap) - # C-04 (NB-ITER2-4 / LSC2-001): walk into IfCondition / Switch / - # ForEach branches so the workflow's cluster_hints aggregation - # picks up cluster config on activities nested inside compound - # activities. Without this the default Standard_DS3_v2 / 15.4.x - # fallback ships even when the inner notebook has an explicit LS. + # C-04 (NB-ITER2-4 / LSC2-001): walk into IfCondition/Switch/ForEach branches so cluster_hints + # picks up nested cluster config; else the default cluster ships even when an inner notebook has an LS. for nested_activity in _iter_activity_with_descendants(activity): if nested_activity.cluster: cluster_hints.append(dict(nested_activity.cluster)) @@ -321,19 +339,13 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: seen_secrets.add(secret_id) unique_secrets.append(secret) - # VAREX3-003: emit a manual_variable_rollup SetupTask whenever a sibling - # IfCondition / Switch / SetVariable reads a variable that is only - # mutated inside a ForEach inner job. ADF semantics treat the post- - # ForEach read as "latest committed value" but that value is unreachable - # across the run_job_task boundary in DAB. Surfacing the warning lets - # the user add a roll-up notebook before the dependent activity runs. + # VAREX3-003: flag a sibling read of a variable mutated only inside a ForEach inner job -- the post-ForEach + # "latest value" is unreachable across the run_job_task boundary in DAB, so emit a manual_variable_rollup. cross_scope_rollups = _detect_cross_foreach_variable_reads(pipeline.tasks) setup_tasks_out = _dedupe_setup_tasks(artifacts.setup_tasks) setup_tasks_out.extend(cross_scope_rollups) - # C-36 (SCHED4-001): emit a manual_schedule_time_of_day SetupTask - # whenever the trigger.periodic schedule carries hours/minutes/weekDays - # that the periodic primitive can't encode. SETUP.md picks it up so - # the user can manually add the time-of-day to the cron expression. + # C-36 (SCHED4-001): the periodic primitive can't encode hours/minutes/weekDays, so emit a + # manual_schedule_time_of_day SetupTask for the user to add the time-of-day to the cron expression. if pipeline.schedule and pipeline.schedule.get("time_of_day_note"): setup_tasks_out.append( SetupTask( @@ -347,11 +359,8 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: ) ) - # C-39 (LSC4-004): when any cluster hint references an ADF - # authentication mode that has no direct Databricks equivalent (MSI, - # CredentialReference) the bundle's default_cluster silently uses - # ``single_user_name: ${workspace.current_user.userName}``. Surface - # a manual_credential SetupTask so SETUP.md flags the substitution. + # C-39 (LSC4-004): ADF auth modes with no Databricks equivalent (MSI, CredentialReference) make the + # default_cluster fall back to single_user_name: ${workspace.current_user.userName}; flag it via SetupTask. seen_auth: set[tuple[str, str]] = set() for hint in cluster_hints: auth = hint.get("_adf_authentication") or "" @@ -382,6 +391,7 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: return PreparedWorkflow( name=pipeline.name, tasks=all_tasks, + parameters=list(pipeline.parameters or []), notebooks=list(artifacts.notebooks), secrets=unique_secrets, setup_tasks=setup_tasks_out, @@ -419,9 +429,7 @@ def _detect_cross_foreach_variable_reads(activities: list[Activity]) -> list[Set if not var_set_inside_foreach: return [] - # Identify variables that are also set OUTSIDE any ForEach -- those are - # not cross-scope dangers because the parent always has a fresh setter - # to point at. + # Variables also set OUTSIDE any ForEach are safe -- the parent always has a fresh setter to point at. set_outside: set[str] = set() for activity in activities: if isinstance(activity, SetVariableActivity): @@ -438,7 +446,7 @@ def _detect_cross_foreach_variable_reads(activities: list[Activity]) -> list[Set def _read_refs(text: str) -> set[str]: if not isinstance(text, str): return set() - return {m.group(1) for m in var_ref_pattern.finditer(text)} + return {match.group(1) for match in var_ref_pattern.finditer(text)} def _walk_activity_strings(activity: Activity) -> Iterable[str]: # Yield every string-like field the variable might appear in. diff --git a/src/orchestra/preparer/workspace_downloader.py b/src/flowx/preparer/workspace_downloader.py similarity index 55% rename from src/orchestra/preparer/workspace_downloader.py rename to src/flowx/preparer/workspace_downloader.py index e5167a1..3d2e42d 100644 --- a/src/orchestra/preparer/workspace_downloader.py +++ b/src/flowx/preparer/workspace_downloader.py @@ -12,6 +12,105 @@ logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Databricks runtime detection and auto-configuration +# --------------------------------------------------------------------------- + +_WORKSPACE_ROOT = Path("/Workspace") + +# Common notebook file extensions on the Databricks workspace filesystem. +_NOTEBOOK_EXTENSIONS = (".py", ".sql", ".scala", ".r", ".R", ".ipynb") + + +def _is_databricks_runtime() -> bool: + """Returns True if running inside a Databricks cluster or serverless compute.""" + return os.environ.get("DATABRICKS_RUNTIME_VERSION") is not None + + +def _local_workspace_accessible() -> bool: + """Returns True if the /Workspace filesystem is mounted and readable.""" + return _WORKSPACE_ROOT.is_dir() + + +def _try_local_workspace_read(workspace_path: str) -> str | None: + """Attempts to read a notebook directly from the local /Workspace filesystem. + + On Databricks compute (classic or serverless), workspace files are mounted + at ``/Workspace/``. Notebooks are stored with a language extension + (e.g. ``.py``, ``.sql``). This function probes the path with each known + extension and returns the source if found — no SDK auth required. + + Args: + workspace_path: Logical workspace path (e.g. ``"/Shared/ETL/transform"``). + + Returns: + Notebook source as a string, or ``None`` if not found locally. + """ + if not _local_workspace_accessible(): + return None + + base = _WORKSPACE_ROOT / workspace_path.lstrip("/") + + # Try the exact path first (already has an extension or is a plain file) + if base.is_file(): + try: + return base.read_text(encoding="utf-8") + except OSError as exc: + logger.debug("Local read failed for %s: %s", base, exc) + + # Probe with common notebook extensions + for ext in _NOTEBOOK_EXTENSIONS: + candidate = base.with_suffix(ext) + if candidate.is_file(): + try: + content = candidate.read_text(encoding="utf-8") + logger.info("Read notebook from local filesystem: %s", candidate) + return content + except OSError as exc: + logger.debug("Local read failed for %s: %s", candidate, exc) + + return None + + +def _ensure_databricks_runtime_auth() -> bool: + """Auto-configures ~/.databrickscfg from the notebook runtime context. + + On Databricks serverless (or classic cluster) compute, no CLI auth is + pre-configured but the runtime provides host + token via the REPL context. + This function detects that situation and writes a DEFAULT profile so the + Databricks SDK can authenticate transparently. + """ + cfg_path = _get_databrickscfg_path() + if cfg_path.exists() and cfg_path.stat().st_size > 0: + return True + + if os.environ.get("DATABRICKS_HOST") and os.environ.get("DATABRICKS_TOKEN"): + return True + + try: + from dbruntime.databricks_repl_context import get_context # type: ignore[import-not-found] + + context = get_context() + host = f"https://{context.browserHostName}" + token = context.apiToken + if not host or not token: + logger.warning("Databricks runtime detected but host/token unavailable from context") + return False + + cfg_path.parent.mkdir(parents=True, exist_ok=True) + with open(cfg_path, "w") as f: + f.write(f"[DEFAULT]\nhost = {host}\ntoken = {token}\n") + logger.info("Auto-configured Databricks auth from runtime context -> %s", cfg_path) + return True + except ImportError: + logger.debug("dbruntime not available; cannot auto-configure auth via REPL context") + return False + except Exception as exc: + logger.warning("Failed to auto-configure Databricks runtime auth: %s", exc) + return False + + # --------------------------------------------------------------------------- # Module-level state — resolved once per process, reused across calls # --------------------------------------------------------------------------- @@ -19,14 +118,13 @@ _resolved_profile: str | None = None _profile_resolved: bool = False -# When False (default), preparers preserve workspace artifact paths in-place -# instead of attempting a network download. The CLI flips this on so that -# `databricks bundle deploy` can ship the source files across environments. +# When False (default), preparers keep workspace paths in-place; the CLI flips it on so +# `databricks bundle deploy` ships the source files across environments. _downloads_enabled: bool = False def _get_databrickscfg_path() -> Path: - """Return the path to the Databricks CLI config file.""" + """Returns the path to the Databricks CLI config file.""" override = os.environ.get("DATABRICKS_CONFIG_FILE") if override: return Path(override) @@ -34,7 +132,7 @@ def _get_databrickscfg_path() -> Path: def _list_profiles() -> list[str]: - """Parses ``~/.databrickscfg`` and return available profile names. + """Parses ``~/.databrickscfg`` and returns available profile names. Returns: Sorted list of profile section names. Empty list if the file @@ -95,7 +193,7 @@ def _resolve_profile() -> str | None: def _prompt_for_profile(profiles: list[str]) -> str: - """Interactively prompt the user to select a profile. + """Interactively prompts the user to select a profile. Args: profiles: Available profile names. @@ -139,7 +237,7 @@ def _prompt_for_profile(profiles: list[str]) -> str: def set_profile(profile: str | None) -> None: - """Explicitly set the profile to use, bypassing auto-resolution. + """Explicitly sets the profile to use, bypassing auto-resolution. Args: profile: Profile name, or ``None`` to reset to auto-resolution. @@ -150,16 +248,17 @@ def set_profile(profile: str | None) -> None: def _get_workspace_client(): - """Return a ``WorkspaceClient`` configured with the resolved profile. - - Returns: - A ``WorkspaceClient`` instance. + """Returns a ``WorkspaceClient`` configured with the resolved profile. - Raises: - ImportError: If ``databricks-sdk`` is not installed. + On Databricks runtime, auto-configures auth from the notebook context + before constructing the client. """ from databricks.sdk import WorkspaceClient # type: ignore[import-not-found] + # Ensure auth is available when running on Databricks compute + if _is_databricks_runtime(): + _ensure_databricks_runtime_auth() + profile = _resolve_profile() if profile: return WorkspaceClient(profile=profile) @@ -172,7 +271,11 @@ def _get_workspace_client(): def download_notebook(workspace_path: str) -> str | None: - """Download a notebook from Databricks workspace. + """Downloads a notebook from Databricks workspace. + + Attempts local filesystem access first (zero-auth, works on any + Databricks compute where /Workspace is mounted). Falls back to the + Databricks SDK export API when local access is unavailable. Args: workspace_path: Workspace path (e.g., ``"/Shared/flowx/transform"``). @@ -180,6 +283,12 @@ def download_notebook(workspace_path: str) -> str | None: Returns: Notebook source code as a string, or ``None`` if download failed. """ + # Fast path: read directly from /Workspace mount (no auth needed) + local_content = _try_local_workspace_read(workspace_path) + if local_content is not None: + return local_content + + # Slow path: SDK-based export (requires auth) try: from databricks.sdk.service.workspace import ExportFormat # type: ignore[import-not-found] @@ -195,7 +304,7 @@ def download_notebook(workspace_path: str) -> str | None: def download_dbfs_file(dbfs_path: str) -> bytes | None: - """Download a file from DBFS. + """Downloads a file from DBFS. Args: dbfs_path: DBFS path (e.g., ``"dbfs:/scripts/process.py"`` or @@ -223,29 +332,53 @@ def download_dbfs_file(dbfs_path: str) -> bytes | None: def enable_workspace_downloads(enabled: bool = True) -> None: - """Globally enable or disable workspace artifact downloads.""" + """Globally enables or disables workspace artifact downloads.""" global _downloads_enabled # noqa: PLW0603 _downloads_enabled = bool(enabled) def workspace_downloads_enabled() -> bool: - """Return True iff preparers should attempt to download workspace artifacts.""" + """Returns True iff preparers should attempt to download workspace artifacts.""" return _downloads_enabled def auth_available() -> bool: - """Return True iff there is any usable Databricks authentication on this host. - - A resolvable ``.databrickscfg`` profile, ``DATABRICKS_CONFIG_PROFILE``, or - the standard ``DATABRICKS_HOST`` + ``DATABRICKS_TOKEN`` env-var pair will - all satisfy this check. This is a pre-flight signal — it does not validate - that the credentials actually authorize against any specific workspace. + """Returns True iff there is any usable Databricks authentication on this host. + + A resolvable ``.databrickscfg`` profile, ``DATABRICKS_CONFIG_PROFILE``, the + standard ``DATABRICKS_HOST`` + ``DATABRICKS_TOKEN`` pair, or OAuth + machine-to-machine creds (``DATABRICKS_HOST`` + ``DATABRICKS_CLIENT_ID`` + + ``DATABRICKS_CLIENT_SECRET`` — what a Databricks App injects for its service + principal) all satisfy this check. Local /Workspace filesystem access + (available on any Databricks compute) also satisfies it since notebooks can + be read directly without API auth. + + This is a pre-flight signal — it does not validate that the credentials + actually authorize against any specific workspace (the SDK call does that, + falling back to a placeholder on failure). It deliberately avoids + constructing an SDK ``Config``/client, since OAuth resolution can trigger a + network round-trip. """ + if _local_workspace_accessible(): + return True if os.environ.get("DATABRICKS_CONFIG_PROFILE"): return True if os.environ.get("DATABRICKS_HOST") and os.environ.get("DATABRICKS_TOKEN"): return True - return bool(_list_profiles()) + # OAuth M2M (e.g. the MCP path: flowx as a Databricks App injects the SP client id/secret, + # which WorkspaceClient() picks up automatically). + if ( + os.environ.get("DATABRICKS_HOST") + and os.environ.get("DATABRICKS_CLIENT_ID") + and os.environ.get("DATABRICKS_CLIENT_SECRET") + ): + return True + if _list_profiles(): + return True + # Last resort: bootstrap auth from the Databricks runtime context + if _is_databricks_runtime(): + return _ensure_databricks_runtime_auth() + return False def prompt_for_auth_if_missing( @@ -253,7 +386,7 @@ def prompt_for_auth_if_missing( *, interactive: bool | None = None, ) -> bool: - """Warn the user when auth is missing and confirm how to proceed. + """Warns the user when auth is missing and confirms how to proceed. Args: sample_paths: Workspace paths the preparer is about to try to download. @@ -269,7 +402,7 @@ def prompt_for_auth_if_missing( if auth_available(): return True - paths = [p for p in sample_paths if p] + paths = [path for path in sample_paths if path] cfg_path = _get_databrickscfg_path() print( @@ -280,7 +413,7 @@ def prompt_for_auth_if_missing( if paths: preview = ", ".join(paths[:3]) suffix = ", …" if len(paths) > 3 else "" - print(f" Artifacts to vendor: {preview}{suffix}", file=sys.stderr) + print(f" Artifacts to download: {preview}{suffix}", file=sys.stderr) print( "\nTo authenticate, run one of:\n" " databricks auth login --host https://.cloud.databricks.com\n" diff --git a/src/flowx/reporting/__init__.py b/src/flowx/reporting/__init__.py new file mode 100644 index 0000000..443eee4 --- /dev/null +++ b/src/flowx/reporting/__init__.py @@ -0,0 +1 @@ +"""Reporting: persist migration coverage results to a UC table and install a dashboard.""" diff --git a/src/flowx/reporting/coverage.py b/src/flowx/reporting/coverage.py new file mode 100644 index 0000000..356c562 --- /dev/null +++ b/src/flowx/reporting/coverage.py @@ -0,0 +1,117 @@ +"""Build per-pipeline migration-coverage rows from the discover-phase metadata. + +Joins the two artifacts the discover phase writes into ``/metadata/``: + +* ``profile_report.csv`` -- per-pipeline complexity (activity/dataset/linked-service + counts, collapsible patterns, activity-category counts, complexity score + size). +* ``inventory.json`` -- per-activity translation strategy, from which the + deterministic / agentic / unsupported counts and coverage % are derived. + +The result is one metric row per pipeline (no run metadata -- ``run_id`` / +``run_date`` / ``run_by`` are stamped on at write time by :mod:`reporting.results`). +""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +# Metric columns (order matters: it drives the results-table column order). +COVERAGE_METRIC_COLUMNS: tuple[str, ...] = ( + "pipeline", + "activities", + "datasets", + "linked_services", + "collapsible_patterns", + "databricks_native_activities", + "control_flow_activities", + "other_activities", + "deterministic_activities", + "agentic_activities", + "unsupported_activities", + "coverage_pct", + "complexity_score", + "complexity_size", +) + +_CSV_INT_COLUMNS: tuple[str, ...] = ( + "datasets", + "linked_services", + "collapsible_patterns", + "databricks_native_activities", + "control_flow_activities", + "other_activities", + "complexity_score", +) + + +def _coverage_pct(deterministic: int, agentic: int, total: int) -> float: + """Coverage % = (deterministic + agentic) / total activities, rounded to 1dp.""" + if total <= 0: + return 0.0 + return round((deterministic + agentic) / total * 100, 1) + + +def build_coverage_rows(metadata_dir: Path) -> list[dict[str, Any]]: + """Builds per-pipeline coverage rows from a migration ``metadata/`` directory. + + Args: + metadata_dir: The bundle's ``metadata/`` folder containing ``inventory.json`` + and ``profile_report.csv``. + + Returns: + One dict per pipeline keyed by :data:`COVERAGE_METRIC_COLUMNS`, ordered by + pipeline name. The inventory's pipeline set is authoritative; complexity + columns are looked up from the CSV (defaulting to 0 / "" when absent). + + Raises: + FileNotFoundError: When ``inventory.json`` is missing. + """ + inventory_path = metadata_dir / "inventory.json" + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + + csv_by_pipeline: dict[str, dict[str, str]] = {} + csv_path = metadata_dir / "profile_report.csv" + if csv_path.exists(): + with csv_path.open(encoding="utf-8") as handle: + for row in csv.DictReader(handle): + csv_by_pipeline[row["pipeline"]] = row + + rows: list[dict[str, Any]] = [] + for pipeline in inventory.get("pipelines", []): + name = pipeline.get("name", "") + strategies = [activity.get("strategy") for activity in pipeline.get("activities", [])] + deterministic = strategies.count("deterministic") + agentic = strategies.count("agentic") + unsupported = strategies.count("unsupported") + total = len(strategies) + csv_row = csv_by_pipeline.get(name, {}) + + def _csv_int(col: str, _csv_row: dict[str, str] = csv_row) -> int: + try: + return int(_csv_row.get(col, 0) or 0) + except (TypeError, ValueError): + return 0 + + rows.append( + { + "pipeline": name, + "activities": total, + "datasets": _csv_int("datasets"), + "linked_services": _csv_int("linked_services"), + "collapsible_patterns": _csv_int("collapsible_patterns"), + "databricks_native_activities": _csv_int("databricks_native_activities"), + "control_flow_activities": _csv_int("control_flow_activities"), + "other_activities": _csv_int("other_activities"), + "deterministic_activities": deterministic, + "agentic_activities": agentic, + "unsupported_activities": unsupported, + "coverage_pct": _coverage_pct(deterministic, agentic, total), + "complexity_score": _csv_int("complexity_score"), + "complexity_size": csv_row.get("complexity_size", "") or "", + } + ) + rows.sort(key=lambda row: row["pipeline"]) + return rows diff --git a/src/flowx/reporting/dashboard.py b/src/flowx/reporting/dashboard.py new file mode 100644 index 0000000..a950445 --- /dev/null +++ b/src/flowx/reporting/dashboard.py @@ -0,0 +1,97 @@ +"""Install a published AI/BI (Lakeview) dashboard that visualizes migration coverage. + +Builds a dashboard from :data:`dashboard_template.json` (datasets + widgets over the +results table written by :mod:`reporting.results`), creates it via the Databricks SDK +Lakeview API, and publishes it so it is immediately viewable. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +from flowx.reporting.results import resolve_warehouse_id + +logger = logging.getLogger(__name__) + +_TEMPLATE_PATH = Path(__file__).with_name("dashboard_template.json") +_TABLE_PLACEHOLDER = "{{RESULTS_TABLE}}" + + +def build_serialized_dashboard(table_fqn: str) -> str: + """Returns the serialized Lakeview dashboard JSON for *table_fqn*. + + Substitutes the ``{{RESULTS_TABLE}}`` placeholder in every dataset query with the + fully-qualified results table, and returns a compact JSON string suitable for the + Lakeview ``serialized_dashboard`` field. + + Raises: + ValueError: When *table_fqn* is empty. + """ + if not table_fqn: + raise ValueError("table_fqn is required to build the coverage dashboard") + spec = json.loads(_TEMPLATE_PATH.read_text(encoding="utf-8")) + for dataset in spec.get("datasets", []): + dataset["queryLines"] = [line.replace(_TABLE_PLACEHOLDER, table_fqn) for line in dataset.get("queryLines", [])] + return json.dumps(spec) + + +def _default_parent_path(client: Any) -> str: + """Returns ``/Workspace/Users/`` for the dashboard's parent folder.""" + try: + user = client.current_user.me().user_name + if user: + return f"/Workspace/Users/{user}" + except Exception as exc: # noqa: BLE001 - fall back to /Workspace + logger.debug("Could not resolve current user for parent path: %s", exc) + return "/Workspace" + + +def install_dashboard( + table_fqn: str, + warehouse_id: str | None = None, + display_name: str | None = None, + parent_path: str | None = None, + client: Any | None = None, +) -> tuple[str, str]: + """Creates and publishes the migration-coverage dashboard. + + Args: + table_fqn: Results table the dashboard reads (``catalog.schema.table``). + warehouse_id: SQL warehouse backing the dashboard; auto-detected when omitted. + display_name: Dashboard name; defaults to ``Migration Coverage —
``. + parent_path: Workspace folder; defaults to the current user's home. + client: Optional ``WorkspaceClient`` (injected in tests). + + Returns: + ``(dashboard_id, url)`` -- ``url`` is best-effort (empty when host is unknown). + """ + from databricks.sdk.service.dashboards import Dashboard + + if client is None: + from flowx.preparer.workspace_downloader import _get_workspace_client + + client = _get_workspace_client() + + resolved_wh = resolve_warehouse_id(client, warehouse_id) + serialized = build_serialized_dashboard(table_fqn) + name = display_name or f"Migration Coverage — {table_fqn}" + parent = parent_path or _default_parent_path(client) + + created = client.lakeview.create( + dashboard=Dashboard( + display_name=name, + serialized_dashboard=serialized, + warehouse_id=resolved_wh, + parent_path=parent, + ) + ) + dashboard_id = created.dashboard_id + client.lakeview.publish(dashboard_id=dashboard_id, warehouse_id=resolved_wh) + + host = str(getattr(getattr(client, "config", None), "host", "") or "").rstrip("/") + url = f"{host}/sql/dashboardsv3/{dashboard_id}" if host else "" + logger.info("Installed coverage dashboard '%s' (%s).", name, dashboard_id) + return dashboard_id, url diff --git a/src/flowx/reporting/dashboard_template.json b/src/flowx/reporting/dashboard_template.json new file mode 100644 index 0000000..25ed878 --- /dev/null +++ b/src/flowx/reporting/dashboard_template.json @@ -0,0 +1,538 @@ +{ + "datasets": [ + { + "name": "latest_summary", + "displayName": "Latest run summary", + "queryLines": [ + "SELECT COUNT(*) AS pipelines, ", + "SUM(activities) AS activities, ", + "SUM(deterministic_activities) AS deterministic_activities, ", + "SUM(agentic_activities) AS agentic_activities, ", + "SUM(unsupported_activities) AS unsupported_activities, ", + "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(activities),0),1) AS coverage_pct ", + "FROM {{RESULTS_TABLE}} ", + "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1)" + ] + }, + { + "name": "latest_by_size", + "displayName": "Pipelines by complexity (latest run)", + "queryLines": [ + "SELECT complexity_size, COUNT(*) AS pipelines ", + "FROM {{RESULTS_TABLE}} ", + "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1) ", + "GROUP BY complexity_size" + ] + }, + { + "name": "latest_pipelines", + "displayName": "Pipeline coverage (latest run)", + "queryLines": [ + "SELECT pipeline, activities, deterministic_activities, agentic_activities, ", + "unsupported_activities, coverage_pct, collapsible_patterns, complexity_size ", + "FROM {{RESULTS_TABLE}} ", + "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1) ", + "ORDER BY coverage_pct ASC, activities DESC" + ] + }, + { + "name": "runs_over_time", + "displayName": "Coverage over runs", + "queryLines": [ + "SELECT DATE_TRUNC('SECOND', run_date) AS run_ts, ", + "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(activities),0),1) AS coverage_pct, ", + "SUM(activities) AS activities ", + "FROM {{RESULTS_TABLE}} ", + "GROUP BY DATE_TRUNC('SECOND', run_date) ", + "ORDER BY run_ts" + ] + } + ], + "pages": [ + { + "name": "coverage", + "displayName": "Migration Coverage", + "pageType": "PAGE_TYPE_CANVAS", + "layout": [ + { + "widget": { + "name": "title", + "multilineTextboxSpec": { + "lines": [ + "## ADF \u2192 Databricks Migration Coverage" + ] + } + }, + "position": { + "x": 0, + "y": 0, + "width": 6, + "height": 1 + } + }, + { + "widget": { + "name": "subtitle", + "multilineTextboxSpec": { + "lines": [ + "Per-pipeline translation coverage from the latest flowx run. Coverage % = (deterministic + agentic) / total activities." + ] + } + }, + "position": { + "x": 0, + "y": 1, + "width": 6, + "height": 1 + } + }, + { + "widget": { + "name": "kpi-pipelines", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "pipelines", + "expression": "`pipelines`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "pipelines", + "displayName": "Pipelines" + } + }, + "frame": { + "title": "Pipelines", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 2, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "kpi-coverage", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "coverage_pct", + "expression": "`coverage_pct`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "coverage_pct", + "displayName": "Coverage %" + } + }, + "frame": { + "title": "Coverage %", + "showTitle": true + } + } + }, + "position": { + "x": 2, + "y": 2, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "kpi-activities", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "activities", + "expression": "`activities`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "activities", + "displayName": "Activities" + } + }, + "frame": { + "title": "Activities", + "showTitle": true + } + } + }, + "position": { + "x": 4, + "y": 2, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "kpi-det", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "deterministic_activities", + "expression": "`deterministic_activities`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "deterministic_activities", + "displayName": "Deterministic" + } + }, + "frame": { + "title": "Deterministic", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 5, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "kpi-agentic", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "agentic_activities", + "expression": "`agentic_activities`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "agentic_activities", + "displayName": "Agentic" + } + }, + "frame": { + "title": "Agentic", + "showTitle": true + } + } + }, + "position": { + "x": 2, + "y": 5, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "kpi-unsup", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "unsupported_activities", + "expression": "`unsupported_activities`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "unsupported_activities", + "displayName": "Unsupported" + } + }, + "frame": { + "title": "Unsupported", + "showTitle": true + } + } + }, + "position": { + "x": 4, + "y": 5, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "by-size", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_by_size", + "fields": [ + { + "name": "complexity_size", + "expression": "`complexity_size`" + }, + { + "name": "pipelines", + "expression": "`pipelines`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 3, + "widgetType": "bar", + "encodings": { + "x": { + "fieldName": "complexity_size", + "scale": { + "type": "categorical" + }, + "displayName": "Complexity" + }, + "y": { + "fieldName": "pipelines", + "scale": { + "type": "quantitative" + }, + "displayName": "Pipelines" + } + }, + "frame": { + "title": "Pipelines by complexity size", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 8, + "width": 3, + "height": 6 + } + }, + { + "widget": { + "name": "coverage-trend", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "runs_over_time", + "fields": [ + { + "name": "run_ts", + "expression": "`run_ts`" + }, + { + "name": "coverage_pct", + "expression": "`coverage_pct`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 3, + "widgetType": "line", + "encodings": { + "x": { + "fieldName": "run_ts", + "scale": { + "type": "temporal" + }, + "displayName": "Run" + }, + "y": { + "fieldName": "coverage_pct", + "scale": { + "type": "quantitative" + }, + "displayName": "Coverage %" + } + }, + "frame": { + "title": "Coverage over runs", + "showTitle": true + } + } + }, + "position": { + "x": 3, + "y": 8, + "width": 3, + "height": 6 + } + }, + { + "widget": { + "name": "pipeline-table", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_pipelines", + "fields": [ + { + "name": "pipeline", + "expression": "`pipeline`" + }, + { + "name": "activities", + "expression": "`activities`" + }, + { + "name": "deterministic_activities", + "expression": "`deterministic_activities`" + }, + { + "name": "agentic_activities", + "expression": "`agentic_activities`" + }, + { + "name": "unsupported_activities", + "expression": "`unsupported_activities`" + }, + { + "name": "coverage_pct", + "expression": "`coverage_pct`" + }, + { + "name": "collapsible_patterns", + "expression": "`collapsible_patterns`" + }, + { + "name": "complexity_size", + "expression": "`complexity_size`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "table", + "encodings": { + "columns": [ + { + "fieldName": "pipeline", + "displayName": "Pipeline" + }, + { + "fieldName": "activities", + "displayName": "Activities" + }, + { + "fieldName": "deterministic_activities", + "displayName": "Deterministic" + }, + { + "fieldName": "agentic_activities", + "displayName": "Agentic" + }, + { + "fieldName": "unsupported_activities", + "displayName": "Unsupported" + }, + { + "fieldName": "coverage_pct", + "displayName": "Coverage %" + }, + { + "fieldName": "collapsible_patterns", + "displayName": "Collapsible patterns" + }, + { + "fieldName": "complexity_size", + "displayName": "Complexity" + } + ] + }, + "frame": { + "title": "Pipeline coverage detail", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 14, + "width": 6, + "height": 7 + } + } + ] + } + ] +} \ No newline at end of file diff --git a/src/flowx/reporting/results.py b/src/flowx/reporting/results.py new file mode 100644 index 0000000..b5d3af7 --- /dev/null +++ b/src/flowx/reporting/results.py @@ -0,0 +1,171 @@ +"""Persist per-pipeline migration coverage to a Unity Catalog table. + +Each migration run is stamped with a single UUID ``run_id`` (shared by every row of +the run), ``run_date`` (``CURRENT_TIMESTAMP()``), and ``run_by`` (``CURRENT_USER()``), +so coverage can be tracked over time and per user. Rows are written via the +Databricks SDK Statement Execution API against a SQL warehouse (auto-detected when one +is not supplied). +""" + +from __future__ import annotations + +import logging +import uuid +from pathlib import Path +from typing import Any + +from flowx.reporting.coverage import COVERAGE_METRIC_COLUMNS, build_coverage_rows + +logger = logging.getLogger(__name__) + +# Full results-table schema: run metadata first, then the per-pipeline metrics. +# ``run_date`` / ``run_by`` are populated by SQL functions (not Python literals). +_METRIC_SQL_TYPES: dict[str, str] = { + "pipeline": "STRING", + "activities": "INT", + "datasets": "INT", + "linked_services": "INT", + "collapsible_patterns": "INT", + "databricks_native_activities": "INT", + "control_flow_activities": "INT", + "other_activities": "INT", + "deterministic_activities": "INT", + "agentic_activities": "INT", + "unsupported_activities": "INT", + "coverage_pct": "DOUBLE", + "complexity_score": "INT", + "complexity_size": "STRING", +} + +RESULTS_COLUMNS: tuple[tuple[str, str], ...] = ( + ("run_id", "STRING"), + ("run_date", "TIMESTAMP"), + ("run_by", "STRING"), + *((col, _METRIC_SQL_TYPES[col]) for col in COVERAGE_METRIC_COLUMNS), +) + +_STRING_METRICS: frozenset[str] = frozenset({"pipeline", "complexity_size"}) + + +def _sql_str(value: Any) -> str: + """Renders a value as a single-quoted SQL string literal (quotes doubled).""" + return "'" + str(value).replace("'", "''") + "'" + + +def _metric_value_sql(column: str, value: Any) -> str: + """Renders one metric column value as a SQL literal.""" + if column in _STRING_METRICS: + return _sql_str(value) + if column == "coverage_pct": + return repr(float(value or 0)) + return str(int(value or 0)) + + +def build_create_table_sql(table_fqn: str) -> str: + """Returns ``CREATE TABLE IF NOT EXISTS`` for the results table.""" + cols = ",\n ".join(f"{name} {sql_type}" for name, sql_type in RESULTS_COLUMNS) + return f"CREATE TABLE IF NOT EXISTS {table_fqn} (\n {cols}\n)" + + +def build_insert_sql(table_fqn: str, rows: list[dict[str, Any]], run_id: str) -> str: + """Returns a single multi-row ``INSERT`` stamping run metadata onto every row. + + ``run_id`` is a literal (same for the whole run); ``run_date`` and ``run_by`` use + the ``CURRENT_TIMESTAMP()`` / ``CURRENT_USER()`` SQL functions so the workspace + records the actual write time and identity. + """ + column_names = ", ".join(name for name, _ in RESULTS_COLUMNS) + tuples: list[str] = [] + for row in rows: + metrics = ", ".join(_metric_value_sql(col, row.get(col)) for col in COVERAGE_METRIC_COLUMNS) + tuples.append(f"({_sql_str(run_id)}, CURRENT_TIMESTAMP(), CURRENT_USER(), {metrics})") + values = ",\n ".join(tuples) + return f"INSERT INTO {table_fqn} ({column_names}) VALUES\n {values}" + + +def resolve_warehouse_id(client: Any, warehouse_id: str | None = None) -> str: + """Resolves a SQL warehouse id, preferring RUNNING then serverless when auto-detecting. + + Args: + client: A ``WorkspaceClient``. + warehouse_id: An explicit id; returned as-is when provided. + + Returns: + The resolved warehouse id. + + Raises: + RuntimeError: When no warehouse is available to auto-detect. + """ + if warehouse_id: + return warehouse_id + warehouses = list(client.warehouses.list()) + if not warehouses: + raise RuntimeError( + "No SQL warehouse found to write results. Pass --warehouse-id with a warehouse " + "that can write to the target table." + ) + + def _rank(warehouse: Any) -> tuple[int, int]: + state = str(getattr(getattr(warehouse, "state", None), "value", getattr(warehouse, "state", "")) or "") + is_running = 1 if state.upper() == "RUNNING" else 0 + type_value = str( + getattr(getattr(warehouse, "warehouse_type", None), "value", getattr(warehouse, "warehouse_type", "")) or "" + ) + is_serverless = ( + 1 if getattr(warehouse, "enable_serverless_compute", False) or "SERVERLESS" in type_value.upper() else 0 + ) + return (is_running, is_serverless) + + best = max(warehouses, key=_rank) + logger.info("Auto-selected SQL warehouse '%s' (%s).", getattr(best, "name", "?"), best.id) + return best.id + + +def _execute(client: Any, statement: str, warehouse_id: str) -> None: + """Runs a SQL statement via the Statement Execution API; raises on failure.""" + resp = client.statement_execution.execute_statement( + statement=statement, warehouse_id=warehouse_id, wait_timeout="50s" + ) + state = getattr(getattr(resp, "status", None), "state", None) + state_str = str(getattr(state, "value", state) or "") + if state_str.upper() not in ("", "SUCCEEDED"): + err = getattr(getattr(resp, "status", None), "error", None) + raise RuntimeError(f"Statement failed ({state_str}): {getattr(err, 'message', err)}") + + +def write_results( + metadata_dir: Path, + table_fqn: str, + warehouse_id: str | None = None, + client: Any | None = None, +) -> tuple[str, int]: + """Writes per-pipeline coverage rows for one run to *table_fqn*. + + Creates the table if needed, then inserts one row per pipeline stamped with a + fresh ``run_id`` (and SQL ``run_date`` / ``run_by``). + + Args: + metadata_dir: The migration ``metadata/`` directory. + table_fqn: Target table as ``catalog.schema.table``. + warehouse_id: Optional SQL warehouse id; auto-detected when omitted. + client: Optional ``WorkspaceClient`` (injected in tests). + + Returns: + ``(run_id, row_count)``. + """ + rows = build_coverage_rows(metadata_dir) + if not rows: + logger.warning("No pipelines found in %s; nothing to record.", metadata_dir) + return "", 0 + + if client is None: + from flowx.preparer.workspace_downloader import _get_workspace_client + + client = _get_workspace_client() + + resolved_wh = resolve_warehouse_id(client, warehouse_id) + run_id = str(uuid.uuid4()) + _execute(client, build_create_table_sql(table_fqn), resolved_wh) + _execute(client, build_insert_sql(table_fqn, rows, run_id), resolved_wh) + logger.info("Recorded %d pipeline rows to %s (run_id=%s).", len(rows), table_fqn, run_id) + return run_id, len(rows) diff --git a/src/orchestra/translator/__init__.py b/src/flowx/translator/__init__.py similarity index 100% rename from src/orchestra/translator/__init__.py rename to src/flowx/translator/__init__.py diff --git a/src/orchestra/translator/activity_translators/__init__.py b/src/flowx/translator/activity_translators/__init__.py similarity index 100% rename from src/orchestra/translator/activity_translators/__init__.py rename to src/flowx/translator/activity_translators/__init__.py diff --git a/src/orchestra/translator/activity_translators/append_variable.py b/src/flowx/translator/activity_translators/append_variable.py similarity index 100% rename from src/orchestra/translator/activity_translators/append_variable.py rename to src/flowx/translator/activity_translators/append_variable.py diff --git a/src/orchestra/translator/activity_translators/copy.py b/src/flowx/translator/activity_translators/copy.py similarity index 96% rename from src/orchestra/translator/activity_translators/copy.py rename to src/flowx/translator/activity_translators/copy.py index 4fdf14d..af595a4 100644 --- a/src/orchestra/translator/activity_translators/copy.py +++ b/src/flowx/translator/activity_translators/copy.py @@ -25,11 +25,8 @@ "DeltaLakeDataset": "delta", } -# Map ADF dataset location types to (uri-scheme, host-template) pairs used -# when constructing the external-volume URL. ``{account}`` is replaced with -# the storage account name (or a ``${var.storage_account}`` placeholder when -# the linked service does not expose one) and ``{bucket}`` with the bucket -# name for AWS / GCS sinks. +# Maps ADF dataset location types to (uri-scheme, host-template) pairs for the external-volume URL; +# {account} -> storage account (or ${var.storage_account}) and {bucket} -> bucket name for AWS/GCS. _LOCATION_URL_TEMPLATE: dict[str, str] = { "AzureBlobFSLocation": "abfss://{container}@{account}.dfs.core.windows.net/", "AzureBlobStorageLocation": "abfss://{container}@{account}.dfs.core.windows.net/", @@ -42,11 +39,8 @@ _ACCOUNT_NAME_RE = re.compile(r"AccountName=([A-Za-z0-9]+)", re.IGNORECASE) _DATASET_PARAM_RE = re.compile(r"^@dataset\(\)\.([A-Za-z_][A-Za-z0-9_]*)$") -# Database connection-string parsers: each picks up the canonical -# host/port/database fields from an ADF linked service's -# ``connectionString``. The patterns are intentionally tolerant of -# casing and surrounding whitespace because ADF accepts both -# ``Server=`` and ``server=`` etc. +# Database connection-string parsers: pull host/port/database from an ADF linked service's +# connectionString. Tolerant of casing/whitespace (ADF accepts Server= and server=). _AZURE_SQL_SERVER_RE = re.compile(r"\bServer=(?:tcp:)?([^,;]+?)(?:,(\d+))?(?:;|$)", re.IGNORECASE) _AZURE_SQL_DATABASE_RE = re.compile(r"\b(?:Initial Catalog|Database)=([^;]+)", re.IGNORECASE) _MYSQL_SERVER_RE = re.compile(r"\b(?:Server|Host)=([^;]+)", re.IGNORECASE) @@ -231,9 +225,8 @@ def _resolve_path_info( if dataset_ref is not None and getattr(dataset_ref, "parameters", None): effective.update(dict(dataset_ref.parameters)) - # Container name is used in the volume URL (no expressions allowed); the - # other path components flow into the notebook write call as f-string - # fragments so date/time expressions evaluate at runtime. + # Container name goes in the volume URL (no expressions allowed); other path components flow into + # the notebook write as f-string fragments so date/time expressions evaluate at runtime. container = _resolve_param_value( location.get("container") or location.get("fileSystem") or location.get("bucketName"), effective, @@ -623,10 +616,8 @@ def translate( sink_dataset_type = sink_dataset_props.get("type") sink_format = _DATASET_TYPE_TO_SPARK_FORMAT.get(sink_dataset_type or "") - # File-on-cloud-storage sinks: compose a UC external volume - # path so the notebook writes through Unity Catalog and the - # bundler can emit the matching SetupTask (storage credential - # + external location + external volume). + # File-on-cloud-storage sinks: compose a UC external-volume path so the notebook writes + # through Unity Catalog and the bundler emits the matching SetupTask. sink_path_info = _resolve_path_info(sink_dataset_ref, sink_dataset_props, definitions, context) if sink_path_info is not None: sink_resolved_path = sink_path_info.uc_volume_path diff --git a/src/orchestra/translator/activity_translators/databricks_job.py b/src/flowx/translator/activity_translators/databricks_job.py similarity index 100% rename from src/orchestra/translator/activity_translators/databricks_job.py rename to src/flowx/translator/activity_translators/databricks_job.py diff --git a/src/orchestra/translator/activity_translators/delete.py b/src/flowx/translator/activity_translators/delete.py similarity index 100% rename from src/orchestra/translator/activity_translators/delete.py rename to src/flowx/translator/activity_translators/delete.py diff --git a/src/orchestra/translator/activity_translators/execute_pipeline.py b/src/flowx/translator/activity_translators/execute_pipeline.py similarity index 100% rename from src/orchestra/translator/activity_translators/execute_pipeline.py rename to src/flowx/translator/activity_translators/execute_pipeline.py diff --git a/src/orchestra/translator/activity_translators/filter.py b/src/flowx/translator/activity_translators/filter.py similarity index 82% rename from src/orchestra/translator/activity_translators/filter.py rename to src/flowx/translator/activity_translators/filter.py index 2f3a104..439cf21 100644 --- a/src/orchestra/translator/activity_translators/filter.py +++ b/src/flowx/translator/activity_translators/filter.py @@ -9,11 +9,8 @@ from flowx.models.ir import Activity, FilterActivity, TranslationContext from flowx.parser.expression_parser import resolve_expression -# The expression resolver translates ``item().X`` into -# ``dbutils.widgets.get('X')`` because ``{{input.X}}`` is the DAB ref it -# emits for ForEach-iteration item access. Inside a Filter notebook the -# items array is iterated locally with a Python ``item`` dict per -# iteration, so we rewrite each widget read to a dict lookup. +# The resolver maps item().X to dbutils.widgets.get('X') (the DAB ForEach item ref). Inside a Filter +# notebook the array is iterated locally, so each widget read is rewritten to a dict lookup. _WIDGET_ITEM_ACCESS_RE = re.compile(r"""dbutils\.widgets\.get\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\)""") @@ -34,9 +31,8 @@ def translate( else: items_expression = items_raw.get("value", "") if isinstance(items_raw, dict) else str(items_raw) - # Preserve the original ADF expression text in ``condition_expression`` - # so the notebook can show it as a documentation comment. The - # *resolved* form lives separately in ``condition_code``. + # Preserve the original ADF expression text in condition_expression for a doc comment; the + # resolved form lives in condition_code. condition_expression = condition_raw.get("value", "") if isinstance(condition_raw, dict) else str(condition_raw) condition_result = resolve_expression(condition_raw, context) diff --git a/src/orchestra/translator/activity_translators/for_each.py b/src/flowx/translator/activity_translators/for_each.py similarity index 90% rename from src/orchestra/translator/activity_translators/for_each.py rename to src/flowx/translator/activity_translators/for_each.py index f13b655..12866d1 100644 --- a/src/orchestra/translator/activity_translators/for_each.py +++ b/src/flowx/translator/activity_translators/for_each.py @@ -41,12 +41,8 @@ def translate( if expr_result is not None and expr_result.kind in ("dab_ref", "literal"): items_expression = expr_result.value elif expr_result is not None and expr_result.kind == "notebook_code": - # C-31 (CF4-001): the preparer used to construct a bare - # TranslationContext() and re-resolve the items expression on the - # JSON-reload path, but ``variable_cache`` is empty there so the - # bridge never fired and DAB rejected the raw @split(...) call. - # Capture the resolved notebook_code here while the full context - # is available; the preparer reads it from these IR fields. + # C-31 (CF4-001): capture the resolved items notebook_code here while the full context is + # available; the JSON-reload path has an empty variable_cache so it can't re-resolve @split(...). if isinstance(items_raw, dict) and items_raw.get("type") == "Expression": items_expression = items_raw.get("value", "") elif isinstance(items_raw, str): diff --git a/src/orchestra/translator/activity_translators/if_condition.py b/src/flowx/translator/activity_translators/if_condition.py similarity index 84% rename from src/orchestra/translator/activity_translators/if_condition.py rename to src/flowx/translator/activity_translators/if_condition.py index 86db2ff..9131222 100644 --- a/src/orchestra/translator/activity_translators/if_condition.py +++ b/src/flowx/translator/activity_translators/if_condition.py @@ -156,9 +156,8 @@ def _parse_condition( if adf_op == "not": inner = m.group(2).strip() resolved, bridge = _resolve_operand(inner, context) - # C-15 (CF3-003 / VAREX3-004): when the operand bridges to a - # Python bool task value, compare against 'False' (not '') so - # the IfCondition can actually evaluate to FALSE. + # C-15 (CF3-003/VAREX3-004): when the operand bridges to a Python bool task value, compare + # against 'False' (not '') so the IfCondition can evaluate to FALSE. right_operand = "False" if bridge is not None else "" return "NOT_EQUAL", resolved, right_operand, bridge @@ -167,39 +166,24 @@ def _parse_condition( right, right_bridge = _resolve_operand(args[1], context) if len(args) > 1 else ("", None) return op, left, right, merge_bridge_requests(left_bridge, right_bridge) - # Fallback: treat the whole expression as a truthy check. C-07: route - # through the bridge path when the expression is an ADF function call - # so the operand ends up as a real task-value reference rather than - # the legacy NOT_EQUAL '0' against a raw expression string. + # Fallback truthy check. C-07: route an ADF function call through the bridge path so the operand + # becomes a real task-value ref rather than legacy NOT_EQUAL '0' against a raw expression string. resolved, bridge = _resolve_operand(expr_str, context) if bridge is not None: return "NOT_EQUAL", _bridge_task_value_placeholder(), "False", bridge - # C-15 (CF3-003 / VAREX3-004): when the truthy operand resolves to a - # task-value ref backed by a SetVariable that writes a Python bool - # (e.g. a previously-cached @variables('continue') with bridge-set - # value), compare against 'False' so the legacy truthy path doesn't - # silently invert behaviour. Detected by the presence of a - # __BRIDGE__:: placeholder or the lowercase 'true'/'false' literal - # body of the upstream SetVariable. + # C-15 (CF3-003/VAREX3-004): when the truthy operand is a task-value ref backed by a Python-bool + # SetVariable (detected via a __BRIDGE__:: placeholder or 'true'/'false' body), compare against 'False'. if isinstance(resolved, str) and "__BRIDGE__" in resolved: return "NOT_EQUAL", resolved, "False", None - # C-43 (CF5-001 / LSC5-001): when the operand is a known-Boolean - # variable that resolves to a parent-job task-value ref - # (``{{tasks._init_X.values.X}}``), prefer recomputing the boolean - # locally via a BridgeRequest, mirroring the Switch path. Without this - # an inner-ForEach IfCondition references a task that lives only in the - # parent job; the bundler then blanks the operand to '' and - # NOT_EQUAL('', '0') is always TRUE, running the true branch - # unconditionally with no SETUP.md signal. The bridge keeps the - # operand local so it survives the dangling-ref safety net. + # C-43 (CF5-001/LSC5-001): for a known-Boolean variable resolving to a parent-job task-value ref, + # recompute the boolean locally via a BridgeRequest (mirroring Switch). Otherwise an inner-ForEach + # IfCondition references a parent-only task, the bundler blanks the operand, and the true branch always runs. if _operand_is_known_boolean(expr_str, context): bridge = _boolean_variable_bridge(expr_str, resolved, context) if bridge is not None: return "NOT_EQUAL", _bridge_task_value_placeholder(), "False", bridge - # C-32 (CF4-002): compare against lowercase ``'false'`` (matching - # C-21 SetVariable rendering) instead of the legacy ``'0'`` — the - # latter is always true for a Boolean-string operand so the false - # branch becomes dead code. + # C-32 (CF4-002): compare against lowercase 'false' (matching C-21 SetVariable rendering), not + # legacy '0' which is always true for a Boolean-string operand (making the false branch dead code). return "NOT_EQUAL", resolved, "false", None return "NOT_EQUAL", resolved, "0", None @@ -251,8 +235,7 @@ def _operand_is_known_boolean(expr: str, context: TranslationContext) -> bool: cached = context.get_variable_dab_ref(var_name) if isinstance(cached, str) and cached.lower() in ("true", "false"): return True - # C-41 (CF5-001): a Boolean variable seeded only by a literal - # default init task never populates variable_value_cache as a + # C-41 (CF5-001): a Boolean variable seeded only by a literal default never caches as a # dab_ref, so fall back to its declared ADF type. declared = context.get_variable_type(var_name) if isinstance(declared, str) and declared.lower() in ("boolean", "bool"): @@ -339,23 +322,23 @@ def _split_args(args_str: str) -> list[str]: current: list[str] = [] in_quote = False - for ch in args_str: - if ch == "'" and depth == 0: + for char in args_str: + if char == "'" and depth == 0: in_quote = not in_quote - current.append(ch) + current.append(char) elif in_quote: - current.append(ch) - elif ch == "(": + current.append(char) + elif char == "(": depth += 1 - current.append(ch) - elif ch == ")": + current.append(char) + elif char == ")": depth -= 1 - current.append(ch) - elif ch == "," and depth == 0: + current.append(char) + elif char == "," and depth == 0: parts.append("".join(current).strip()) current = [] else: - current.append(ch) + current.append(char) if current: parts.append("".join(current).strip()) diff --git a/src/orchestra/translator/activity_translators/lookup.py b/src/flowx/translator/activity_translators/lookup.py similarity index 86% rename from src/orchestra/translator/activity_translators/lookup.py rename to src/flowx/translator/activity_translators/lookup.py index 674ad00..1700e4f 100644 --- a/src/orchestra/translator/activity_translators/lookup.py +++ b/src/flowx/translator/activity_translators/lookup.py @@ -110,9 +110,8 @@ def translate( first_row_only = type_properties.get("firstRowOnly", True) - # Resolve the Lookup's dataset reference (lookup-translator-ignores-dataset-reference): - # typeProperties.dataset is the canonical place for ADF; activity.inputs - # is the legacy fall-back used by the loader for flattened activity shapes. + # Resolve the Lookup's dataset reference: typeProperties.dataset is canonical; activity.inputs is the + # legacy fall-back the loader uses for flattened activity shapes. dataset_ref = _resolve_lookup_dataset(activity, definitions) if dataset_ref is not None: dataset_props = dataset_ref["properties"] @@ -121,22 +120,15 @@ def translate( location = type_props.get("location") or {} if dataset_type in _FILE_DATASET_TYPES: source_properties.setdefault("dataset_type", dataset_type) - # Stash the dataset path components so the code generator can - # build the right spark.read call. Avoid pulling in the full - # copy translator dataset-path machinery — we only need the - # raw container + folder + filename to surface to the user. - # C-37 (LSC4-001): unwrap any ADF expression dict shapes so - # downstream code can treat these as plain strings. + # Stash the dataset path components (container/folder/filename) for the code generator's + # spark.read call. C-37 (LSC4-001): unwrap any ADF expression dict shapes to plain strings. container = _unwrap_expression( location.get("container") or location.get("fileSystem") or location.get("bucketName") ) folder = _unwrap_expression(location.get("folderPath")) filename = _unwrap_expression(location.get("fileName")) - # C-47 (LSC5-001): substitute dataset().X param refs using the - # Lookup dataset reference's parameter bindings, then resolve the - # result so the path default is a real literal / interpolated - # {{job.parameters.X}} string rather than a verbatim dataset() - # expression the code generator would bake into a broken path. + # C-47 (LSC5-001): substitute dataset().X param refs from the dataset reference's bindings, then + # resolve so the path is a real literal/{{job.parameters.X}} string, not a verbatim dataset() expr. ds_scope = _dataset_parameter_scope(activity, context) if ds_scope: if isinstance(folder, str) and folder: @@ -156,9 +148,8 @@ def translate( for key in ("multiLineJson", "filePattern"): if key in format_settings: source_properties.setdefault(key, format_settings[key]) - # LSC3-005: surface the linked service URL when present so the - # generator can assemble the abfss:// path for AzureBlobFS / ADLS - # backed file datasets. + # LSC3-005: surface the linked service URL when present so the generator can assemble the + # abfss:// path for AzureBlobFS/ADLS-backed file datasets. ls_url = dataset_props.get("linked_service_url") if ls_url: source_properties.setdefault("linked_service_url", ls_url) @@ -201,9 +192,8 @@ def _resolve_lookup_dataset( if dataset is None: return None properties = dict(dataset.properties or {}) - # Thread linkedService typeProperties.url through onto the properties so - # the lookup notebook can assemble the abfss:// file path for file-source - # datasets where the URL is only known on the linked service. + # Thread linkedService typeProperties.url onto the properties so the lookup notebook can assemble the + # abfss:// file path for file-source datasets where the URL is only on the linked service. linked_service = definitions.get_linked_service(dataset.linked_service_name) if linked_service is not None: ls_props = linked_service.properties or {} diff --git a/src/orchestra/translator/activity_translators/notebook.py b/src/flowx/translator/activity_translators/notebook.py similarity index 86% rename from src/orchestra/translator/activity_translators/notebook.py rename to src/flowx/translator/activity_translators/notebook.py index 986de03..74dd5bc 100644 --- a/src/orchestra/translator/activity_translators/notebook.py +++ b/src/flowx/translator/activity_translators/notebook.py @@ -30,10 +30,8 @@ def translate( """ type_properties = activity.type_properties or {} - # C-28 (NB-ITER4-001): when notebookPath is an ADF expression that lowers - # to notebook_code (e.g. @trim(json(...).notebook_path)), preserve the raw - # expression and mark the activity so the preparer emits a dispatch stub - # rather than inlining Python source as the workspace path. + # C-28 (NB-ITER4-001): when notebookPath lowers to notebook_code (e.g. @trim(json(...).notebook_path)), + # preserve the raw expression and mark the activity so the preparer emits a dispatch stub, not inline Python. notebook_path_raw = type_properties.get("notebookPath", "") notebook_path, notebook_path_unresolved, notebook_path_expression = _resolve_notebook_path_field( notebook_path_raw, context @@ -41,12 +39,8 @@ def translate( raw_params = type_properties.get("baseParameters") or {} libraries, unresolved_libraries = _resolve_libraries(type_properties.get("libraries"), context) - # Resolve base_parameters at translate time so ADF expressions like - # @variables('runTimestamp') are inlined to DAB refs while the full - # translation context (with variable_value_cache) is available. Any - # caveat notes the resolver emits (e.g. utcnow() approximations) are - # captured into parameter_approximations so the bundler can surface - # them in SETUP.md. + # Resolve base_parameters at translate time (while variable_value_cache is available) so ADF + # expressions inline to DAB refs; resolver caveats (e.g. utcnow()) go to parameter_approximations for SETUP.md. resolved_params: dict[str, Any] = {} approximations: list[dict[str, str]] = [] for key, value in raw_params.items(): @@ -122,10 +116,8 @@ def _raw_expression_text(value: Any) -> str: return str(value) -# Library entry keys that may carry ADF expressions (jar/whl paths, -# maven coordinates with @concat, etc). PyPI uses ``package`` and CRAN -# uses ``package``; we walk all of them through the resolver and only -# emit the entry when every expression resolves to a clean literal/dab_ref. +# Library entry keys that may carry ADF expressions (jar/whl paths, maven coords with @concat); each is +# walked through the resolver and only emitted when every expression resolves to a clean literal/dab_ref. _LIBRARY_VALUE_KEYS: tuple[str, ...] = ("jar", "whl", "egg", "requirements") @@ -188,19 +180,15 @@ def _resolve_libraries( for key, value in lib.items(): if key in _LIBRARY_VALUE_KEYS and isinstance(value, (str, dict)): result = resolve_expression(value, context) - # C-13 (NB-ITER3-004): accept both literal and dab_ref so a - # jar path like @pipeline().parameters.libName collapses to - # {{job.parameters.libName}} (symmetric with custom_tags - # resolution in _resolve_ls_parameters). + # C-13 (NB-ITER3-004): accept both literal and dab_ref so a jar path like + # @pipeline().parameters.libName collapses to {{job.parameters.libName}}. if result is not None and result.kind in ("literal", "dab_ref"): resolved_entry[key] = result.value else: expression_text = _raw_expression_text(value) resolved_entry[key] = value - # Only surface library entries whose value carried an - # ADF expression (starts with ``@``). Bare literal - # paths that already resolved successfully don't need a - # SETUP.md callout. + # Only surface library entries whose value carried an ADF expression (starts with @); + # already-resolved literal paths need no SETUP.md callout. if isinstance(expression_text, str) and expression_text.startswith("@"): unresolved.append( { diff --git a/src/orchestra/translator/activity_translators/resolve.py b/src/flowx/translator/activity_translators/resolve.py similarity index 92% rename from src/orchestra/translator/activity_translators/resolve.py rename to src/flowx/translator/activity_translators/resolve.py index 7f8d434..b2f8e77 100644 --- a/src/orchestra/translator/activity_translators/resolve.py +++ b/src/flowx/translator/activity_translators/resolve.py @@ -63,12 +63,12 @@ def merge_bridge_requests(*requests: BridgeRequest | None) -> BridgeRequest | No single bridge task with a boolean truthiness result. Returns ``None`` when no non-None requests are supplied. """ - populated = [r for r in requests if r is not None] + populated = [request for request in requests if request is not None] if not populated: return None if len(populated) == 1: return populated[0] - expression = " and ".join(f"({r.notebook_code})" for r in populated) + expression = " and ".join(f"({request.notebook_code})" for request in populated) imports: list[str] = [] required: dict[str, str] = {} for req in populated: @@ -129,16 +129,16 @@ def resolve_field_int(value: Any, context: TranslationContext, default: int = 0) return default -def resolve_dict_values(d: dict[str, Any] | None, context: TranslationContext) -> dict[str, str]: +def resolve_dict_values(fields: dict[str, Any] | None, context: TranslationContext) -> dict[str, str]: """Resolves all values in a dict that may contain ADF expressions. Args: - d: Dict of field name to raw values. + fields: Dict of field name to raw values. context: Translation context for variable resolution. Returns: Dict with all values resolved to strings. """ - if not d: + if not fields: return {} - return {k: resolve_field(v, context) for k, v in d.items()} + return {key: resolve_field(value, context) for key, value in fields.items()} diff --git a/src/orchestra/translator/activity_translators/set_variable.py b/src/flowx/translator/activity_translators/set_variable.py similarity index 79% rename from src/orchestra/translator/activity_translators/set_variable.py rename to src/flowx/translator/activity_translators/set_variable.py index 6f127ea..9e93d5f 100644 --- a/src/orchestra/translator/activity_translators/set_variable.py +++ b/src/flowx/translator/activity_translators/set_variable.py @@ -77,14 +77,9 @@ def translate( variable_name = type_properties.get("variableName", "") value_raw = type_properties.get("value", "") - # C-42 (VAREX5-001): a Set Pipeline Return Value activity carries a - # list of {key, value} pairs (e.g. - # [{'key': 'result', 'value': {'type': 'Expression', - # 'content': "@variables('executionOutputs')"}}]). The legacy path - # fails _is_adf_expression and stringifies the whole list, which the - # bundler then blanks. The inner expression is resolvable, so unwrap a - # single pair's value and route it through the normal resolution - # pipeline instead of losing the reference. + # C-42 (VAREX5-001): a Set Pipeline Return Value activity carries a list of {key, value} pairs whose + # inner value is a resolvable expression. The legacy path stringifies the whole list (then blanked by + # the bundler), so unwrap a single pair's value and route it through normal resolution. value_raw = _unwrap_return_value_pairs(value_raw) expr_result = resolve_expression(value_raw, context) @@ -98,12 +93,9 @@ def translate( notebook_imports = list(expr_result.imports) if expr_result.kind == "notebook_code" else [] required_parameters = dict(expr_result.required_parameters) elif _is_adf_expression(value_raw): - # C-33 (VAREX4-001 / CF4-003): when the value is an ADF expression - # the resolver couldn't handle (e.g. a nested function call we - # don't model), do NOT stamp value_kind='literal' with the raw - # @concat text — that ships uninterpretable Python source through - # SETUP.md. Blank the value and mark it unresolved so the bundler - # emits a manual_variable_init SetupTask the user can act on. + # C-33 (VAREX4-001/CF4-003): when the resolver can't handle the value, do NOT stamp + # value_kind='literal' with raw @concat text (uninterpretable Python). Blank it and mark + # unresolved so the bundler emits a manual_variable_init SetupTask. variable_value = "" value_kind = "unresolved" notebook_code = None @@ -115,10 +107,8 @@ def translate( elif isinstance(value_raw, str): variable_value = value_raw elif isinstance(value_raw, bool): - # VAREX3-002: render Python bool as lowercase 'true'/'false' so - # downstream ADF comparisons like @equals(variables('X'), true) - # match consistently. ``str(True)`` would emit 'True' and silently - # invert the comparison. + # VAREX3-002: render Python bool as lowercase 'true'/'false' so @equals(variables('X'), true) + # matches; str(True) would emit 'True' and silently invert the comparison. variable_value = "true" if value_raw else "false" else: variable_value = str(value_raw) @@ -137,10 +127,8 @@ def translate( raw_expression=raw_expression_text if value_kind == "unresolved" else None, ) - # Register variable -> task_key mapping in context. - # When the value is a DAB ref (e.g. {{job.start_time.iso_datetime}} from - # @utcNow()), store it so downstream @variables() calls can inline it - # instead of routing through the task value. + # Register variable -> task_key mapping in context. When the value is a DAB ref (e.g. + # {{job.start_time.iso_datetime}} from @utcNow()), store it so downstream @variables() calls inline it. dab_ref_value = variable_value if value_kind == "dab_ref" else None new_context = context.with_variable( variable_name, diff --git a/src/orchestra/translator/activity_translators/spark_jar.py b/src/flowx/translator/activity_translators/spark_jar.py similarity index 100% rename from src/orchestra/translator/activity_translators/spark_jar.py rename to src/flowx/translator/activity_translators/spark_jar.py diff --git a/src/orchestra/translator/activity_translators/spark_python.py b/src/flowx/translator/activity_translators/spark_python.py similarity index 95% rename from src/orchestra/translator/activity_translators/spark_python.py rename to src/flowx/translator/activity_translators/spark_python.py index b0b230a..87c53de 100644 --- a/src/orchestra/translator/activity_translators/spark_python.py +++ b/src/flowx/translator/activity_translators/spark_python.py @@ -57,7 +57,7 @@ def translate( raw_parameters = type_properties.get("parameters") or [] libraries = type_properties.get("libraries") - parameters = [_resolve_parameter(p, context) for p in raw_parameters] + parameters = [_resolve_parameter(parameter, context) for parameter in raw_parameters] return SparkPythonActivity( **base_kwargs, diff --git a/src/orchestra/translator/activity_translators/switch.py b/src/flowx/translator/activity_translators/switch.py similarity index 100% rename from src/orchestra/translator/activity_translators/switch.py rename to src/flowx/translator/activity_translators/switch.py diff --git a/src/orchestra/translator/activity_translators/wait.py b/src/flowx/translator/activity_translators/wait.py similarity index 100% rename from src/orchestra/translator/activity_translators/wait.py rename to src/flowx/translator/activity_translators/wait.py diff --git a/src/flowx/translator/activity_translators/web_activity.py b/src/flowx/translator/activity_translators/web_activity.py new file mode 100644 index 0000000..c1da35b --- /dev/null +++ b/src/flowx/translator/activity_translators/web_activity.py @@ -0,0 +1,162 @@ +"""Translates ADF WebActivity activities to Databricks WebActivity IR.""" + +from __future__ import annotations + +import json +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, TranslationContext +from flowx.models.ir import WebActivity as WebActivityIR +from flowx.parser.expression_parser import ( + resolve_expression, + resolve_interpolated_string_for_notebook, +) +from flowx.translator.activity_translators.resolve import resolve_dict_values, resolve_field + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a WebActivity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + + Returns: + A :class:`WebActivity` IR node. + """ + type_properties = activity.type_properties or {} + + url = resolve_field(type_properties.get("url", ""), context) + method = type_properties.get("method", "GET") + headers = resolve_dict_values(type_properties.get("headers"), context) or None + body = type_properties.get("body") + body_code, body_imports, body_required = _resolve_body_to_code(body, context) + authentication = type_properties.get("authentication") + disable_cert_validation = type_properties.get("disableCertValidation", False) + http_request_timeout = type_properties.get("httpRequestTimeout") + + timeout_seconds: int | None = None + if http_request_timeout and isinstance(http_request_timeout, str): + timeout_seconds = _parse_timeout_to_seconds(http_request_timeout) + + return WebActivityIR( + **base_kwargs, + url=url, + method=method, + body=body, + headers=headers, + authentication=authentication, + disable_cert_validation=disable_cert_validation, + http_request_timeout_seconds=timeout_seconds, + body_code=body_code, + body_imports=body_imports, + body_required_parameters=body_required, + ) + + +def _py_literal(value: Any) -> str: + """Renders a resolved literal as a Python expression.""" + if isinstance(value, str): + return json.dumps(value) + if isinstance(value, bool) or value is None: + return repr(value) + return json.dumps(value) + + +def _value_to_code(value: Any, context: TranslationContext) -> tuple[str | None, list[str], dict[str, str]]: + """Lowers a single body value to a Python expression string. + + Returns ``(code, imports, required_parameters)`` where ``code`` is + ``None`` when the value is a plain literal the code generator can render + directly (no ``@``-expression present). + """ + if isinstance(value, str): + if "@{" in value: + resolved = resolve_interpolated_string_for_notebook(value, context) + return f"f{json.dumps(resolved)}", [], {} + if value.startswith("@"): + result = resolve_expression(value, context) + if result is None: + return None, [], {} + if result.kind == "notebook_code": + return result.value, list(result.imports), dict(result.required_parameters) + if result.kind == "dab_ref": + # A bare variable/pipeline ref is read from a widget at runtime; bind the DAB ref into + # base_parameters via the returned required_parameters mapping. + widget = result.value.strip("{}").split(".")[-1] + return f"dbutils.widgets.get({json.dumps(widget)})", [], {widget: result.value} + return _py_literal(result.value), [], dict(result.required_parameters) + return None, [], {} + if isinstance(value, dict): + if value.get("type") == "Expression" and "value" in value: + return _value_to_code(value["value"], context) + # Nested dict body (e.g. {"text": {"value": "@concat(...)"}}). + parts: list[str] = [] + imports: list[str] = [] + required: dict[str, str] = {} + any_code = False + for key, inner in value.items(): + code, imps, req = _value_to_code(inner, context) + if code is None: + parts.append(f"{json.dumps(key)}: {_py_literal(inner)}") + else: + any_code = True + parts.append(f"{json.dumps(key)}: {code}") + imports.extend(imps) + required.update(req) + if not any_code: + return None, [], {} + return "{" + ", ".join(parts) + "}", imports, required + return None, [], {} + + +def _resolve_body_to_code(body: Any, context: TranslationContext) -> tuple[str | None, list[str], dict[str, str]]: + """Pre-resolves an ADF request body to Python code at translate time. + + ADF web-activity bodies frequently embed ``@concat`` / ``@variables`` / + ``@{...}`` expressions, either at the top level or nested inside a dict + (``{"text": {"value": "@concat(...)"}}``). Resolving them here -- while + the real :class:`TranslationContext` (and its variable cache) is + available -- lets the code generator emit parsed Python instead of the + raw ADF token. + + Returns ``(body_code, imports, required_parameters)``. ``body_code`` is + ``None`` for a plain-literal body (the generator renders it directly). + """ + if body is None: + return None, [], {} + return _value_to_code(body, context) + + +def _parse_timeout_to_seconds(timeout_str: str) -> int | None: + """Parses an ADF timeout string to seconds. + + Args: + timeout_str: Timeout in ``"d.hh:mm:ss"`` or ``"hh:mm:ss"`` format. + + Returns: + Total seconds, or ``None`` if the format is unrecognised. + """ + try: + parts = timeout_str.split(".") + if len(parts) == 2: + days = int(parts[0]) + time_part = parts[1] + else: + days = 0 + time_part = parts[0] + time_parts = time_part.split(":") + hours = int(time_parts[0]) if len(time_parts) > 0 else 0 + minutes = int(time_parts[1]) if len(time_parts) > 1 else 0 + seconds = int(time_parts[2]) if len(time_parts) > 2 else 0 + return days * 86400 + hours * 3600 + minutes * 60 + seconds + except (ValueError, IndexError): + return None diff --git a/src/orchestra/translator/engine.py b/src/flowx/translator/engine.py similarity index 77% rename from src/orchestra/translator/engine.py rename to src/flowx/translator/engine.py index cb90617..b99c318 100644 --- a/src/orchestra/translator/engine.py +++ b/src/flowx/translator/engine.py @@ -126,12 +126,9 @@ def translate_pipeline( global_parameters=MappingProxyType(dict(definitions.global_parameters)), ) - # C-41 (CF5-001): seed declared variable types so the IfCondition - # fallback can recognise Boolean variables that are backed only by a - # literal default init task (and thus never populate - # variable_value_cache). Without this a `continue`-style Boolean - # condition emits NOT_EQUAL(left, '0'), always true for a - # 'true'/'false' string, making the false branch dead code. + # C-41 (CF5-001): seed declared variable types so the IfCondition fallback recognises Boolean + # variables backed only by a literal default (which never populate variable_value_cache); else the + # false branch is dead code. if pipeline.variables: default_literals: dict[str, str] = {} for name, var in pipeline.variables.items(): @@ -148,11 +145,8 @@ def translate_pipeline( gaps: list[AgenticGap] = [] warnings: list[str] = [] - # C-05 (VAREX-002): synthesise init SetVariable tasks for pipeline - # variables carrying a defaultValue. This seeds variable_cache so - # downstream @variables('X') references resolve to the init task's - # value reference instead of falling back to a self-referential - # {{tasks.X.values.X}} dangler. + # C-05 (VAREX-002): synthesise init SetVariable tasks for variables with a defaultValue, seeding + # variable_cache so @variables('X') resolves to the init task's value instead of a self-referential dangler. init_variable_activities, context = _build_variable_init_activities(pipeline, context) translated_activities: list[Activity] = list(init_variable_activities) @@ -170,32 +164,21 @@ def translate_pipeline( deterministic_count += 1 elif strategy is TranslationStrategy.AGENTIC: agentic_count += 1 - gaps.append( - AgenticGap( - activity_name=adf_activity.name, - activity_type=adf_activity.type, - recommended_skill=skill, - raw_definition=adf_activity.type_properties, - ) - ) else: unsupported_count += 1 - gaps.append( - AgenticGap( - activity_name=adf_activity.name, - activity_type=adf_activity.type, - recommended_skill=None, - raw_definition=adf_activity.type_properties, - ) - ) - warnings.append(f"Activity '{adf_activity.name}' (type={adf_activity.type}) has no translation path.") + + # Collect every agentic/unsupported activity across the whole tree (including IfCondition/ForEach/Until + # children) so each gap reaches the agent with its full ADF/ARM JSON. + gaps = _collect_agentic_gaps(pipeline.activities, warnings) parameter_entries: list[dict[str, Any]] = [] if pipeline.parameters: for param_name, param_def in pipeline.parameters.items(): entry: dict[str, Any] = {"name": param_name, "type": param_def.type} if param_def.default_value is not None: - entry["default"] = _coerce_parameter_default(param_def.default_value, param_def.type) + entry["default"] = _resolve_parameter_default( + param_def.default_value, param_def.type, context, warnings + ) parameter_entries.append(entry) schedule = _compile_pipeline_schedule(pipeline, definitions) @@ -208,17 +191,12 @@ def translate_pipeline( schedule=schedule, ) - # Whole-IR expression rewrite: catches @{...} tokens the per-activity - # translators didn't address (raw SQL WHERE clauses inside source_properties, - # REST request bodies, dataset folder paths, ...). Unresolved tokens are - # surfaced as translation warnings. + # Whole-IR expression rewrite: catches @{...} tokens the per-activity translators missed (raw SQL + # WHERE, REST bodies, dataset folder paths, ...). Unresolved tokens become translation warnings. pipeline_ir = rewrite_pipeline_expressions(pipeline_ir, warnings=warnings) - # Motif detection: scan for known multi-activity patterns. Collapsing is - # gated on the per-motif preference -- when *motif_consolidations* is - # ``None`` we preserve back-compat behaviour and collapse every detected - # motif; otherwise only motifs whose motif_id maps to ``"consolidate"`` are - # collapsed. + # Motif detection. Collapsing is gated on motif_consolidations: None preserves back-compat (collapse + # every detected motif), otherwise only motifs mapped to "consolidate" are collapsed. detected_motifs = detect_motifs(pipeline, definitions) motifs_to_collapse = _filter_motifs_for_collapse(detected_motifs, motif_consolidations) if motifs_to_collapse: @@ -249,6 +227,9 @@ def translate_pipeline( ) +_OPT_IN_ONLY_MOTIFS: frozenset[str] = frozenset({"activity_and_notify"}) + + def _filter_motifs_for_collapse( detected_motifs: list, motif_consolidations: dict[str, str] | None, @@ -266,9 +247,45 @@ def _filter_motifs_for_collapse( The subset of motifs to pass to :func:`flowx.motifs.collapser.collapse_motifs`. """ + # activity_and_notify is destructive (drops the notify activities for job-task notifications), so it's + # never collapsed implicitly -- only when the user opts into a notification destination. if motif_consolidations is None: - return list(detected_motifs) - return [m for m in detected_motifs if motif_consolidations.get(m.definition.motif_id) == "consolidate"] + return [motif for motif in detected_motifs if motif.definition.motif_id not in _OPT_IN_ONLY_MOTIFS] + return [motif for motif in detected_motifs if motif_consolidations.get(motif.definition.motif_id) == "consolidate"] + + +def _collect_agentic_gaps(activities: list[AdfActivity], warnings: list[str]) -> list[AgenticGap]: + """Walk the activity tree and emit a gap for every agentic / unsupported activity. + + Recurses into IfCondition branches and ForEach / Until container children so + that nested activities (e.g. an ``Until`` inside an ``IfCondition``) are not + lost. Each gap carries the activity's full ADF/ARM JSON (``raw``) so the + agentic handler can translate directly from source. + """ + gaps: list[AgenticGap] = [] + seen: set[str] = set() + + def _walk(acts: list[AdfActivity] | None) -> None: + for act in acts or []: + strategy, skill = classify_activity(act.type) + if strategy is not TranslationStrategy.DETERMINISTIC and act.name not in seen: + seen.add(act.name) + gaps.append( + AgenticGap( + activity_name=act.name, + activity_type=act.type, + recommended_skill=skill, + raw_definition=act.raw if act.raw is not None else act.type_properties, + ) + ) + if strategy is TranslationStrategy.UNSUPPORTED: + warnings.append(f"Activity '{act.name}' (type={act.type}) has no translation path.") + _walk(act.if_true_activities) + _walk(act.if_false_activities) + _walk(act.activities) + + _walk(activities) + return gaps def _dispatch_activity( @@ -355,6 +372,8 @@ def _dispatch_activity( **base_kwargs, original_type=activity.type, comment=reason, + agentic_skill=skill, + raw_definition=activity.raw, ) context = context.with_activity(activity.name, placeholder) return placeholder, context @@ -385,10 +404,8 @@ def _translate_activity_list( return results, context -# C-10 (SCHED-001): map Windows timezone names ADF emits onto IANA names -# the Databricks DAB ``schedule.timezone_id`` field expects. Only the -# ones observed in the corpus are mapped explicitly; anything else passes -# through unchanged (Databricks accepts any IANA zone). +# C-10 (SCHED-001): map the Windows timezone names ADF emits onto the IANA names DAB's +# schedule.timezone_id expects. Only corpus-observed ones are mapped; anything else passes through. _ADF_TIMEZONE_TO_IANA: dict[str, str] = { "UTC": "UTC", "Coordinated Universal Time": "UTC", @@ -433,20 +450,17 @@ def _compile_pipeline_schedule( """ triggers = getattr(definitions, "triggers", None) or [] pipeline_name = pipeline.name - matching_triggers = [t for t in triggers if _trigger_references(t, pipeline_name)] + matching_triggers = [trigger for trigger in triggers if _trigger_references(trigger, pipeline_name)] if not matching_triggers: return None - # First matching trigger wins -- ADF allows multiple triggers per - # pipeline but DAB schedules are 1:1. Subsequent triggers can be - # surfaced via SETUP.md by downstream tooling. + # First matching trigger wins -- ADF allows multiple triggers per pipeline but DAB schedules are 1:1; + # downstream tooling can surface the rest via SETUP.md. trigger = matching_triggers[0] spec = _adf_trigger_to_schedule(trigger) if spec is not None: - # SCHED3-003: pull per-pipeline parameter overrides off the - # matching pipelineReference so trigger-injected params (e.g. - # ``{applicationName: 'app0001', negocio: 'GLP'}``) propagate to - # the job's default parameter values. + # SCHED3-003: pull per-pipeline parameter overrides off the matching pipelineReference so + # trigger-injected params propagate to the job's default parameter values. overrides = _extract_trigger_parameter_overrides(trigger, pipeline_name) if overrides: spec["parameter_overrides"] = overrides @@ -499,9 +513,8 @@ def _adf_trigger_to_schedule(trigger: Any) -> dict[str, Any] | None: trigger_type = trigger.type if trigger_type == "ScheduleTrigger": recurrence = type_properties.get("recurrence") or {} - # SCHED3-002: Day/Week/Month with interval > 1 cannot be represented - # in quartz cron without enumerating every Nth occurrence; use the - # trigger.periodic primitive so it ships correctly. + # SCHED3-002: Day/Week/Month with interval > 1 can't be expressed in quartz cron without + # enumerating every Nth occurrence; use the trigger.periodic primitive instead. periodic = _recurrence_to_periodic(recurrence) if periodic is not None: spec: dict[str, Any] = { @@ -515,10 +528,8 @@ def _adf_trigger_to_schedule(trigger: Any) -> dict[str, Any] | None: if "time_of_day_note" in periodic: spec["time_of_day_note"] = periodic["time_of_day_note"] return spec - # C-45 (SCHED5-002): an interval > 1 Month recurrence has no - # monthly-cron-expressible form (cron fires every month, ignoring the - # interval) and the DAB periodic enum has no MONTHS unit, so surface a - # manual setup note instead of silently emitting a monthly cron. + # C-45 (SCHED5-002): interval > 1 Month has no monthly-cron form (cron ignores the interval) and + # the DAB periodic enum has no MONTHS unit, so surface a manual setup note instead of a wrong cron. if _is_multi_month_recurrence(recurrence): return { "kind": "manual_setup", @@ -598,12 +609,8 @@ def _recurrence_to_periodic(recurrence: dict[str, Any]) -> dict[str, Any] | None interval = int(interval) if not isinstance(interval, int) or interval <= 1: return None - # C-45 (SCHED5-002): the DAB PeriodicTriggerConfigurationTimeUnit enum - # only defines DAYS / HOURS / WEEKS — emitting MONTHS makes bundle - # validate/deploy reject the trigger. Month frequencies are routed to - # the quartz cron path (monthDays) instead; an interval > 1 Month, which - # is not monthly-cron-expressible, is surfaced as a setup note by the - # caller. + # C-45 (SCHED5-002): the DAB periodic enum only has DAYS/HOURS/WEEKS, so month frequencies route to the + # quartz cron path (monthDays); an interval > 1 Month (not monthly-cron-expressible) is a setup note. unit_map = {"Day": "DAYS", "Week": "WEEKS"} unit = unit_map.get(frequency or "") if unit is None: @@ -649,12 +656,8 @@ def _recurrence_to_quartz_cron(recurrence: dict[str, Any]) -> str | None: week_days = schedule.get("weekDays") or [] month_days = schedule.get("monthDays") or [] - # C-44 (SCHED5-001): when the schedule block carries no explicit - # time-of-day, ADF defaults it to the first-execution time derived from - # ``startTime``. Reading only ``schedule.minutes/hours`` (falling back - # to '0'/'0') silently shifts a ``startTime`` of 21:00 to midnight. - # Derive the hour/minute from ``startTime`` so the cron fires at the - # ADF-intended time. + # C-44 (SCHED5-001): when the schedule has no explicit time-of-day, derive hour/minute from startTime + # (ADF's default); reading only schedule.minutes/hours would shift a 21:00 startTime to midnight. start_hour, start_minute = _start_time_hour_minute(recurrence.get("startTime")) minute_default = str(start_minute) if start_minute is not None else "0" hour_default = str(start_hour) if start_hour is not None else "0" @@ -675,7 +678,7 @@ def _recurrence_to_quartz_cron(recurrence: dict[str, Any]) -> str | None: if frequency == "Day": return f"0 {minute_field} {hour_field} * * ?" if frequency == "Week": - days = ",".join(_DAYS_OF_WEEK_MAP.get(d, d) for d in week_days) or "MON" + days = ",".join(_DAYS_OF_WEEK_MAP.get(day, day) for day in week_days) or "MON" return f"0 {minute_field} {hour_field} ? * {days}" if frequency == "Month": dom_field = _list_or_default(month_days, "1") @@ -690,7 +693,7 @@ def _list_or_default(value: Any, default: str) -> str: if isinstance(value, list): if not value: return default - return ",".join(str(v) for v in value) + return ",".join(str(item) for item in value) return str(value) @@ -760,10 +763,8 @@ def _build_variable_init_activities( notebook_imports = list(expr_result.imports) if expr_result.kind == "notebook_code" else [] required_parameters = dict(expr_result.required_parameters) else: - # VAREX3-002: Boolean defaults must render lowercase ('true'/'false') - # so downstream ``@equals(variables('continue'), true)`` evaluates - # consistently with ADF semantics. Python ``str(True)`` would - # produce title-case 'True' and silently invert the comparison. + # VAREX3-002: Boolean defaults render lowercase 'true'/'false' so @equals(variables('continue'), + # true) matches ADF; Python str(True) would emit 'True' and silently invert the comparison. if isinstance(default, bool): variable_value = "true" if default else "false" else: @@ -790,9 +791,8 @@ def _build_variable_init_activities( required_parameters=required_parameters, ) init_tasks.append(init_activity) - # Register the synthesised setter so @variables('X') resolves to - # {{tasks._init_X.values.X}}. When the value is itself a DAB ref - # (e.g. from @utcNow()), inline it directly per existing semantics. + # Register the synthesised setter so @variables('X') resolves to {{tasks._init_X.values.X}}; when + # the value is itself a DAB ref (e.g. @utcNow()), inline it directly. dab_ref_value = variable_value if value_kind == "dab_ref" else None context = context.with_variable(var_name, task_key, dab_ref_value=dab_ref_value) context = context.with_activity(init_activity.name, init_activity) @@ -961,7 +961,7 @@ def _map_dependency_conditions(conditions: list[str] | None) -> str | None: """ if not conditions: return None - normalized = [c for c in conditions if c] + normalized = [condition for condition in conditions if condition] if not normalized: return None if len(normalized) == 1: @@ -1028,14 +1028,12 @@ def _resolve_ls_parameters( if isinstance(activity_supplied, dict): for pname, pval in activity_supplied.items(): raw = _unwrap_expression_value(pval) - # C-03: route @-prefixed activity-supplied values through the - # expression parser so @pipeline().globalParameters.X collapses - # to the factory value when one is set. + # C-03: route @-prefixed activity-supplied values through the expression parser so + # @pipeline().globalParameters.X collapses to the factory value when one is set. if context is not None and isinstance(raw, str) and raw.startswith("@"): result = resolve_expression(raw, context) - # C-13 (NB-ITER3-002 / LSC3-003 / VAREX3-006): accept both - # literal and dab_ref so @pipeline().parameters.X collapses - # to {{job.parameters.X}} (valid in custom_tags map values). + # C-13 (NB-ITER3-002/LSC3-003/VAREX3-006): accept both literal and dab_ref so + # @pipeline().parameters.X collapses to {{job.parameters.X}} (valid in custom_tags values). if result is not None and result.kind in ("literal", "dab_ref"): raw = result.value resolved[pname] = raw @@ -1055,15 +1053,13 @@ def _unwrap_expression_value(value: Any) -> Any: # Bare {"value": X, "type": "Expression"} -- collapse to inner X. if "value" in value and value.get("type") == "Expression": return _unwrap_expression_value(value["value"]) - # Some payloads omit the explicit type marker but follow the same - # single-key shape. Conservatively unwrap only when the dict has - # the exact two keys {"value", "type"} so we don't corrupt regular - # nested config blocks like {"workspace": {"destination": ...}}. + # Some payloads omit the type marker but share the shape; conservatively unwrap only when the dict + # has exactly {value, type} so we don't corrupt nested config like {"workspace": {"destination": ...}}. if set(value.keys()) == {"value", "type"}: return _unwrap_expression_value(value["value"]) return {k: _unwrap_expression_value(v) for k, v in value.items()} if isinstance(value, list): - return [_unwrap_expression_value(v) for v in value] + return [_unwrap_expression_value(item) for item in value] return value @@ -1098,7 +1094,7 @@ def _sub(match: re.Match[str]) -> str: if isinstance(value, dict): return {k: _substitute_ls_params(v, params) for k, v in value.items()} if isinstance(value, list): - return [_substitute_ls_params(v, params) for v in value] + return [_substitute_ls_params(item, params) for item in value] return value @@ -1114,6 +1110,45 @@ def _coerce_int(value: Any) -> Any: return value +def _resolve_parameter_default( + value: Any, + declared_type: str, + context: TranslationContext, + warnings: list[str], +) -> Any: + """Resolve a pipeline parameter default for emission as a job parameter. + + ADF parameter defaults may themselves be ``@``-expressions -- most + commonly ``@utcNow('yyyy-MM-dd')``. Those must be lowered to a DAB + dynamic value reference (e.g. ``{{job.start_time.iso_date}}``) so the + generated job-parameter default is valid Databricks YAML rather than a + raw ADF token. Non-expression defaults fall through to type coercion. + + Args: + value: The raw ADF default value. + declared_type: The ADF parameter type (``String`` / ``Int`` / ...). + context: Current translation context (for variable/expression refs). + warnings: Mutable warning list; an entry is appended when an + ``@``-expression default cannot be lowered deterministically. + + Returns: + The resolved default -- a DAB ref / literal for ``@``-expressions, + otherwise the type-coerced value. + """ + if isinstance(value, str) and value.strip().startswith("@"): + from flowx.parser.expression_parser import resolve_expression + + result = resolve_expression(value, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + warnings.append( + f"Pipeline parameter default '{value}' could not be lowered to a DAB value " + "reference; emitted as-is. Set it explicitly at deploy time if needed." + ) + return value + return _coerce_parameter_default(value, declared_type) + + def _coerce_parameter_default(value: Any, declared_type: str) -> Any: """Coerce an ADF parameter default into a Python type matching its declared type. @@ -1166,10 +1201,8 @@ def _extract_cluster_config( if overrides: ls_properties = _substitute_ls_params(ls_properties, overrides) - # C-02 (NB-ITER2-2 / LSC2-003): unwrap any {value, type:'Expression'} - # dicts that survived the substitution pass. Map fields like - # custom_tags and spark_env_vars must be plain Map[String, String] for - # Databricks to accept the cluster YAML. + # C-02 (NB-ITER2-2/LSC2-003): unwrap any {value, type:'Expression'} dicts that survived substitution - + # map fields like custom_tags / spark_env_vars must be plain Map[String, String] for the cluster YAML. ls_properties = _unwrap_expression_value(ls_properties) nested = ls_properties.get("typeProperties") or {} @@ -1212,10 +1245,8 @@ def _extract_cluster_config( if cluster_log_conf: config["cluster_log_conf"] = cluster_log_conf - # C-39 (LSC4-004): capture the ADF authentication shape (e.g. "MSI" or - # any CredentialReference) so the bundler can emit a manual_credential - # SetupTask warning that ``single_user_name`` was rewritten to the - # deploying user. + # C-39 (LSC4-004): capture the ADF auth shape (e.g. MSI / a CredentialReference) so the bundler emits + # a manual_credential SetupTask noting single_user_name was rewritten to the deploying user. authentication = fields.get("authentication") if authentication: config["_adf_authentication"] = authentication @@ -1242,30 +1273,30 @@ def _pipeline_to_dict(pipeline: Pipeline) -> dict[str, Any]: "tags": pipeline.tags, "tasks": [_activity_to_dict(task) for task in pipeline.tasks], } - if pipeline.translation_preferences is not None: - result["translation_preferences"] = _preferences_to_dict(pipeline.translation_preferences) + if pipeline.translation_configuration is not None: + result["translation_configuration"] = _configuration_to_dict(pipeline.translation_configuration) return result -def _preferences_to_dict(preferences: Any) -> dict[str, Any]: - """Serialise a TranslationPreferences instance to a JSON-friendly dictionary. +def _configuration_to_dict(configuration: Any) -> dict[str, Any]: + """Serialise a TranslationConfiguration instance to a JSON-friendly dictionary. Args: - preferences: The :class:`TranslationPreferences` snapshot to serialise. + configuration: The :class:`TranslationConfiguration` snapshot to serialise. Returns: Dictionary with each StrEnum field rendered as its string value and per-task overrides preserved verbatim. """ return { - "copy_activity_paradigm": str(preferences.copy_activity_paradigm), - "non_databricks_task_compute": str(preferences.non_databricks_task_compute), - "use_lakeflow_connectors": str(preferences.use_lakeflow_connectors), - "lakeflow_connector_type": str(preferences.lakeflow_connector_type), + "copy_activity_paradigm": str(configuration.copy_activity_paradigm), + "non_databricks_task_compute": str(configuration.non_databricks_task_compute), + "use_lakeflow_connectors": str(configuration.use_lakeflow_connectors), + "lakeflow_connector_type": str(configuration.lakeflow_connector_type), "motif_consolidations": { - motif_id: str(choice) for motif_id, choice in preferences.motif_consolidations.items() + motif_id: str(choice) for motif_id, choice in configuration.motif_consolidations.items() }, - "per_task": dict(preferences.per_task), + "per_task": dict(configuration.per_task), } @@ -1301,6 +1332,8 @@ def _activity_to_dict(task: Activity) -> dict[str, Any]: task_dict["existing_cluster_id"] = task.existing_cluster_id if task.compute_mode: task_dict["compute_mode"] = task.compute_mode + if task.notifications: + task_dict["notifications"] = task.notifications if task.libraries: task_dict["libraries"] = task.libraries if task.parameter_approximations: @@ -1444,6 +1477,16 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["headers"] = activity.headers if activity.authentication: extra["authentication"] = activity.authentication + if activity.body_code is not None: + extra["body_code"] = activity.body_code + if activity.body_imports: + extra["body_imports"] = activity.body_imports + if activity.body_required_parameters: + extra["body_required_parameters"] = activity.body_required_parameters + if activity.disable_cert_validation: + extra["disable_cert_validation"] = activity.disable_cert_validation + if activity.http_request_timeout_seconds: + extra["http_request_timeout_seconds"] = activity.http_request_timeout_seconds case DeleteActivity(): extra["dataset_name"] = activity.dataset_name if activity.folder_path: @@ -1556,14 +1599,104 @@ def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: } -if __name__ == "__main__": +def _find_and_replace_task(tasks: list[dict[str, Any]], activity_name: str, replacement: dict[str, Any]) -> bool: + """Replace the task named *activity_name* with *replacement*, recursing into containers. + + Searches top-level tasks and the nested activity lists of IfCondition / + ForEach / Switch containers. Preserves the placeholder's ``task_key`` and + ``depends_on`` when the replacement omits them so downstream dependency + edges stay intact. Returns True when a match was replaced. + """ + nested_keys = ("inner_activities", "if_true_activities", "if_false_activities", "default_activities") + for index, task in enumerate(tasks): + if task.get("name") == activity_name: + replacement.setdefault("task_key", task.get("task_key")) + replacement.setdefault("name", activity_name) + if "depends_on" not in replacement and task.get("depends_on"): + replacement["depends_on"] = task["depends_on"] + tasks[index] = replacement + return True + for key in nested_keys: + child = task.get(key) + if isinstance(child, list) and _find_and_replace_task(child, activity_name, replacement): + return True + for case in task.get("cases") or []: + if isinstance(case, dict) and isinstance(case.get("activities"), list): + if _find_and_replace_task(case["activities"], activity_name, replacement): + return True + return False + + +def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Path | None = None) -> tuple[int, int]: + """Merge agent-produced per-activity translations into a translation report. + + Each ``*.json`` file in *results_dir* describes one resolved agentic gap:: + + { + "activity_name": "", # required + "pipeline": "", # optional; for multi-pipeline reports + "task": { ...IR task dict... } # required; replacement task + } + + The matching placeholder task (located by ``name``, recursing into + IfCondition / ForEach / Switch containers) is replaced by ``task``. Use a + ``NotebookActivity`` whose ``notebook_path`` points at a notebook the agent + wrote to the workspace; the prepare phase then references it directly. + + Args: + report_path: ``translation_report.json`` produced by the translate phase. + results_dir: Directory of per-activity result JSON files. + output_path: Where to write the merged report; defaults to overwriting + *report_path*. + + Returns: + ``(merged, unmatched)`` counts. + """ + report = json.loads(report_path.read_text(encoding="utf-8")) + pipelines = report["pipelines"] if isinstance(report, dict) and "pipelines" in report else [report] + + merged = 0 + unmatched = 0 + for result_file in sorted(results_dir.glob("*.json")): + data = json.loads(result_file.read_text(encoding="utf-8")) + activity_name = data.get("activity_name") or data.get("activity") + task = data.get("task") or data.get("ir") + if not activity_name or not isinstance(task, dict): + logger.warning("Skipping %s: missing 'activity_name' or 'task'.", result_file.name) + unmatched += 1 + continue + wanted = data.get("pipeline") + candidates = [pipeline for pipeline in pipelines if not wanted or pipeline.get("name") == wanted] + if any(_find_and_replace_task(pipeline.get("tasks", []), activity_name, dict(task)) for pipeline in candidates): + merged += 1 + logger.info("Merged agentic result for '%s' from %s", activity_name, result_file.name) + else: + logger.warning("No placeholder named '%s' found for %s", activity_name, result_file.name) + unmatched += 1 + + destination = output_path or report_path + destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") + logger.info("Wrote merged report to %s (%d merged, %d unmatched)", destination, merged, unmatched) + return merged, unmatched + + +def main(argv: list[str] | None = None) -> int: + """Convert-phase entry point: translate ADF pipelines to IR (or merge agentic results). + + Exposed as a callable so the adapter can run the phase in-process instead of spawning a + second interpreter. + """ parser = argparse.ArgumentParser(description="Translate ADF pipelines to Databricks IR.") - parser.add_argument("--source-dir", required=True, type=Path, help="Root directory containing ADF JSON exports.") + parser.add_argument("--source-dir", required=False, type=Path, help="Root directory containing ADF JSON exports.") parser.add_argument( "--output-dir", type=Path, - default=Path("./orchestra_output/translate"), - help="Directory to write translation results into.", + default=Path("./flowx_output"), + help=( + "Migration output directory. The translation report and other " + "intermediate IR are written to its transient .work/ subfolder " + "(consumed by the adapter/prepare phases; pruned by prepare)." + ), ) parser.add_argument( "--pipeline", @@ -1576,20 +1709,60 @@ def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: action="store_true", help="Write a full debug IR dump alongside the normal output.", ) - args = parser.parse_args() + parser.add_argument( + "--merge-agentic", + action="store_true", + help="Merge agent-produced results from --agentic-results into --report instead of translating.", + ) + parser.add_argument( + "--report", + type=Path, + default=None, + help="Translation report to merge agentic results into (with --merge-agentic).", + ) + parser.add_argument( + "--agentic-results", + type=Path, + default=None, + help="Directory of per-activity agentic result JSON files (with --merge-agentic).", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Where to write the merged report (default: overwrite --report).", + ) + args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + if args.merge_agentic: + if not args.report or not args.agentic_results: + parser.error("--merge-agentic requires --report and --agentic-results") + merged_count, unmatched_count = merge_agentic_results(args.report, args.agentic_results, args.output) + print("\nAgentic Merge Summary") + print("=====================") + print(f"Merged: {merged_count}") + print(f"Unmatched: {unmatched_count}") + return 0 if unmatched_count == 0 else 1 + + if not args.source_dir: + parser.error("--source-dir is required (unless using --merge-agentic)") + definitions = load_adf_definitions(args.source_dir) logger.info("Loaded %d pipeline(s) from %s", len(definitions.pipelines), args.source_dir) output_dir: Path = args.output_dir.resolve() - output_dir.mkdir(parents=True, exist_ok=True) + # Translation IR is intermediate: write it to a transient .work/ subfolder; the prepare phase consumes + # the report from there and prunes .work/, leaving metadata/ curated. + work_dir = output_dir / ".work" + work_dir.mkdir(parents=True, exist_ok=True) total_deterministic = 0 total_agentic = 0 total_unsupported = 0 all_gaps: list[dict[str, Any]] = [] + all_pipeline_dicts: list[dict[str, Any]] = [] for pipeline in definitions.pipelines: if args.pipeline and pipeline.name != args.pipeline: @@ -1600,14 +1773,15 @@ def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: total_agentic += report.agentic_count total_unsupported += report.unsupported_count - pipeline_file = output_dir / f"{_sanitize_task_key(pipeline.name)}.json" + pipeline_file = work_dir / f"{_sanitize_task_key(pipeline.name)}.json" pipeline_dict = _pipeline_to_dict(report.pipeline) pipeline_file.write_text(json.dumps(pipeline_dict, indent=2, default=str), encoding="utf-8") logger.info("Wrote pipeline IR to %s", pipeline_file) + all_pipeline_dicts.append(pipeline_dict) # Write debug IR if requested if args.debug: - debug_file = output_dir / f"{_sanitize_task_key(pipeline.name)}.debug.json" + debug_file = work_dir / f"{_sanitize_task_key(pipeline.name)}.debug.json" debug_dict = _pipeline_to_debug_dict(report.pipeline) debug_file.write_text(json.dumps(debug_dict, indent=2, default=str), encoding="utf-8") logger.info("Wrote debug IR to %s", debug_file) @@ -1619,8 +1793,18 @@ def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: for warning in report.warnings: logger.warning(warning) + # Write canonical translation_report.json so downstream tools (inspect, workspace-paths, dab_writer) + # can reference a well-known filename regardless of --pipeline. + report_file = work_dir / "translation_report.json" + if len(all_pipeline_dicts) == 1: + report_payload = all_pipeline_dicts[0] + else: + report_payload = {"pipelines": all_pipeline_dicts} + report_file.write_text(json.dumps(report_payload, indent=2, default=str), encoding="utf-8") + logger.info("Wrote translation_report.json to %s", report_file) + if all_gaps: - gaps_file = output_dir / "gaps.json" + gaps_file = work_dir / "gaps.json" gaps_file.write_text(json.dumps(all_gaps, indent=2, default=str), encoding="utf-8") logger.info("Wrote %d gap(s) to %s", len(all_gaps), gaps_file) @@ -1631,3 +1815,9 @@ def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: print(f"Agentic: {total_agentic}") print(f"Unsupported: {total_unsupported}") print(f"Total: {total}") + print(f"\nTranslation report (intermediate): {report_file}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/orchestra/translator/query_analysis.py b/src/flowx/translator/query_analysis.py similarity index 100% rename from src/orchestra/translator/query_analysis.py rename to src/flowx/translator/query_analysis.py diff --git a/src/orchestra/utils.py b/src/flowx/utils.py similarity index 94% rename from src/orchestra/utils.py rename to src/flowx/utils.py index fbbd882..7fa6c61 100644 --- a/src/orchestra/utils.py +++ b/src/flowx/utils.py @@ -7,9 +7,7 @@ from flowx.models.adf_ast import AdfPolicy -# --------------------------------------------------------------------------- -# Default ADF timeout (12 hours) used when a timeout string cannot be parsed. -# --------------------------------------------------------------------------- +# Default ADF timeout (12 hours), used when a timeout string cannot be parsed. DEFAULT_TIMEOUT_SECONDS = 43_200 # --------------------------------------------------------------------------- diff --git a/src/flowx/validate/__init__.py b/src/flowx/validate/__init__.py new file mode 100644 index 0000000..a6544ce --- /dev/null +++ b/src/flowx/validate/__init__.py @@ -0,0 +1,27 @@ +"""Tier-0 static validation: motif-aware DAG equivalence between ADF and IR.""" + +from __future__ import annotations + +from flowx.validate.bundle_invariants import ( + BundleFinding, + BundleInvariantResult, + check_bundle_dir, + check_job, +) +from flowx.validate.dag_equivalence import ( + DagEquivalenceResult, + DagFinding, + check_dag_equivalence, + format_result, +) + +__all__ = [ + "DagEquivalenceResult", + "DagFinding", + "check_dag_equivalence", + "format_result", + "BundleFinding", + "BundleInvariantResult", + "check_bundle_dir", + "check_job", +] diff --git a/src/flowx/validate/bundle_invariants.py b/src/flowx/validate/bundle_invariants.py new file mode 100644 index 0000000..c822802 --- /dev/null +++ b/src/flowx/validate/bundle_invariants.py @@ -0,0 +1,181 @@ +"""Structural-invariant checks for a generated Databricks Asset Bundle. + +These guard against output that is valid YAML / valid Python but invalid as a +Databricks job -- e.g. a job parameter declared twice (the duplicate-``region`` +regression), a duplicate task key, a ``{{job.parameters.X}}`` reference to an +undeclared parameter, a ``depends_on`` edge to a missing task, or a leaked YAML +anchor/alias (the fingerprint of a shared mutable object reaching serialization). + +Run :func:`check_bundle_dir` over a generated bundle in tests (and optionally as +a Tier-0 prepare step) so these never ship silently. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +# PyYAML emits anchors/aliases as ``&id001`` / ``*id001`` when the same object +# appears more than once in the tree. flowx never intends to emit these. +_ANCHOR_RE = re.compile(r"[&*]id\d+\b") +_JOB_PARAM_REF_RE = re.compile(r"\{\{\s*job\.parameters\.([A-Za-z0-9_]+)\s*\}\}") + + +@dataclass(slots=True, kw_only=True) +class BundleFinding: + """A single invariant violation. + + Attributes: + code: Stable machine-readable identifier. + message: Human-readable explanation. + location: File / job / task the finding concerns. + """ + + code: str + message: str + location: str = "" + severity: str = "violation" + + +@dataclass(slots=True, kw_only=True) +class BundleInvariantResult: + """Outcome of :func:`check_bundle_dir` / :func:`check_job`.""" + + findings: list[BundleFinding] = field(default_factory=list) + + @property + def violations(self) -> list[BundleFinding]: + """Hard, always-invalid findings (these fail a bundle).""" + return [finding for finding in self.findings if finding.severity == "violation"] + + @property + def warnings(self) -> list[BundleFinding]: + """Soft findings worth surfacing but not build-failing.""" + return [finding for finding in self.findings if finding.severity == "warning"] + + @property + def ok(self) -> bool: + """True when no hard invariant was violated (warnings are allowed).""" + return not self.violations + + +def _collect_task_keys(tasks: list[dict[str, Any]]) -> list[str]: + """Top-level task keys plus any single nested ``for_each_task.task`` key.""" + keys: list[str] = [] + for task in tasks or []: + if "task_key" in task: + keys.append(task["task_key"]) + nested = (task.get("for_each_task") or {}).get("task") + if isinstance(nested, dict) and "task_key" in nested: + keys.append(nested["task_key"]) + return keys + + +def _dump(obj: Any) -> str: + """Serialise a structure to a string for reference scanning.""" + return yaml.safe_dump(obj, default_flow_style=False) + + +def check_job(job_key: str, job: dict[str, Any]) -> list[BundleFinding]: + """Check the structural invariants of a single job resource dict.""" + findings: list[BundleFinding] = [] + where = f"job '{job_key}'" + + # 1. No duplicate job-parameter names. + param_names = [param.get("name") for param in (job.get("parameters") or []) if isinstance(param, dict)] + duplicate_params = sorted({name for name in param_names if name is not None and param_names.count(name) > 1}) + for name in duplicate_params: + findings.append( + BundleFinding( + code="duplicate_job_parameter", + location=where, + message=f"Job parameter '{name}' is declared more than once.", + ) + ) + + # 2. No duplicate task keys. + task_keys = _collect_task_keys(job.get("tasks") or []) + duplicate_keys = sorted({key for key in task_keys if task_keys.count(key) > 1}) + for key in duplicate_keys: + findings.append( + BundleFinding( + code="duplicate_task_key", location=where, message=f"Task key '{key}' is used more than once." + ) + ) + + # 3. Every {{job.parameters.X}} reference is declared. + declared = {name for name in param_names if name is not None} + referenced = set(_JOB_PARAM_REF_RE.findall(_dump(job))) + for name in sorted(referenced - declared): + findings.append( + BundleFinding( + code="undeclared_job_parameter", + severity="warning", + location=where, + message=f"'{{{{job.parameters.{name}}}}}' is referenced but '{name}' is not a declared job parameter.", + ) + ) + + # 4. Every top-level depends_on target exists. + top_level_keys = {task.get("task_key") for task in (job.get("tasks") or []) if isinstance(task, dict)} + for task in job.get("tasks") or []: + for dep in task.get("depends_on") or []: + target = dep.get("task_key") + if target and target not in top_level_keys: + findings.append( + BundleFinding( + code="dangling_depends_on", + location=f"{where}, task '{task.get('task_key')}'", + message=f"depends_on references unknown task '{target}'.", + ) + ) + return findings + + +def check_resource_text(text: str, *, filename: str = "") -> list[BundleFinding]: + """Check one resource YAML document (raw text): anchors + per-job invariants.""" + findings: list[BundleFinding] = [] + if _ANCHOR_RE.search(text): + findings.append( + BundleFinding( + code="yaml_anchor", + location=filename, + message=( + "Emitted YAML contains an anchor/alias (&idN/*idN); a shared mutable object " + "leaked into the bundle structure. This usually means a value was added twice." + ), + ) + ) + doc = yaml.safe_load(text) or {} + jobs = ((doc.get("resources") or {}).get("jobs") or {}) if isinstance(doc, dict) else {} + for job_key, job in jobs.items(): + if isinstance(job, dict): + findings.extend(check_job(job_key, job)) + return findings + + +def check_bundle_dir(bundle_dir: Path) -> BundleInvariantResult: + """Run all structural invariants over every resource YAML in a bundle directory.""" + bundle_dir = Path(bundle_dir) + findings: list[BundleFinding] = [] + resources_dir = bundle_dir / "resources" + yaml_files = sorted(resources_dir.glob("*.yml")) if resources_dir.exists() else [] + databricks_yml = bundle_dir / "databricks.yml" + if databricks_yml.exists(): + yaml_files.append(databricks_yml) + for path in yaml_files: + findings.extend(check_resource_text(path.read_text(encoding="utf-8"), filename=path.name)) + return BundleInvariantResult(findings=findings) + + +def format_result(result: BundleInvariantResult) -> str: + """Render a result as a compact human-readable report.""" + if result.ok: + return "Bundle invariants: OK" + lines = ["Bundle invariants: FAILED"] + lines.extend(f" - [{finding.code}] {finding.location}: {finding.message}" for finding in result.findings) + return "\n".join(lines) diff --git a/src/flowx/validate/dag_equivalence.py b/src/flowx/validate/dag_equivalence.py new file mode 100644 index 0000000..29b41c4 --- /dev/null +++ b/src/flowx/validate/dag_equivalence.py @@ -0,0 +1,408 @@ +"""Motif-aware DAG equivalence check (Tier-0 static validation). + +flowx translates an Azure Data Factory pipeline into a Databricks IR +pipeline. The two top-level dependency DAGs are *not* expected to be +identical, because motif collapsing rewrites the graph: each detected motif +contracts its matched activity set ``S`` into a single +:class:`~flowx.models.ir.MotifActivity`, dropping the edges internal to +``S`` and rewiring every cross-boundary edge onto the collapsed node. + +This module checks that the IR DAG equals the *quotient* of the ADF DAG under +the motif partition (so motif-induced differences are tolerated), and that +each contraction is **safe** -- i.e. it did not silently reorder anything. + +The safety condition is graph-theoretic convexity: contracting a set ``S`` to +a point preserves every ordering constraint **iff** no activity outside ``S`` +lies on a dependency path *between* two members of ``S``. If such an external +activity exists, the collapse would force it to run both before and after the +motif (a reordering / cycle); that is the invalidation we flag. + +Findings are graded: + +* ``violation`` -- the migrated DAG is not a faithful quotient of the source + (a cross-boundary ordering edge was dropped, an activity vanished, a motif + set was non-convex, or the quotient is cyclic). These block equivalence. +* ``warning`` -- a difference that is over-constraining or lossy but not + unsafe (an extra ordering edge, a merged/changed dependency outcome, an + unexplained IR-only task). +* ``tolerated`` -- a difference fully explained by motif contraction or by a + synthesised IR-only helper task; recorded only for transparency. +""" + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field + +from flowx.models.adf_ast import AdfPipeline +from flowx.models.ir import MotifActivity, Pipeline +from flowx.utils import normalize_task_key + +_DEFAULT_OUTCOME = "Succeeded" +# Task-key prefixes of synthesised variable-initialiser tasks the translator injects (engine +# ``_init_``); these have no ADF preimage and are expected to be IR-only. +_SYNTHESISED_KEY_PREFIXES = ("_init_",) + + +@dataclass(slots=True, kw_only=True) +class DagFinding: + """A single observation from the equivalence check. + + Attributes: + code: Stable machine-readable identifier (e.g. ``"missing_edge"``). + severity: One of ``"violation"`` / ``"warning"`` / ``"tolerated"``. + message: Human-readable explanation. + nodes: Block labels / activity names the finding concerns. + """ + + code: str + severity: str + message: str + nodes: tuple[str, ...] = () + + +@dataclass(slots=True, kw_only=True) +class DagEquivalenceResult: + """Outcome of :func:`check_dag_equivalence`. + + Attributes: + equivalent: True when there are no ``violation`` findings. + findings: All findings, in detection order. + """ + + equivalent: bool + findings: list[DagFinding] = field(default_factory=list) + + @property + def violations(self) -> list[DagFinding]: + """Findings that block equivalence.""" + return [finding for finding in self.findings if finding.severity == "violation"] + + @property + def warnings(self) -> list[DagFinding]: + """Non-blocking findings worth surfacing to the user.""" + return [finding for finding in self.findings if finding.severity == "warning"] + + @property + def tolerated(self) -> list[DagFinding]: + """Differences explained by motif collapse or synthesised tasks.""" + return [finding for finding in self.findings if finding.severity == "tolerated"] + + +# --------------------------------------------------------------------------- +# Graph helpers +# --------------------------------------------------------------------------- + + +def _reachable(adjacency: dict[str, set[str]], sources: set[str]) -> set[str]: + """Returns every node reachable from *sources* (sources excluded).""" + seen: set[str] = set() + queue: deque[str] = deque(sources) + while queue: + node = queue.popleft() + for successor in adjacency.get(node, ()): # noqa: B007 + if successor not in seen: + seen.add(successor) + queue.append(successor) + return seen + + +def _has_cycle(nodes: set[str], edges: set[tuple[str, str]]) -> bool: + """Kahn's algorithm: True when the directed graph has a cycle.""" + adjacency: dict[str, set[str]] = {node: set() for node in nodes} + in_degree: dict[str, int] = {node: 0 for node in nodes} + for upstream, downstream in edges: + if downstream not in adjacency[upstream]: + adjacency[upstream].add(downstream) + in_degree[downstream] = in_degree.get(downstream, 0) + 1 + queue: deque[str] = deque(node for node in nodes if in_degree[node] == 0) + visited = 0 + while queue: + node = queue.popleft() + visited += 1 + for successor in adjacency[node]: + in_degree[successor] -= 1 + if in_degree[successor] == 0: + queue.append(successor) + return visited != len(nodes) + + +def _is_synthesised(task_key: str) -> bool: + return any(task_key.startswith(prefix) for prefix in _SYNTHESISED_KEY_PREFIXES) + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def check_dag_equivalence(adf: AdfPipeline, ir: Pipeline) -> DagEquivalenceResult: + """Check that the IR DAG is a safe motif-quotient of the ADF DAG. + + Args: + adf: The source ADF pipeline (typed AST). + ir: The translated, *motif-collapsed* IR pipeline. + + Returns: + A :class:`DagEquivalenceResult`. ``equivalent`` is True when the IR + top-level DAG equals the quotient of the ADF top-level DAG under the + motif partition recorded in the IR, and every collapsed motif set is + convex (so nothing was reordered) and the result is acyclic. + """ + findings: list[DagFinding] = [] + + adf_names = [activity.name for activity in adf.activities] + adf_name_set = set(adf_names) + + # ADF top-level edges (upstream -> downstream) and per-edge conditions. + adf_adj: dict[str, set[str]] = {name: set() for name in adf_names} + adf_edge_conditions: dict[tuple[str, str], set[str]] = {} + # Per (downstream, upstream) the declared condition set -- used by the + # merged-outcome check, which mirrors the collapser's dedupe-by-source. + inbound_conditions: dict[str, dict[str, frozenset[str]]] = {} + for activity in adf.activities: + for adf_dep in activity.depends_on or []: + upstream, downstream = adf_dep.activity, activity.name + conditions = set(adf_dep.dependency_conditions or [_DEFAULT_OUTCOME]) + if upstream not in adf_name_set: + findings.append( + DagFinding( + code="dangling_adf_dependency", + severity="warning", + message=( + f"ADF activity '{downstream}' depends on '{upstream}', " + "which is not a top-level activity; edge ignored." + ), + nodes=(upstream, downstream), + ) + ) + continue + adf_adj[upstream].add(downstream) + adf_edge_conditions.setdefault((upstream, downstream), set()).update(conditions) + inbound_conditions.setdefault(downstream, {})[upstream] = frozenset(conditions) + + # Motif partition + IR block bookkeeping, read straight from the IR so we + # validate the *actual* collapse rather than re-running detection. + block_of_name: dict[str, str] = {} + motif_members: dict[str, set[str]] = {} + ir_blocks: set[str] = set() + synthesised_blocks: set[str] = set() + key_to_block: dict[str, str] = {} + + for task in ir.tasks: + label = task.task_key + ir_blocks.add(label) + key_to_block[task.task_key] = label + if isinstance(task, MotifActivity): + member_names = set(task.matched_activity_names) + motif_members[label] = member_names + for name in member_names: + block_of_name[name] = label + elif _is_synthesised(task.task_key): + synthesised_blocks.add(label) + elif task.name in adf_name_set: + # task.name is the original ADF activity name; key the singleton block by the IR task's + # actual task_key so labels match the engine's case-preserving sanitiser. Anything else is an + # unexpected IR-only task, left to surface as unmapped_ir_task. + block_of_name[task.name] = label + + # Any ADF activity with no corresponding IR task at all -> sentinel block. + # The label is absent from ``ir_blocks`` so it surfaces as ``missing_node``. + for name in adf_names: + block_of_name.setdefault(name, normalize_task_key(name)) + + adf_blocks = set(block_of_name.values()) + + # ----- Quotient of the ADF DAG under the partition ----- + quotient_edges: set[tuple[str, str]] = set() + quotient_conditions: dict[tuple[str, str], set[str]] = {} + collapsed_internal = 0 + for (upstream, downstream), conditions in adf_edge_conditions.items(): + block_upstream, block_downstream = block_of_name[upstream], block_of_name[downstream] + if block_upstream == block_downstream: + collapsed_internal += 1 + continue + quotient_edges.add((block_upstream, block_downstream)) + quotient_conditions.setdefault((block_upstream, block_downstream), set()).update(conditions) + if collapsed_internal: + findings.append( + DagFinding( + code="collapsed_internal_edges", + severity="tolerated", + message=( + f"{collapsed_internal} intra-motif edge(s) absorbed into collapsed " + "motif node(s); expected and ignored." + ), + ) + ) + + # ----- IR block-space edges ----- + ir_edges: set[tuple[str, str]] = set() + ir_conditions: dict[tuple[str, str], set[str]] = {} + for task in ir.tasks: + block_downstream = key_to_block[task.task_key] + for ir_dep in task.depends_on or []: + block_upstream = key_to_block.get(ir_dep.task_key, ir_dep.task_key) + if block_upstream == block_downstream: + continue + if block_upstream in synthesised_blocks or block_downstream in synthesised_blocks: + findings.append( + DagFinding( + code="synthesised_edge", + severity="tolerated", + message=f"Edge involving synthesised task '{block_upstream}' -> '{block_downstream}' ignored.", + nodes=(block_upstream, block_downstream), + ) + ) + continue + ir_edges.add((block_upstream, block_downstream)) + ir_conditions.setdefault((block_upstream, block_downstream), set()).add(ir_dep.outcome or _DEFAULT_OUTCOME) + + # ----- Node comparison ----- + for block in sorted(adf_blocks - ir_blocks): + claimed = motif_members.get(block) + label = block if claimed is None else f"motif[{', '.join(sorted(claimed))}]" + findings.append( + DagFinding( + code="missing_node", + severity="violation", + message=f"ADF activity/block '{label}' has no corresponding IR task.", + nodes=(block,), + ) + ) + for block in sorted(ir_blocks - adf_blocks - synthesised_blocks): + findings.append( + DagFinding( + code="unmapped_ir_task", + severity="warning", + message=f"IR task '{block}' has no ADF preimage and is not a recognised synthesised task.", + nodes=(block,), + ) + ) + for block in sorted(synthesised_blocks): + findings.append( + DagFinding( + code="synthesised_task", + severity="tolerated", + message=f"IR-only synthesised task '{block}' ignored.", + nodes=(block,), + ) + ) + + # ----- Edge comparison (only between blocks present on both sides) ----- + comparable = adf_blocks & ir_blocks + for edge in sorted(quotient_edges - ir_edges): + if edge[0] in comparable and edge[1] in comparable: + findings.append( + DagFinding( + code="missing_edge", + severity="violation", + message=f"Ordering edge '{edge[0]}' -> '{edge[1]}' present in ADF is missing from the IR DAG.", + nodes=edge, + ) + ) + for edge in sorted(ir_edges - quotient_edges): + findings.append( + DagFinding( + code="extra_edge", + severity="warning", + message=( + f"IR DAG adds ordering edge '{edge[0]}' -> '{edge[1]}' not implied by the ADF DAG " + "(over-constraining)." + ), + nodes=edge, + ) + ) + for edge in sorted(quotient_edges & ir_edges): + adf_outcomes = quotient_conditions.get(edge, set()) + ir_outcomes = ir_conditions.get(edge, set()) + if adf_outcomes and adf_outcomes != ir_outcomes: + findings.append( + DagFinding( + code="outcome_mismatch", + severity="warning", + message=( + f"Edge '{edge[0]}' -> '{edge[1]}' dependency condition changed: " + f"ADF {sorted(adf_outcomes)} vs IR {sorted(ir_outcomes)}." + ), + nodes=edge, + ) + ) + + # ----- Convexity: nothing reordered by any contraction ----- + reverse_adj: dict[str, set[str]] = {name: set() for name in adf_names} + for upstream, downstreams in adf_adj.items(): + for downstream in downstreams: + reverse_adj[downstream].add(upstream) + + for label, members in motif_members.items(): + valid_members = members & adf_name_set + if len(valid_members) < 2: + continue + descendants = _reachable(adf_adj, set(valid_members)) + ancestors = _reachable(reverse_adj, set(valid_members)) + between = (descendants & ancestors) - valid_members + if between: + findings.append( + DagFinding( + code="non_convex_motif", + severity="violation", + message=( + f"Motif '{label}' is non-convex: external activity(ies) " + f"{sorted(between)} lie on a dependency path between collapsed " + "members, so collapsing reorders them. Collapse is unsafe." + ), + nodes=tuple(sorted(between)), + ) + ) + + # Merged-outcome: collapser dedupes external deps by source, keeping + # the first outcome -- flag when members disagree on a shared source. + per_source: dict[str, set[frozenset[str]]] = {} + for member in valid_members: + for source, conds in inbound_conditions.get(member, {}).items(): + if source not in valid_members: + per_source.setdefault(source, set()).add(conds) + for source, condition_sets in per_source.items(): + if len(condition_sets) > 1: + findings.append( + DagFinding( + code="merged_outcome", + severity="warning", + message=( + f"Motif '{label}' collapses edges from '{source}' that carried " + f"differing conditions {[sorted(conditions) for conditions in condition_sets]}; " + "collapse keeps only one." + ), + nodes=(source, label), + ) + ) + + # ----- Acyclicity backstop on the deployable IR DAG ----- + if _has_cycle(ir_blocks, ir_edges): + findings.append( + DagFinding( + code="cycle", + severity="violation", + message="The translated IR DAG contains a dependency cycle.", + ) + ) + + equivalent = not any(finding.severity == "violation" for finding in findings) + return DagEquivalenceResult(equivalent=equivalent, findings=findings) + + +def format_result(result: DagEquivalenceResult) -> str: + """Render a result as a compact human-readable report.""" + status = "EQUIVALENT" if result.equivalent else "NOT EQUIVALENT" + lines = [f"DAG equivalence: {status}"] + for label, items in ( + ("Violations", result.violations), + ("Warnings", result.warnings), + ("Tolerated", result.tolerated), + ): + if not items: + continue + lines.append(f" {label} ({len(items)}):") + lines.extend(f" - [{finding.code}] {finding.message}" for finding in items) + return "\n".join(lines) diff --git a/src/orchestra/__init__.py b/src/orchestra/__init__.py deleted file mode 100644 index 16c449e..0000000 --- a/src/orchestra/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Flowx - ADF to Databricks translation plugin for Claude Code.""" - -__version__ = "0.1.0" diff --git a/src/orchestra/adapter/__init__.py b/src/orchestra/adapter/__init__.py deleted file mode 100644 index 6add5d5..0000000 --- a/src/orchestra/adapter/__init__.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Agent-facing surfaces and the matching pipeline modifier for flowx translation. - -This package draws a deliberate line between two roles: - -* **Agent adapter** -- :mod:`flowx.adapter.session` plus the question - shapes in :mod:`flowx.adapter.models`. This is the layer an agent - calls. It converts tool-call arguments into deterministic service calls - and maps "need more input" signals into structured objects (and the - :exc:`TranslationInputRequired` exception) the agent can hand back to - the user. - -* **Pipeline modifier** -- :mod:`flowx.adapter.operations`. The - deterministic transformation that consumes a validated - :class:`TranslationPreferences` snapshot and stamps concrete decisions - onto a Pipeline IR. It has no awareness of agents or user prompts and - is safely importable from non-agent contexts (CLI, tests, batch jobs). - -The package is organised into three primary modules plus the predicates -and session helpers: - -* :mod:`~flowx.adapter.models` -- StrEnums and dataclasses. -* :mod:`~flowx.adapter.operations` -- Free functions - (``gather_questions``, ``apply_preferences``, ``validate_answer``, - ``allowed_values_for``). -* :mod:`~flowx.adapter.constants` -- Question IDs, compute-mode - strings, replacement names, and other shared constants. -* :mod:`~flowx.adapter.predicates` -- Pure IR predicates used by - both ``operations`` and the bundler. -* :mod:`~flowx.adapter.session` -- The agent adapter class. -""" - -from __future__ import annotations - -from flowx.adapter.models import ( - DEFAULT_PREFERENCES, - CopyActivityParadigm, - LakeflowConnectorType, - MetadataDrivenAccess, - MetadataDrivenConsolidate, - MetadataDrivenLookupTool, - MetadataDrivenSize, - MigrationInputQuestion, - MotifConsolidate, - NonDatabricksTaskCompute, - PendingMigrationInputs, - PendingQuestions, - QuestionOption, - TranslationPreferences, - TranslationQuestion, - UseLakeflowConnectors, -) -from flowx.adapter.operations import ( - allowed_values_for, - apply_preferences, - collect_workspace_artifact_paths, - detect_databricks_hosts, - enum_for, - gather_questions, - validate_answer, -) -from flowx.adapter.session import ( - MigrationInputSession, - TranslationInputRequired, - TranslationSession, - UnknownMigrationPhaseError, -) - -__all__ = [ - "DEFAULT_PREFERENCES", - "CopyActivityParadigm", - "LakeflowConnectorType", - "MetadataDrivenAccess", - "MetadataDrivenConsolidate", - "MetadataDrivenLookupTool", - "MetadataDrivenSize", - "MigrationInputQuestion", - "MigrationInputSession", - "MotifConsolidate", - "NonDatabricksTaskCompute", - "PendingMigrationInputs", - "PendingQuestions", - "QuestionOption", - "TranslationInputRequired", - "TranslationPreferences", - "TranslationQuestion", - "TranslationSession", - "UnknownMigrationPhaseError", - "UseLakeflowConnectors", - "allowed_values_for", - "apply_preferences", - "collect_workspace_artifact_paths", - "detect_databricks_hosts", - "enum_for", - "gather_questions", - "validate_answer", -] diff --git a/src/orchestra/adapter/__main__.py b/src/orchestra/adapter/__main__.py deleted file mode 100644 index 490de59..0000000 --- a/src/orchestra/adapter/__main__.py +++ /dev/null @@ -1,597 +0,0 @@ -"""CLI bridge that lets the flowx skills drive the adapter via subprocesses. - -The skills (`/flowx:translate`, `/flowx:migrate`) cannot keep a -Python session alive across user prompts, so this module exposes two -stateless subcommands: - -* ``inspect`` reads a translation report and emits the pending questions - as JSON for the agent to surface to the user. -* ``modify`` reads the same report plus a JSON file of answers and writes - a preference-stamped report the prepare phase consumes verbatim. -""" - -from __future__ import annotations - -import argparse -import json -import sys -from dataclasses import asdict -from pathlib import Path -from typing import Any - -from flowx.adapter.constants import MOTIF_CONSOLIDATE_QUESTION_PREFIX -from flowx.adapter.models import ( - DEFAULT_PREFERENCES, - CopyActivityParadigm, - LakeflowConnectorType, - MetadataDrivenAccess, - MetadataDrivenConsolidate, - MetadataDrivenLookupTool, - MetadataDrivenSize, - MotifConsolidate, - NonDatabricksTaskCompute, - PendingQuestions, - TranslationPreferences, - TranslationQuestion, - UseLakeflowConnectors, -) -from flowx.adapter.operations import ( - apply_preferences, - collect_workspace_artifact_paths, - detect_databricks_hosts, - gather_questions, - validate_answer, -) -from flowx.bundler.dab_writer import pipeline_dict_to_ir -from flowx.translator.engine import _pipeline_to_dict - - -def main(argv: list[str] | None = None) -> int: - """Dispatches an ``inspect`` or ``modify`` subcommand. - - Args: - argv: CLI arguments to parse. Defaults to :data:`sys.argv` when - ``None``. - - Returns: - Exit code (0 on success, non-zero on usage or runtime errors). - """ - parser = _build_parser() - args = parser.parse_args(argv) - if args.command == "inspect": - return _run_inspect(args) - if args.command == "modify": - return _run_modify(args) - if args.command == "materialize-lookup": - return _run_materialize_lookup(args) - if args.command == "inputs": - return _run_inputs(args) - if args.command == "workspace-paths": - return _run_workspace_paths(args) - parser.print_help(sys.stderr) - return 2 - - -def _run_workspace_paths(args: argparse.Namespace) -> int: - """Implements the ``workspace-paths`` subcommand. - - Args: - args: Parsed CLI namespace carrying ``report``, ``source_dir``, - and ``out``. - - Returns: - ``0`` on success. The command always succeeds when the report - can be read; missing or unreadable inputs simply produce empty - path / host lists so the skill can detect the no-op case. - """ - paths = collect_workspace_artifact_paths(args.report) - suggested_hosts = detect_databricks_hosts(args.source_dir) if args.source_dir else [] - payload = { - "paths": paths, - "suggested_hosts": suggested_hosts, - "needs_auth": bool(paths), - } - _emit_json(payload, args.out) - return 0 - - -def _run_inputs(args: argparse.Namespace) -> int: - """Implements the ``inputs`` subcommand. - - Args: - args: Parsed CLI namespace carrying ``phase`` and ``out``. - - Returns: - ``0`` on success. The CLI never raises here because the phase - argument is constrained by argparse. - """ - from flowx.adapter.session import MigrationInputSession - - session = MigrationInputSession(phase=args.phase) - pending = session.pending() - payload = { - "phase": pending.phase, - "questions": [ - { - "question_id": question.question_id, - "prompt": question.prompt, - "description": question.description, - "default": question.default, - "required": question.required, - } - for question in pending.questions - ], - } - _emit_json(payload, args.out) - return 0 - - -def _build_parser() -> argparse.ArgumentParser: - """Builds the top-level argparse parser with the two subcommands. - - Returns: - Configured :class:`argparse.ArgumentParser`. - """ - parser = argparse.ArgumentParser( - prog="python -m flowx.adapter", - description="Inspect and modify a translated flowx pipeline IR.", - ) - subparsers = parser.add_subparsers(dest="command", required=True) - - inspect = subparsers.add_parser( - "inspect", - help="Emit pending translation questions for a report as JSON.", - ) - inspect.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") - inspect.add_argument( - "--answers", - type=Path, - default=None, - help=( - "Optional path to a JSON file of answers already collected; " - "questions whose conditions depend on those answers will surface " - "only when their conditions are met." - ), - ) - inspect.add_argument( - "--out", - type=Path, - default=None, - help="Optional output file; defaults to stdout.", - ) - - modify = subparsers.add_parser( - "modify", - help="Apply collected answers to a translation report and write the stamped IR.", - ) - modify.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") - modify.add_argument("answers", type=Path, help="Path to a JSON file mapping question_id to answer string.") - modify.add_argument( - "--out", - type=Path, - required=True, - help="Destination path for the preference-stamped IR JSON.", - ) - modify.add_argument( - "--lookup-values", - type=Path, - default=None, - help=( - "Optional path to a JSON list of lookup-value rows that consolidated " - "metadata-driven motifs should ingest. Each row is a dict mirroring " - "a row from the source Lookup query." - ), - ) - - workspace_paths = subparsers.add_parser( - "workspace-paths", - help=( - "Detect absolute workspace paths in a stamped report and suggest " - "Databricks workspace hosts from the ADF linked services." - ), - ) - workspace_paths.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") - workspace_paths.add_argument( - "--source-dir", - type=Path, - default=None, - help=( - "Optional path to the ADF JSON export directory. When supplied, " - "the command reads ``linked_services/*.json`` to suggest the " - "workspace host that ``databricks auth login --host`` should use." - ), - ) - workspace_paths.add_argument( - "--out", - type=Path, - default=None, - help="Optional output file; defaults to stdout.", - ) - - inputs = subparsers.add_parser( - "inputs", - help="Emit the migration-phase input questions for an flowx phase as JSON.", - ) - inputs.add_argument( - "phase", - choices=("ingest", "translate", "prepare"), - help="Migration phase whose input prompts the agent should surface.", - ) - inputs.add_argument( - "--out", - type=Path, - default=None, - help="Optional output file; defaults to stdout.", - ) - - materialize = subparsers.add_parser( - "materialize-lookup", - help="Parse CSV-shaped lookup values into the JSON shape modify consumes.", - ) - materialize.add_argument( - "source", - help=( - "Either a path to a CSV file or a literal CSV string. The first row " - "is treated as headers and every subsequent row is emitted as one dict." - ), - ) - materialize.add_argument( - "--out", - type=Path, - required=True, - help="Destination path for the lookup-values JSON list.", - ) - return parser - - -def _run_inspect(args: argparse.Namespace) -> int: - """Implements the ``inspect`` subcommand. - - Args: - args: Parsed CLI namespace carrying ``report``, ``answers``, and - ``out``. - - Returns: - ``0`` when the report was inspected successfully, ``1`` when the - report could not be loaded. - """ - pipelines = _load_pipelines(args.report) - if pipelines is None: - return 1 - answers = _read_answers_optional(args.answers) if getattr(args, "answers", None) else {} - payload = { - "pipelines": [_pending_to_payload(gather_questions(pipeline, [], answers=answers)) for pipeline in pipelines], - } - _emit_json(payload, args.out) - return 0 - - -def _read_answers_optional(answers_path: Path) -> dict[str, str]: - """Loads an answers JSON file supplied to ``inspect``. - - Args: - answers_path: Path to a JSON file mapping question_id to answer. - - Returns: - Mapping of question_id to answer string. Returns an empty dict - when the file is missing or unparseable so ``inspect`` still - succeeds (question gating just sees no prior answers). - """ - try: - raw = json.loads(answers_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return {} - return {key: str(value) for key, value in raw.items() if isinstance(value, str)} - - -def _run_modify(args: argparse.Namespace) -> int: - """Implements the ``modify`` subcommand. - - Args: - args: Parsed CLI namespace carrying ``report``, ``answers``, - ``out``, and the optional ``lookup_values``. - - Returns: - ``0`` when the modified IR was written successfully, ``1`` when - the report could not be loaded, ``2`` when the answers failed - validation. - """ - pipelines = _load_pipelines(args.report) - if pipelines is None: - return 1 - try: - answers = _load_answers(args.answers) - preferences = _preferences_from_answers(answers) - except ValueError as error: - print(f"Invalid answers payload: {error}", file=sys.stderr) - return 2 - lookup_values = _load_lookup_values(args.lookup_values) if args.lookup_values else [] - stamped_pipelines = [ - _stamp_lookup_values_into_metadata_driven_motifs(apply_preferences(pipeline, preferences), lookup_values) - for pipeline in pipelines - ] - modified = [_pipeline_to_dict(pipeline) for pipeline in stamped_pipelines] - _write_modified_report(args.report, modified, args.out) - return 0 - - -def _run_materialize_lookup(args: argparse.Namespace) -> int: - """Implements the ``materialize-lookup`` subcommand. - - Args: - args: Parsed CLI namespace carrying ``source`` (file path or - literal CSV string) and ``out``. - - Returns: - ``0`` when the JSON was written successfully, ``2`` when the - source could not be parsed as CSV. - """ - try: - rows = _parse_csv_source(args.source) - except ValueError as error: - print(f"Invalid CSV source: {error}", file=sys.stderr) - return 2 - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8") - return 0 - - -def _parse_csv_source(source: str) -> list[dict[str, str]]: - """Parses a CSV file path or literal CSV string into a list of row dicts. - - Args: - source: Either a path to a CSV file or a literal CSV string with - a header row. - - Returns: - List of dicts, one per data row, keyed by the header names. - - Raises: - ValueError: When the CSV has no header row or is empty. - """ - import csv - - source_path = Path(source) - text = source_path.read_text(encoding="utf-8") if source_path.exists() else source - reader = csv.DictReader(text.splitlines()) - if reader.fieldnames is None: - raise ValueError("Source CSV is empty or missing a header row") - return [dict(row) for row in reader] - - -def _load_lookup_values(lookup_values_path: Path) -> list[dict[str, Any]]: - """Loads materialised lookup values from a JSON file. - - Args: - lookup_values_path: Path to a JSON list of row dicts. - - Returns: - The parsed list of row dicts. Returns an empty list when the - file is missing or unparseable so the modify pass still succeeds - (consolidated motifs will warn and fall back to the scaffold). - """ - try: - raw = json.loads(lookup_values_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return [] - if not isinstance(raw, list): - return [] - return [row for row in raw if isinstance(row, dict)] - - -def _stamp_lookup_values_into_metadata_driven_motifs(pipeline, lookup_values: list[dict[str, Any]]): - """Stamps lookup values onto every metadata-driven motif marked for consolidation. - - Args: - pipeline: Preference-stamped pipeline IR. - lookup_values: Rows materialised by the agent or the user. - - Returns: - A new :class:`Pipeline` whose metadata-driven motif activities - carry the supplied lookup rows. When *lookup_values* is empty - the pipeline is returned unchanged. - """ - if not lookup_values: - return pipeline - import dataclasses as _dataclasses - - from flowx.models.ir import MotifActivity as _MotifActivity - - stamped_tasks = [] - for task in pipeline.tasks: - if isinstance(task, _MotifActivity) and task.consolidate_metadata_driven: - stamped_tasks.append(_dataclasses.replace(task, lookup_values=list(lookup_values))) - else: - stamped_tasks.append(task) - return _dataclasses.replace(pipeline, tasks=stamped_tasks) - - -def _load_pipelines(report_path: Path) -> list[Any] | None: - """Loads every pipeline IR contained in a report file. - - Args: - report_path: Path to a translation report or pipeline IR JSON. - - Returns: - List of rehydrated :class:`Pipeline` objects, or ``None`` when - the file could not be parsed. - """ - try: - raw = json.loads(report_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as error: - print(f"Failed to read {report_path}: {error}", file=sys.stderr) - return None - pipeline_dicts = _extract_pipeline_dicts(raw) - return [pipeline_dict_to_ir(pipeline_dict)[0] for pipeline_dict in pipeline_dicts] - - -def _extract_pipeline_dicts(raw: Any) -> list[dict[str, Any]]: - """Normalises a translation report into a list of pipeline IR dicts. - - Args: - raw: Parsed JSON content from a report file. - - Returns: - List of dicts, each in the shape ``engine._pipeline_to_dict`` - produces. Empty when *raw* does not contain a recognisable - pipeline payload. - """ - if isinstance(raw, dict) and "tasks" in raw and "name" in raw: - return [raw] - if isinstance(raw, dict) and "translations" in raw: - return [ - {"name": entry["pipeline"], **entry["ir"]} - for entry in raw.get("translations", []) - if entry.get("status") == "translated" and entry.get("ir") - ] - return [] - - -def _load_answers(answers_path: Path) -> dict[str, str]: - """Loads a JSON answers file and validates its top-level shape. - - Args: - answers_path: Path to a JSON file mapping question_id to answer. - - Returns: - Mapping of question_id to answer string. - - Raises: - ValueError: When the file is not a JSON object of string values. - """ - raw = json.loads(answers_path.read_text(encoding="utf-8")) - if not isinstance(raw, dict): - raise ValueError(f"Expected a JSON object at {answers_path}; got {type(raw).__name__}") - coerced: dict[str, str] = {} - for key, value in raw.items(): - if not isinstance(value, str): - raise ValueError(f"Answer for {key!r} must be a string; got {type(value).__name__}") - coerced[key] = value - return coerced - - -def _preferences_from_answers(answers: dict[str, str]) -> TranslationPreferences: - """Builds a :class:`TranslationPreferences` from a validated answers dict. - - Args: - answers: Validated mapping of question_id to answer string. - - Returns: - Preferences with every answered field overridden and every - unanswered field defaulted. - - Raises: - ValueError: When an answer is not in the allowed set for its - question. - """ - validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} - motif_consolidations: dict[str, MotifConsolidate] = {} - for qid, value in validated.items(): - if qid.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): - motif_consolidations[qid[len(MOTIF_CONSOLIDATE_QUESTION_PREFIX) :]] = MotifConsolidate(value) - return TranslationPreferences( - copy_activity_paradigm=CopyActivityParadigm( - validated.get("copy_activity_paradigm", DEFAULT_PREFERENCES.copy_activity_paradigm) - ), - non_databricks_task_compute=NonDatabricksTaskCompute( - validated.get("non_databricks_task_compute", DEFAULT_PREFERENCES.non_databricks_task_compute) - ), - use_lakeflow_connectors=UseLakeflowConnectors( - validated.get("use_lakeflow_connectors", DEFAULT_PREFERENCES.use_lakeflow_connectors) - ), - lakeflow_connector_type=LakeflowConnectorType( - validated.get("lakeflow_connector_type", DEFAULT_PREFERENCES.lakeflow_connector_type) - ), - metadata_driven_consolidate=MetadataDrivenConsolidate( - validated.get("metadata_driven_consolidate", DEFAULT_PREFERENCES.metadata_driven_consolidate) - ), - metadata_driven_access=MetadataDrivenAccess( - validated.get("metadata_driven_access", DEFAULT_PREFERENCES.metadata_driven_access) - ), - metadata_driven_size=MetadataDrivenSize( - validated.get("metadata_driven_size", DEFAULT_PREFERENCES.metadata_driven_size) - ), - metadata_driven_lookup_tool=MetadataDrivenLookupTool( - validated.get("metadata_driven_lookup_tool", DEFAULT_PREFERENCES.metadata_driven_lookup_tool) - ), - motif_consolidations=motif_consolidations, - ) - - -def _pending_to_payload(pending: PendingQuestions) -> dict[str, Any]: - """Serialises pending questions for transmission over stdout. - - Args: - pending: Outstanding questions for a single pipeline. - - Returns: - JSON-friendly dict the agent can iterate over to prompt the user. - """ - return { - "pipeline_name": pending.pipeline_name, - "questions": [_question_to_payload(question) for question in pending.questions], - } - - -def _question_to_payload(question: TranslationQuestion) -> dict[str, Any]: - """Serialises a single :class:`TranslationQuestion` to a JSON-friendly dict. - - Args: - question: Question to serialise. - - Returns: - Dict containing the question's fields with options flattened to - plain dicts. - """ - return { - "question_id": question.question_id, - "prompt": question.prompt, - "rationale": question.rationale, - "options": [asdict(option) for option in question.options], - "affected_task_keys": list(question.affected_task_keys), - "default": question.default, - } - - -def _emit_json(payload: dict[str, Any], out: Path | None) -> None: - """Writes a JSON payload to a file or to stdout. - - Args: - payload: JSON-serialisable mapping to emit. - out: Destination path; ``None`` selects stdout. - """ - encoded = json.dumps(payload, indent=2, default=str) - if out is None: - print(encoded) - return - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(encoded + "\n", encoding="utf-8") - - -def _write_modified_report(report_path: Path, pipelines: list[dict[str, Any]], out: Path) -> None: - """Writes the preference-stamped IR to *out* using the input report's shape. - - Args: - report_path: Path the modified report was sourced from. Used - only to detect whether the input was a single pipeline IR - or an aggregated translation report. - pipelines: Stamped pipeline IR dicts to write. - out: Destination path for the modified report. - """ - raw = json.loads(report_path.read_text(encoding="utf-8")) - if isinstance(raw, dict) and "translations" in raw: - by_name = {pipeline["name"]: pipeline for pipeline in pipelines} - for entry in raw.get("translations", []): - stamped = by_name.get(entry.get("pipeline")) - if stamped is not None and entry.get("ir") is not None: - entry["ir"] = {key: value for key, value in stamped.items() if key != "name"} - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(raw, indent=2, default=str) + "\n", encoding="utf-8") - return - payload = pipelines[0] if len(pipelines) == 1 else {"pipelines": pipelines} - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/orchestra/adapter/session.py b/src/orchestra/adapter/session.py deleted file mode 100644 index 318622d..0000000 --- a/src/orchestra/adapter/session.py +++ /dev/null @@ -1,450 +0,0 @@ -"""Agent adapter that drives the ask-validate-resume loop. - -:class:`TranslationSession` is the entry point an agent uses to -translate tool-call arguments into validated preferences. When the IR -raises questions the agent cannot answer from context alone, the -session surfaces them as structured :class:`TranslationQuestion` -objects (and, via :exc:`TranslationInputRequired`, as exceptions) so -the agent can route them back to the user. The pipeline modifier is -invoked only once every question has an answer. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field - -from flowx.adapter.constants import ( - INPUT_ADF_RESOURCE_URL, - INPUT_ADF_SOURCE_PATH, - INPUT_BUNDLE_NAME, - INPUT_CATALOG, - INPUT_DATABRICKS_PROFILE, - INPUT_INVENTORY_PATH, - INPUT_OUTPUT_BUNDLE_PATH, - INPUT_OUTPUT_DIR, - INPUT_SCHEMA, - INPUT_TRANSLATION_REPORT_PATH, - PHASE_INGEST, - PHASE_PREPARE, - PHASE_TRANSLATE, -) -from flowx.adapter.models import ( - DEFAULT_PREFERENCES, - CopyActivityParadigm, - LakeflowConnectorType, - MetadataDrivenAccess, - MetadataDrivenConsolidate, - MetadataDrivenLookupTool, - MetadataDrivenSize, - MigrationInputQuestion, - MotifConsolidate, - NonDatabricksTaskCompute, - PendingMigrationInputs, - PendingQuestions, - TranslationPreferences, - TranslationQuestion, - UseLakeflowConnectors, -) -from flowx.adapter.operations import ( - apply_preferences, - gather_questions, - validate_answer, -) -from flowx.models.ir import Pipeline -from flowx.models.motifs import DetectedMotif - - -class TranslationInputRequired(Exception): - """Raised by :meth:`TranslationSession.run` when answers are still missing. - - Attributes: - pending: The outstanding questions the agent should route to the - user before retrying :meth:`TranslationSession.run`. - """ - - def __init__(self, pending: PendingQuestions) -> None: - """Stores the pending questions on the exception. - - Args: - pending: Outstanding questions surfaced by the session. - """ - super().__init__( - f"{len(pending.questions)} translation question(s) require user input " - f"for pipeline {pending.pipeline_name!r}" - ) - self.pending = pending - - -@dataclass(slots=True, kw_only=True) -class TranslationSession: - """Coordinates the ask-validate-resume loop for one translated pipeline. - - A session is single-use: the caller drives it by either polling via - :meth:`pending` and :meth:`answer`, or calling :meth:`run` and - handling :exc:`TranslationInputRequired`. When every question is - answered, :meth:`run` (or :meth:`resume`) returns the - preference-stamped pipeline. - - Attributes: - pipeline: Translated pipeline IR after motif collapsing. - motifs: Detected motifs for the pipeline. Optional; only used to - decide whether the Lakeflow Connect question applies. - defaults: Baseline preferences applied when the caller skips a - question. Per-task overrides on this object are preserved - verbatim when :meth:`build_preferences` composes the final - snapshot. - """ - - pipeline: Pipeline - motifs: list[DetectedMotif] = field(default_factory=list) - defaults: TranslationPreferences = DEFAULT_PREFERENCES - _answers: dict[str, str] = field(default_factory=dict) - - def pending(self) -> PendingQuestions: - """Returns the questions still awaiting an answer. - - Returns: - A :class:`PendingQuestions` instance containing only the - questions whose preconditions are met by the IR and whose - IDs are not yet in the answer set. - """ - return gather_questions( - self.pipeline, - self.motifs, - answers=self._answers, - ) - - def answer(self, question_id: str, value: str) -> None: - """Validates and records a single answer. - - Args: - question_id: Stable question identifier from - :class:`TranslationQuestion`. - value: Caller-supplied answer string. - - Raises: - ValueError: When *question_id* is unknown or *value* is not - in the allowed set for the question. - """ - self._answers[question_id] = validate_answer(question_id, value) - - def answer_many(self, answers: dict[str, str]) -> None: - """Validates and records multiple answers atomically. - - Args: - answers: Mapping of question_id to the caller-supplied answer. - - Raises: - ValueError: When any pair fails validation. No answers from - the batch are recorded when the call raises. - """ - validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} - self._answers.update(validated) - - def find_question(self, question_id: str) -> TranslationQuestion | None: - """Looks up a pending question by its identifier. - - Args: - question_id: Stable question identifier. - - Returns: - The matching :class:`TranslationQuestion` if it is still - pending, otherwise ``None``. - """ - return next( - (question for question in self.pending().questions if question.question_id == question_id), - None, - ) - - def build_preferences(self) -> TranslationPreferences: - """Composes the validated preferences snapshot from collected answers. - - Returns: - A :class:`TranslationPreferences` where every answered field - takes the caller-supplied value and every unanswered field - falls back to the corresponding value on ``defaults``. - """ - return TranslationPreferences( - copy_activity_paradigm=CopyActivityParadigm( - self._answers.get("copy_activity_paradigm", self.defaults.copy_activity_paradigm) - ), - non_databricks_task_compute=NonDatabricksTaskCompute( - self._answers.get("non_databricks_task_compute", self.defaults.non_databricks_task_compute) - ), - use_lakeflow_connectors=UseLakeflowConnectors( - self._answers.get("use_lakeflow_connectors", self.defaults.use_lakeflow_connectors) - ), - lakeflow_connector_type=LakeflowConnectorType( - self._answers.get("lakeflow_connector_type", self.defaults.lakeflow_connector_type) - ), - metadata_driven_consolidate=MetadataDrivenConsolidate( - self._answers.get("metadata_driven_consolidate", self.defaults.metadata_driven_consolidate) - ), - metadata_driven_access=MetadataDrivenAccess( - self._answers.get("metadata_driven_access", self.defaults.metadata_driven_access) - ), - metadata_driven_size=MetadataDrivenSize( - self._answers.get("metadata_driven_size", self.defaults.metadata_driven_size) - ), - metadata_driven_lookup_tool=MetadataDrivenLookupTool( - self._answers.get("metadata_driven_lookup_tool", self.defaults.metadata_driven_lookup_tool) - ), - motif_consolidations=self._collect_motif_consolidations(), - per_task=self.defaults.per_task, - ) - - def _collect_motif_consolidations(self) -> dict[str, MotifConsolidate]: - """Returns the per-motif consolidation answers gathered so far. - - Returns: - Dict mapping ``motif_id`` to the user's :class:`MotifConsolidate` - answer. Motifs the user did not answer fall back to the - value carried on ``self.defaults`` (default - :data:`MotifConsolidate.KEEP`). The dict is the union of - the defaults and any answers whose ``question_id`` starts - with ``consolidate_motif:``. - """ - from flowx.adapter.constants import MOTIF_CONSOLIDATE_QUESTION_PREFIX - - consolidations: dict[str, MotifConsolidate] = dict(self.defaults.motif_consolidations) - for question_id, answer in self._answers.items(): - if not question_id.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): - continue - motif_id = question_id[len(MOTIF_CONSOLIDATE_QUESTION_PREFIX) :] - consolidations[motif_id] = MotifConsolidate(answer) - return consolidations - - def resume(self) -> Pipeline: - """Returns the preference-stamped pipeline IR. - - Returns: - A new :class:`Pipeline` produced by applying the composed - preferences to ``self.pipeline``. The input pipeline is not - mutated. - """ - return apply_preferences(self.pipeline, self.build_preferences()) - - def run(self) -> Pipeline: - """Returns the modified pipeline, raising when input is still required. - - Returns: - The preference-stamped pipeline IR when every applicable - question has an answer. - - Raises: - TranslationInputRequired: When one or more questions are - still outstanding. The exception carries the pending - questions so the agent can route them to the user. - """ - pending = self.pending() - if pending.questions: - raise TranslationInputRequired(pending) - return self.resume() - - -_INGEST_QUESTIONS: tuple[MigrationInputQuestion, ...] = ( - MigrationInputQuestion( - question_id=INPUT_ADF_SOURCE_PATH, - prompt="Where are the ADF JSON exports?", - description=( - "Unity Catalog volume path (``/Volumes///``) " - "or a local directory containing the ADF ARM/JSON export." - ), - required=True, - ), - MigrationInputQuestion( - question_id=INPUT_ADF_RESOURCE_URL, - prompt="ADF resource URL?", - description=( - "Azure portal URL of the source Data Factory. Captured for " - "traceability and surfaced in the generated bundle README; " - "leave blank when the source is exported from a local copy." - ), - default="", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_OUTPUT_DIR, - prompt="Where should flowx write the ingest output?", - description="Directory the ingest phase writes ``inventory.json`` and ``ast/`` into.", - default="./orchestra_output/ingest", - required=False, - ), -) - -_TRANSLATE_QUESTIONS: tuple[MigrationInputQuestion, ...] = ( - MigrationInputQuestion( - question_id=INPUT_INVENTORY_PATH, - prompt="Path to the inventory.json from the ingest phase?", - description="Inventory produced by the ingest phase that the translator consumes.", - default="./orchestra_output/ingest/inventory.json", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_ADF_SOURCE_PATH, - prompt="Path to the ADF JSON exports?", - description="Same source directory the ingest phase consumed; needed for cross-references.", - required=True, - ), - MigrationInputQuestion( - question_id=INPUT_OUTPUT_DIR, - prompt="Where should flowx write the translate output?", - description="Directory the translate phase writes the report and IR into.", - default="./orchestra_output/translate", - required=False, - ), -) - -_PREPARE_QUESTIONS: tuple[MigrationInputQuestion, ...] = ( - MigrationInputQuestion( - question_id=INPUT_TRANSLATION_REPORT_PATH, - prompt="Path to the translation report?", - description=( - "Preference-stamped report from `python -m flowx.adapter modify`, " - "or the raw translate-phase report when no preferences were applied." - ), - default="./orchestra_output/translate/translation_report.stamped.json", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_OUTPUT_BUNDLE_PATH, - prompt="Where should the generated DAB bundle be written?", - description="Root directory for the emitted Databricks Declarative Automation Bundle.", - default="./dab_output", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_CATALOG, - prompt="Target Unity Catalog catalog?", - description="Default ``catalog`` bundle variable used by emitted notebooks and pipelines.", - default="main", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_SCHEMA, - prompt="Target Unity Catalog schema?", - description="Default ``schema`` bundle variable used by emitted notebooks and pipelines.", - default="default", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_BUNDLE_NAME, - prompt="Bundle name override?", - description="Defaults to the first translated pipeline's resource key when blank.", - default="", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_DATABRICKS_PROFILE, - prompt="Databricks CLI profile?", - description=( - "Profile used to download workspace-resident notebooks during the " - "prepare phase. Leave blank to use the default profile from " - "``~/.databrickscfg`` or the active ``DATABRICKS_*`` env vars." - ), - default="", - required=False, - ), -) - -_QUESTIONS_BY_PHASE: dict[str, tuple[MigrationInputQuestion, ...]] = { - PHASE_INGEST: _INGEST_QUESTIONS, - PHASE_TRANSLATE: _TRANSLATE_QUESTIONS, - PHASE_PREPARE: _PREPARE_QUESTIONS, -} - - -class UnknownMigrationPhaseError(ValueError): - """Raised when a MigrationInputSession is constructed with an unrecognised phase.""" - - -@dataclass(slots=True, kw_only=True) -class MigrationInputSession: - """Coordinates the free-text input prompts at the top of an flowx phase. - - A session is single-use: the caller drives it by polling - :meth:`pending` and recording answers via :meth:`answer`, then reads - them out with :meth:`collected` once every required input has a - value. The session is intentionally distinct from - :class:`TranslationSession` because the inputs it gathers are - free-text paths and identifiers rather than enum-backed choices. - - Attributes: - phase: One of ``"ingest"``, ``"translate"``, ``"prepare"``. - """ - - phase: str - _answers: dict[str, str] = field(default_factory=dict) - - def __post_init__(self) -> None: - """Validates that *phase* is one of the supported migration phases. - - Raises: - UnknownMigrationPhaseError: When *phase* is not registered in - :data:`_QUESTIONS_BY_PHASE`. - """ - if self.phase not in _QUESTIONS_BY_PHASE: - raise UnknownMigrationPhaseError( - f"Unknown migration phase {self.phase!r}; expected one of {sorted(_QUESTIONS_BY_PHASE)}" - ) - - def pending(self) -> PendingMigrationInputs: - """Returns the input questions still awaiting an answer. - - Returns: - A :class:`PendingMigrationInputs` with the unanswered - questions for ``self.phase`` in registration order. - """ - questions = [ - question for question in _QUESTIONS_BY_PHASE[self.phase] if question.question_id not in self._answers - ] - return PendingMigrationInputs(phase=self.phase, questions=questions) - - def answer(self, question_id: str, value: str) -> None: - """Records an answer to one input question. - - Args: - question_id: Stable identifier of the question. - value: Caller-supplied string value. - - Raises: - ValueError: When *question_id* is not a known input for the - session's phase. - """ - if not any(question.question_id == question_id for question in _QUESTIONS_BY_PHASE[self.phase]): - raise ValueError(f"Unknown input question {question_id!r} for phase {self.phase!r}") - self._answers[question_id] = value - - def answer_many(self, answers: dict[str, str]) -> None: - """Records multiple input answers atomically. - - Args: - answers: Mapping of question_id to the caller-supplied value. - - Raises: - ValueError: When any pair references an unknown question. - No answers are recorded when the call raises. - """ - known_ids = {question.question_id for question in _QUESTIONS_BY_PHASE[self.phase]} - unknown = set(answers) - known_ids - if unknown: - raise ValueError(f"Unknown input questions for phase {self.phase!r}: {sorted(unknown)}") - self._answers.update(answers) - - def collected(self) -> dict[str, str]: - """Returns the collected answers merged with each question's default. - - Returns: - A dict keyed by question_id covering every question for the - phase: caller-supplied answers take precedence; otherwise - the question's ``default`` value (which may be the empty - string) is used. Required questions whose answers are - missing are omitted so the caller can detect them. - """ - collected: dict[str, str] = {} - for question in _QUESTIONS_BY_PHASE[self.phase]: - if question.question_id in self._answers: - collected[question.question_id] = self._answers[question.question_id] - elif question.default is not None: - collected[question.question_id] = question.default - return collected diff --git a/src/orchestra/translator/activity_translators/web_activity.py b/src/orchestra/translator/activity_translators/web_activity.py deleted file mode 100644 index 7f33c6b..0000000 --- a/src/orchestra/translator/activity_translators/web_activity.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Translates ADF WebActivity activities to Databricks WebActivity IR.""" - -from __future__ import annotations - -from typing import Any - -from flowx.models.adf_ast import AdfActivity, AdfDefinitions -from flowx.models.ir import Activity, TranslationContext -from flowx.models.ir import WebActivity as WebActivityIR -from flowx.parser.expression_parser import resolve_expression -from flowx.translator.activity_translators.resolve import resolve_dict_values, resolve_field - - -def translate( - activity: AdfActivity, - base_kwargs: dict[str, Any], - context: TranslationContext, - definitions: AdfDefinitions, -) -> Activity: - """Translates a WebActivity. - - Args: - activity: The ADF activity AST node. - base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). - context: Current translation context. - definitions: Full ADF definitions for cross-referencing. - - Returns: - A :class:`WebActivity` IR node. - """ - type_properties = activity.type_properties or {} - - url = resolve_field(type_properties.get("url", ""), context) - method = type_properties.get("method", "GET") - headers = resolve_dict_values(type_properties.get("headers"), context) or None - body = _resolve_body(type_properties.get("body"), context) - authentication = type_properties.get("authentication") - disable_cert_validation = type_properties.get("disableCertValidation", False) - http_request_timeout = type_properties.get("httpRequestTimeout") - - timeout_seconds: int | None = None - if http_request_timeout and isinstance(http_request_timeout, str): - timeout_seconds = _parse_timeout_to_seconds(http_request_timeout) - - return WebActivityIR( - **base_kwargs, - url=url, - method=method, - body=body, - headers=headers, - authentication=authentication, - disable_cert_validation=disable_cert_validation, - http_request_timeout_seconds=timeout_seconds, - ) - - -def _resolve_body(body: Any, context: TranslationContext) -> Any: - """Pre-resolve ADF expressions in the request body at translate time. - - Args: - body: Raw body from the ADF typeProperties. - context: Current translation context with variable caches. - - Returns: - Resolved body — either a Python code string (for notebook_code), - the original body dict, or ``None``. - """ - if body is None: - return None - - if isinstance(body, dict) and body.get("type") == "Expression" and "value" in body: - result = resolve_expression(body, context) - if result is not None and result.kind == "notebook_code": - return result.value - if result is not None and result.kind == "literal": - return result.value - - return body - - -def _parse_timeout_to_seconds(timeout_str: str) -> int | None: - """Parses an ADF timeout string to seconds. - - Args: - timeout_str: Timeout in ``"d.hh:mm:ss"`` or ``"hh:mm:ss"`` format. - - Returns: - Total seconds, or ``None`` if the format is unrecognised. - """ - try: - parts = timeout_str.split(".") - if len(parts) == 2: - days = int(parts[0]) - time_part = parts[1] - else: - days = 0 - time_part = parts[0] - time_parts = time_part.split(":") - hours = int(time_parts[0]) if len(time_parts) > 0 else 0 - minutes = int(time_parts[1]) if len(time_parts) > 1 else 0 - seconds = int(time_parts[2]) if len(time_parts) > 2 else 0 - return days * 86400 + hours * 3600 + minutes * 60 + seconds - except (ValueError, IndexError): - return None diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py index f7744fb..a89456f 100644 --- a/tests/integration/test_end_to_end.py +++ b/tests/integration/test_end_to_end.py @@ -1,6 +1,6 @@ """End-to-end integration tests for the flowx translation pipeline. -These tests exercise the full ingest -> translate -> prepare -> bundle pipeline +These tests exercise the full profile -> translate -> prepare -> bundle pipeline against realistic ADF fixture files, simulating what happens when a user invokes the flowx skills. """ diff --git a/tests/integration/test_path_equivalence.py b/tests/integration/test_path_equivalence.py new file mode 100644 index 0000000..f4b4c89 --- /dev/null +++ b/tests/integration/test_path_equivalence.py @@ -0,0 +1,59 @@ +"""Guard #1: the in-process bundle path and the report round-trip (CLI) path +must produce identical bundles; and every generated bundle must satisfy the +structural invariants (guard #2).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from flowx.bundler.dab_writer import _pipeline_dict_to_workflow, write_bundle +from flowx.parser.adf_loader import load_adf_definitions +from flowx.preparer.workflow_preparer import prepare_workflow +from flowx.translator.engine import _pipeline_to_dict, translate_pipeline +from flowx.validate.bundle_invariants import check_bundle_dir, format_result + +FIXTURES_DIR = Path(__file__).parent.parent / "resources" / "json" +_DEFS = load_adf_definitions(FIXTURES_DIR) +_PIPELINE_NAMES = sorted(p.name for p in _DEFS.pipelines) + + +def _jobs(bundle_dir: Path) -> dict: + """Merge all `resources.jobs` mappings across the bundle's resource files.""" + jobs: dict = {} + for path in sorted((bundle_dir / "resources").glob("*.yml")): + doc = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + jobs.update(((doc.get("resources") or {}).get("jobs") or {})) + return jobs + + +@pytest.mark.parametrize("name", _PIPELINE_NAMES) +def test_inprocess_and_report_paths_agree(name: str, tmp_path: Path) -> None: + pipeline = next(p for p in _DEFS.pipelines if p.name == name) + report = translate_pipeline(pipeline, _DEFS) + + # Serialize the report BEFORE the in-process write (write_bundle mutates the + # workflow it is given, not the IR, but serialize first to be safe). + report_dict = _pipeline_to_dict(report.pipeline) + + in_process = tmp_path / "in_process" + write_bundle(prepare_workflow(report.pipeline), in_process, catalog="c", schema="s") + + report_path = tmp_path / "report_path" + write_bundle(_pipeline_dict_to_workflow(report_dict), report_path, catalog="c", schema="s") + + assert _jobs(in_process) == _jobs(report_path), ( + f"in-process vs report round-trip bundle diverged for pipeline '{name}'" + ) + + +@pytest.mark.parametrize("name", _PIPELINE_NAMES) +def test_generated_bundle_satisfies_invariants(name: str, tmp_path: Path) -> None: + pipeline = next(p for p in _DEFS.pipelines if p.name == name) + report = translate_pipeline(pipeline, _DEFS) + out = tmp_path / "bundle" + write_bundle(_pipeline_dict_to_workflow(_pipeline_to_dict(report.pipeline)), out, catalog="c", schema="s") + result = check_bundle_dir(out) + assert result.ok, format_result(result) diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 92c2615..3807af1 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -11,13 +11,13 @@ from flowx.adapter import ( CopyActivityParadigm, NonDatabricksTaskCompute, + TranslationConfiguration, TranslationInputRequired, - TranslationPreferences, - TranslationQuestion, + TranslationOption, TranslationSession, UseLakeflowConnectors, - apply_preferences, - gather_questions, + apply_configuration, + gather_options, validate_answer, ) from flowx.adapter.__main__ import main as adapter_cli_main @@ -27,14 +27,14 @@ COMPUTE_MODE_INHERIT, COMPUTE_MODE_SERVERLESS, LAKEFLOW_CONNECT_REPLACEMENT, - QUESTION_COPY_ACTIVITY_PARADIGM, - QUESTION_LAKEFLOW_CONNECTOR_TYPE, - QUESTION_METADATA_DRIVEN_ACCESS, - QUESTION_METADATA_DRIVEN_CONSOLIDATE, - QUESTION_METADATA_DRIVEN_LOOKUP_TOOL, - QUESTION_METADATA_DRIVEN_SIZE, - QUESTION_NON_DATABRICKS_TASK_COMPUTE, - QUESTION_USE_LAKEFLOW_CONNECTORS, + OPTION_COPY_ACTIVITY_PARADIGM, + OPTION_LAKEFLOW_CONNECTOR_TYPE, + OPTION_METADATA_DRIVEN_ACCESS, + OPTION_METADATA_DRIVEN_CONSOLIDATE, + OPTION_METADATA_DRIVEN_LOOKUP_TOOL, + OPTION_METADATA_DRIVEN_SIZE, + OPTION_NON_DATABRICKS_TASK_COMPUTE, + OPTION_USE_LAKEFLOW_CONNECTORS, ) from flowx.adapter.operations import allowed_values_for, enum_for from flowx.models.ir import ( @@ -121,15 +121,15 @@ def _file_copy(name: str = "copy_files") -> CopyActivity: ) -class TestPreferences: - def test_default_preferences_are_conservative(self): - prefs = TranslationPreferences() +class TestConfiguration: + def test_default_configuration_are_conservative(self): + prefs = TranslationConfiguration() assert prefs.copy_activity_paradigm is CopyActivityParadigm.NOTEBOOK assert prefs.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS assert prefs.use_lakeflow_connectors is UseLakeflowConnectors.EXISTING def test_string_values_coerce_to_enums(self): - prefs = TranslationPreferences( + prefs = TranslationConfiguration( copy_activity_paradigm="sdp", non_databricks_task_compute="classic", ) @@ -138,10 +138,10 @@ def test_string_values_coerce_to_enums(self): def test_invalid_value_raises(self): with pytest.raises(ValueError, match="not a valid CopyActivityParadigm"): - TranslationPreferences(copy_activity_paradigm="bogus") + TranslationConfiguration(copy_activity_paradigm="bogus") def test_per_task_override_takes_precedence(self): - base = TranslationPreferences( + base = TranslationConfiguration( copy_activity_paradigm="notebook", per_task={"copy_a": {"copy_activity_paradigm": "sdp"}}, ) @@ -151,7 +151,7 @@ def test_per_task_override_takes_precedence(self): assert other.copy_activity_paradigm is CopyActivityParadigm.NOTEBOOK def test_effective_for_returns_self_when_no_override(self): - prefs = TranslationPreferences() + prefs = TranslationConfiguration() assert prefs.effective_for("missing") is prefs def test_enum_for_and_allowed_values_for(self): @@ -161,37 +161,37 @@ def test_enum_for_and_allowed_values_for(self): assert allowed_values_for("unknown") == () -class TestGatherQuestions: - def test_no_questions_for_empty_pipeline(self): +class TestGatherOptions: + def test_no_options_for_empty_pipeline(self): pipeline = Pipeline(name="empty", tasks=[]) - pending = gather_questions(pipeline) + pending = gather_options(pipeline) assert pending.pipeline_name == "empty" - assert pending.questions == [] + assert pending.options == [] - def test_copy_paradigm_question_only_when_delta_sink_present(self): + def test_copy_paradigm_option_only_when_delta_sink_present(self): delta_pipeline = Pipeline(name="p", tasks=[_delta_copy()]) - question_ids = {q.question_id for q in gather_questions(delta_pipeline).questions} - assert QUESTION_COPY_ACTIVITY_PARADIGM in question_ids + option_ids = {q.option_id for q in gather_options(delta_pipeline).options} + assert OPTION_COPY_ACTIVITY_PARADIGM in option_ids non_delta = Pipeline(name="p", tasks=[_file_copy()]) - question_ids = {q.question_id for q in gather_questions(non_delta).questions} - assert QUESTION_COPY_ACTIVITY_PARADIGM not in question_ids + option_ids = {q.option_id for q in gather_options(non_delta).options} + assert OPTION_COPY_ACTIVITY_PARADIGM not in option_ids - def test_non_databricks_compute_question_when_any_non_db_task(self): + def test_non_databricks_compute_option_when_any_non_db_task(self): pipeline = Pipeline(name="p", tasks=[WaitActivity(**_make_base("w"), wait_time_seconds=1)]) - ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_NON_DATABRICKS_TASK_COMPUTE in ids + ids = {q.option_id for q in gather_options(pipeline).options} + assert OPTION_NON_DATABRICKS_TASK_COMPUTE in ids - def test_lakeflow_connect_question_only_for_db_to_delta(self): + def test_lakeflow_connect_option_only_for_db_to_delta(self): with_db = Pipeline(name="p", tasks=[_delta_copy()]) - ids = {q.question_id for q in gather_questions(with_db).questions} - assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids + ids = {q.option_id for q in gather_options(with_db).options} + assert OPTION_USE_LAKEFLOW_CONNECTORS in ids without_db = Pipeline(name="p", tasks=[_file_copy()]) - ids = {q.question_id for q in gather_questions(without_db).questions} - assert QUESTION_USE_LAKEFLOW_CONNECTORS not in ids + ids = {q.option_id for q in gather_options(without_db).options} + assert OPTION_USE_LAKEFLOW_CONNECTORS not in ids - def test_lakeflow_connect_question_surfaces_for_database_motif_without_detected_motifs(self): + def test_lakeflow_connect_option_surfaces_for_database_motif_without_detected_motifs(self): """CLI callers don't have DetectedMotif objects; eligibility should derive from the IR alone.""" motif_activity = MotifActivity( **_make_base("motif_incremental_load_watermark", "motif_incremental_load_watermark"), @@ -202,13 +202,13 @@ def test_lakeflow_connect_question_surfaces_for_database_motif_without_detected_ source_type_hint="database", ) pipeline = Pipeline(name="p", tasks=[motif_activity]) - pending = gather_questions(pipeline) - ids = {q.question_id for q in pending.questions} - assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids - question = next(q for q in pending.questions if q.question_id == QUESTION_USE_LAKEFLOW_CONNECTORS) - assert "motif_incremental_load_watermark" in question.affected_task_keys + pending = gather_options(pipeline) + ids = {q.option_id for q in pending.options} + assert OPTION_USE_LAKEFLOW_CONNECTORS in ids + option = next(q for q in pending.options if q.option_id == OPTION_USE_LAKEFLOW_CONNECTORS) + assert "motif_incremental_load_watermark" in option.affected_task_keys - def test_lakeflow_connect_question_surfaces_for_database_motif(self): + def test_lakeflow_connect_option_surfaces_for_database_motif(self): motif_activity = MotifActivity( **_make_base("motif_incremental_load_watermark", "motif_incremental_load_watermark"), motif_id="incremental_load_watermark", @@ -226,35 +226,35 @@ def test_lakeflow_connect_question_surfaces_for_database_motif(self): confidence_notes=[], ) ] - pending = gather_questions(pipeline, motifs) - ids = {q.question_id for q in pending.questions} - assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids - lfc_question = next(q for q in pending.questions if q.question_id == QUESTION_USE_LAKEFLOW_CONNECTORS) - assert "motif_incremental_load_watermark" in lfc_question.affected_task_keys - - def test_no_databricks_task_compute_question_for_notebook(self): - """The serverless-replacement question for Databricks tasks was removed.""" + pending = gather_options(pipeline, motifs) + ids = {q.option_id for q in pending.options} + assert OPTION_USE_LAKEFLOW_CONNECTORS in ids + lfc_option = next(q for q in pending.options if q.option_id == OPTION_USE_LAKEFLOW_CONNECTORS) + assert "motif_incremental_load_watermark" in lfc_option.affected_task_keys + + def test_no_databricks_task_compute_option_for_notebook(self): + """The serverless-replacement option for Databricks tasks was removed.""" pipeline = Pipeline( name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")], ) - ids = {q.question_id for q in gather_questions(pipeline).questions} + ids = {q.option_id for q in gather_options(pipeline).options} assert "databricks_task_compute" not in ids - def test_no_databricks_task_compute_question_for_spark_python(self): - """The serverless-replacement question for Databricks tasks was removed.""" + def test_no_databricks_task_compute_option_for_spark_python(self): + """The serverless-replacement option for Databricks tasks was removed.""" pipeline = Pipeline( name="p", tasks=[SparkPythonActivity(**_make_base("py"), python_file="dbfs:/scripts/etl.py")], ) - ids = {q.question_id for q in gather_questions(pipeline).questions} + ids = {q.option_id for q in gather_options(pipeline).options} assert "databricks_task_compute" not in ids def test_already_answered_filters_pending(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) - pending = gather_questions(pipeline, answers={QUESTION_COPY_ACTIVITY_PARADIGM: "sdp"}) - ids = {q.question_id for q in pending.questions} - assert QUESTION_COPY_ACTIVITY_PARADIGM not in ids + pending = gather_options(pipeline, answers={OPTION_COPY_ACTIVITY_PARADIGM: "sdp"}) + ids = {q.option_id for q in pending.options} + assert OPTION_COPY_ACTIVITY_PARADIGM not in ids def test_walks_into_for_each_inner_activities(self): inner_copy = _delta_copy("inner_copy") @@ -264,15 +264,15 @@ def test_walks_into_for_each_inner_activities(self): inner_activities=[inner_copy], ) pipeline = Pipeline(name="p", tasks=[for_each]) - question = next( - (q for q in gather_questions(pipeline).questions if q.question_id == QUESTION_COPY_ACTIVITY_PARADIGM), + option = next( + (q for q in gather_options(pipeline).options if q.option_id == OPTION_COPY_ACTIVITY_PARADIGM), None, ) - assert question is not None - assert "inner_copy" in question.affected_task_keys + assert option is not None + assert "inner_copy" in option.affected_task_keys - def test_motif_consolidation_question_emitted_per_detected_motif(self): - """Each detected motif produces a ``consolidate_motif:`` question.""" + def test_motif_consolidation_option_emitted_per_detected_motif(self): + """Each detected motif produces a ``consolidate_motif:`` option.""" pipeline = Pipeline(name="p", tasks=[_delta_copy()]) motifs = [ DetectedMotif( @@ -282,20 +282,18 @@ def test_motif_consolidation_question_emitted_per_detected_motif(self): confidence_notes=["Detector matched Lookup→Copy→SP chain"], ) ] - pending = gather_questions(pipeline, motifs) - ids = {q.question_id for q in pending.questions} + pending = gather_options(pipeline, motifs) + ids = {q.option_id for q in pending.options} assert "consolidate_motif:incremental_load_watermark" in ids - motif_question = next( - q for q in pending.questions if q.question_id == "consolidate_motif:incremental_load_watermark" - ) - assert motif_question.default == "keep" - assert {opt.value for opt in motif_question.options} == {"keep", "consolidate"} - assert "WatermarkLookup" in motif_question.affected_task_keys + motif_option = next(q for q in pending.options if q.option_id == "consolidate_motif:incremental_load_watermark") + assert motif_option.default == "keep" + assert {opt.value for opt in motif_option.options} == {"keep", "consolidate"} + assert "WatermarkLookup" in motif_option.affected_task_keys # Confidence note must surface in the rationale so the agent can quote it - assert "Detector matched Lookup→Copy→SP chain" in motif_question.rationale + assert "Detector matched Lookup→Copy→SP chain" in motif_option.rationale - def test_motif_consolidation_question_filtered_by_answer(self): - """Once answered the per-motif question must drop out of pending.""" + def test_motif_consolidation_option_filtered_by_answer(self): + """Once answered the per-motif option must drop out of pending.""" pipeline = Pipeline(name="p", tasks=[_delta_copy()]) motifs = [ DetectedMotif( @@ -305,12 +303,12 @@ def test_motif_consolidation_question_filtered_by_answer(self): confidence_notes=[], ) ] - pending = gather_questions( + pending = gather_options( pipeline, motifs, answers={"consolidate_motif:incremental_load_watermark": "consolidate"}, ) - ids = {q.question_id for q in pending.questions} + ids = {q.option_id for q in pending.options} assert "consolidate_motif:incremental_load_watermark" not in ids def test_motif_consolidation_validate_answer_accepts_keep_or_consolidate(self): @@ -326,19 +324,19 @@ class TestValidateAnswer: def test_accepts_allowed_value(self): assert validate_answer("copy_activity_paradigm", "sdp") == "sdp" - def test_rejects_unknown_question(self): - with pytest.raises(ValueError, match="Unknown question_id"): - validate_answer("not_a_question", "x") + def test_rejects_unknown_option(self): + with pytest.raises(ValueError, match="Unknown option_id"): + validate_answer("not_a_option", "x") def test_rejects_invalid_value(self): with pytest.raises(ValueError, match="Invalid answer"): validate_answer("copy_activity_paradigm", "yaml") -class TestApplyPreferences: +class TestApplyConfiguration: def test_serverless_default_leaves_activities_on_serverless_compute(self): pipeline = Pipeline(name="p", tasks=[_delta_copy(), WaitActivity(**_make_base("w"), wait_time_seconds=1)]) - modified = apply_preferences(pipeline, TranslationPreferences()) + modified = apply_configuration(pipeline, TranslationConfiguration()) copy_task = modified.tasks[0] wait_task = modified.tasks[1] assert copy_task.compute_mode == COMPUTE_MODE_SERVERLESS @@ -346,8 +344,8 @@ def test_serverless_default_leaves_activities_on_serverless_compute(self): def test_classic_compute_routes_copy_to_multi_node_cluster(self): pipeline = Pipeline(name="p", tasks=[_delta_copy(), WaitActivity(**_make_base("w"), wait_time_seconds=1)]) - prefs = TranslationPreferences(non_databricks_task_compute="classic") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(non_databricks_task_compute="classic") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].compute_mode == COMPUTE_MODE_CLASSIC_MULTI_NODE assert modified.tasks[1].compute_mode == COMPUTE_MODE_CLASSIC_SINGLE_NODE @@ -355,7 +353,7 @@ def test_databricks_task_always_inherits_linked_service_cluster(self): """DatabricksNotebook activities always inherit the source linked-service cluster binding; the serverless replacement option was removed.""" pipeline = Pipeline(name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")]) - modified = apply_preferences(pipeline, TranslationPreferences()) + modified = apply_configuration(pipeline, TranslationConfiguration()) assert modified.tasks[0].compute_mode == COMPUTE_MODE_INHERIT def test_spark_python_always_inherits_linked_service_cluster(self): @@ -364,31 +362,31 @@ def test_spark_python_always_inherits_linked_service_cluster(self): pipeline = Pipeline( name="p", tasks=[SparkPythonActivity(**_make_base("py"), python_file="dbfs:/scripts/etl.py")] ) - modified = apply_preferences(pipeline, TranslationPreferences()) + modified = apply_configuration(pipeline, TranslationConfiguration()) assert modified.tasks[0].compute_mode == COMPUTE_MODE_INHERIT def test_copy_paradigm_sdp_stamps_target_format(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) - prefs = TranslationPreferences(copy_activity_paradigm="sdp") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(copy_activity_paradigm="sdp") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].target_format == "sdp" def test_copy_paradigm_does_not_apply_to_non_delta_copy(self): pipeline = Pipeline(name="p", tasks=[_file_copy()]) - prefs = TranslationPreferences(copy_activity_paradigm="sdp") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(copy_activity_paradigm="sdp") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].target_format == "notebook" def test_lakeflow_connect_flag_set_for_eligible_copy(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].use_lakeflow_connector is True def test_lakeflow_connect_skipped_for_non_database_copy(self): pipeline = Pipeline(name="p", tasks=[_file_copy()]) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].use_lakeflow_connector is False def test_motif_replacement_swapped_for_lakeflow_connect_when_database(self): @@ -401,8 +399,8 @@ def test_motif_replacement_swapped_for_lakeflow_connect_when_database(self): source_type_hint="database", ) pipeline = Pipeline(name="p", tasks=[motif]) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].databricks_replacement == LAKEFLOW_CONNECT_REPLACEMENT def test_motif_replacement_unchanged_for_file_source(self): @@ -415,31 +413,31 @@ def test_motif_replacement_unchanged_for_file_source(self): source_type_hint="files", ) pipeline = Pipeline(name="p", tasks=[motif]) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].databricks_replacement == "auto_loader_file_notification" def test_per_task_override_wins(self): pipeline = Pipeline(name="p", tasks=[_delta_copy("c1"), _delta_copy("c2")]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( copy_activity_paradigm="notebook", per_task={"c1": {"copy_activity_paradigm": "sdp"}}, ) - modified = apply_preferences(pipeline, prefs) + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].target_format == "sdp" assert modified.tasks[1].target_format == "notebook" - def test_preferences_attached_to_pipeline(self): + def test_configuration_attached_to_pipeline(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) - prefs = TranslationPreferences(copy_activity_paradigm="sdp") - modified = apply_preferences(pipeline, prefs) - assert modified.translation_preferences is prefs + prefs = TranslationConfiguration(copy_activity_paradigm="sdp") + modified = apply_configuration(pipeline, prefs) + assert modified.translation_configuration is prefs - def test_apply_preferences_does_not_mutate_input(self): + def test_apply_configuration_does_not_mutate_input(self): original = Pipeline(name="p", tasks=[_delta_copy()]) - apply_preferences(original, TranslationPreferences(copy_activity_paradigm="sdp")) + apply_configuration(original, TranslationConfiguration(copy_activity_paradigm="sdp")) assert original.tasks[0].target_format is None - assert original.translation_preferences is None + assert original.translation_configuration is None def test_recurses_into_for_each_inner_activities(self): inner = _delta_copy("inner") @@ -449,74 +447,74 @@ def test_recurses_into_for_each_inner_activities(self): inner_activities=[inner], ) pipeline = Pipeline(name="p", tasks=[for_each]) - prefs = TranslationPreferences(copy_activity_paradigm="sdp") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(copy_activity_paradigm="sdp") + modified = apply_configuration(pipeline, prefs) inner_after = modified.tasks[0].inner_activities[0] assert inner_after.target_format == "sdp" class TestTranslationSession: - def test_pending_returns_only_outstanding_questions(self): + def test_pending_returns_only_outstanding_options(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) session = TranslationSession(pipeline=pipeline) first = session.pending() - assert len(first.questions) > 0 - session.answer(QUESTION_COPY_ACTIVITY_PARADIGM, "sdp") - ids_after = {q.question_id for q in session.pending().questions} - assert QUESTION_COPY_ACTIVITY_PARADIGM not in ids_after + assert len(first.options) > 0 + session.answer(OPTION_COPY_ACTIVITY_PARADIGM, "sdp") + ids_after = {q.option_id for q in session.pending().options} + assert OPTION_COPY_ACTIVITY_PARADIGM not in ids_after - def test_run_raises_when_questions_outstanding(self): + def test_run_raises_when_options_outstanding(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) session = TranslationSession(pipeline=pipeline) with pytest.raises(TranslationInputRequired) as info: session.run() assert info.value.pending.pipeline_name == "p" - assert any(q.question_id == QUESTION_COPY_ACTIVITY_PARADIGM for q in info.value.pending.questions) + assert any(q.option_id == OPTION_COPY_ACTIVITY_PARADIGM for q in info.value.pending.options) def test_run_returns_modified_pipeline_when_complete(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) session = TranslationSession(pipeline=pipeline) pending = session.pending() - answers = {q.question_id: q.default for q in pending.questions} + answers = {q.option_id: q.default for q in pending.options} session.answer_many(answers) modified = session.run() - assert modified.translation_preferences is not None + assert modified.translation_configuration is not None def test_answer_validates(self): session = TranslationSession(pipeline=Pipeline(name="p", tasks=[_delta_copy()])) with pytest.raises(ValueError): - session.answer(QUESTION_COPY_ACTIVITY_PARADIGM, "yaml") + session.answer(OPTION_COPY_ACTIVITY_PARADIGM, "yaml") def test_answer_many_is_atomic(self): session = TranslationSession(pipeline=Pipeline(name="p", tasks=[_delta_copy()])) with pytest.raises(ValueError): - session.answer_many({QUESTION_COPY_ACTIVITY_PARADIGM: "sdp", "bogus": "x"}) - assert QUESTION_COPY_ACTIVITY_PARADIGM not in session._answers + session.answer_many({OPTION_COPY_ACTIVITY_PARADIGM: "sdp", "bogus": "x"}) + assert OPTION_COPY_ACTIVITY_PARADIGM not in session._answers - def test_find_question_returns_pending_question(self): + def test_find_option_returns_pending_option(self): session = TranslationSession(pipeline=Pipeline(name="p", tasks=[_delta_copy()])) - found = session.find_question(QUESTION_COPY_ACTIVITY_PARADIGM) - assert isinstance(found, TranslationQuestion) - session.answer(QUESTION_COPY_ACTIVITY_PARADIGM, "sdp") - assert session.find_question(QUESTION_COPY_ACTIVITY_PARADIGM) is None + found = session.find_option(OPTION_COPY_ACTIVITY_PARADIGM) + assert isinstance(found, TranslationOption) + session.answer(OPTION_COPY_ACTIVITY_PARADIGM, "sdp") + assert session.find_option(OPTION_COPY_ACTIVITY_PARADIGM) is None class TestSerializationRoundtrip: - def test_preferences_survive_json_roundtrip(self): + def test_configuration_survive_json_roundtrip(self): from flowx.bundler.dab_writer import pipeline_dict_to_ir from flowx.translator.engine import _pipeline_to_dict pipeline = Pipeline( name="p", tasks=[_delta_copy(), NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")] ) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( copy_activity_paradigm="sdp", non_databricks_task_compute="classic", use_lakeflow_connectors="lakeflow_connect", ) - stamped = apply_preferences(pipeline, prefs) + stamped = apply_configuration(pipeline, prefs) roundtripped, _ = pipeline_dict_to_ir(json.loads(json.dumps(_pipeline_to_dict(stamped), default=str))) - assert roundtripped.translation_preferences.copy_activity_paradigm is CopyActivityParadigm.SDP + assert roundtripped.translation_configuration.copy_activity_paradigm is CopyActivityParadigm.SDP assert roundtripped.tasks[0].target_format == "sdp" assert roundtripped.tasks[0].use_lakeflow_connector is True assert roundtripped.tasks[0].compute_mode == COMPUTE_MODE_CLASSIC_MULTI_NODE @@ -526,26 +524,26 @@ def test_preferences_survive_json_roundtrip(self): class TestMigrationInputSession: - def test_ingest_session_lists_expected_questions(self): + def test_discover_session_lists_expected_options(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="ingest") - ids = [q.question_id for q in session.pending().questions] + session = MigrationInputSession(phase="discover") + ids = [q.option_id for q in session.pending().options] assert ids == ["adf_source_path", "adf_resource_url", "output_dir"] - def test_translate_session_lists_expected_questions(self): + def test_convert_session_lists_expected_options(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="translate") - ids = [q.question_id for q in session.pending().questions] + session = MigrationInputSession(phase="convert") + ids = [q.option_id for q in session.pending().options] assert "inventory_path" in ids assert "adf_source_path" in ids - def test_prepare_session_lists_expected_questions(self): + def test_package_session_lists_expected_options(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="prepare") - ids = {q.question_id for q in session.pending().questions} + session = MigrationInputSession(phase="package") + ids = {q.option_id for q in session.pending().options} assert {"translation_report_path", "output_bundle_path", "catalog", "schema"} <= ids def test_unknown_phase_raises(self): @@ -557,22 +555,22 @@ def test_unknown_phase_raises(self): def test_answer_records_value_and_drops_from_pending(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="ingest") + session = MigrationInputSession(phase="discover") session.answer("adf_source_path", "/Volumes/main/default/adf") - ids = [q.question_id for q in session.pending().questions] + ids = [q.option_id for q in session.pending().options] assert "adf_source_path" not in ids - def test_answer_rejects_unknown_question(self): + def test_answer_rejects_unknown_option(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="ingest") - with pytest.raises(ValueError, match="Unknown input question"): + session = MigrationInputSession(phase="discover") + with pytest.raises(ValueError, match="Unknown input option"): session.answer("not_a_field", "x") def test_collected_merges_answers_with_defaults(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="prepare") + session = MigrationInputSession(phase="package") session.answer("translation_report_path", "/tmp/report.json") collected = session.collected() assert collected["translation_report_path"] == "/tmp/report.json" @@ -582,10 +580,10 @@ def test_collected_merges_answers_with_defaults(self): def test_collected_omits_required_when_missing(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="ingest") + session = MigrationInputSession(phase="discover") collected = session.collected() assert "adf_source_path" not in collected - assert collected["output_dir"] == "./orchestra_output/ingest" + assert collected["output_dir"] == "./flowx_output" class TestWorkspacePathsCli: @@ -650,26 +648,26 @@ def test_workspace_paths_suggests_host_from_databricks_linked_service(self, tmp_ class TestInputsCli: - def test_inputs_emits_ingest_questions(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): - exit_code = adapter_cli_main(["inputs", "ingest"]) + def test_inputs_emits_discover_options(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): + exit_code = adapter_cli_main(["inputs", "discover"]) assert exit_code == 0 payload = json.loads(capsys.readouterr().out) - assert payload["phase"] == "ingest" - ids = [q["question_id"] for q in payload["questions"]] + assert payload["phase"] == "discover" + ids = [q["option_id"] for q in payload["options"]] assert ids == ["adf_source_path", "adf_resource_url", "output_dir"] def test_inputs_writes_to_file(self, tmp_path: Path): - out = tmp_path / "questions.json" - exit_code = adapter_cli_main(["inputs", "prepare", "--out", str(out)]) + out = tmp_path / "options.json" + exit_code = adapter_cli_main(["inputs", "package", "--out", str(out)]) assert exit_code == 0 payload = json.loads(out.read_text()) - assert payload["phase"] == "prepare" - ids = {q["question_id"] for q in payload["questions"]} + assert payload["phase"] == "package" + ids = {q["option_id"] for q in payload["options"]} assert "output_bundle_path" in ids class TestCli: - def test_inspect_emits_pending_questions(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): + def test_inspect_emits_pending_options(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): from flowx.translator.engine import _pipeline_to_dict pipeline = Pipeline(name="p", tasks=[_delta_copy()]) @@ -679,30 +677,38 @@ def test_inspect_emits_pending_questions(self, tmp_path: Path, capsys: pytest.Ca assert exit_code == 0 payload = json.loads(capsys.readouterr().out) assert payload["pipelines"][0]["pipeline_name"] == "p" - question_ids = {q["question_id"] for q in payload["pipelines"][0]["questions"]} - assert QUESTION_COPY_ACTIVITY_PARADIGM in question_ids + option_ids = {q["option_id"] for q in payload["pipelines"][0]["options"]} + assert OPTION_COPY_ACTIVITY_PARADIGM in option_ids - def test_modify_stamps_preferences(self, tmp_path: Path): + def test_modify_stamps_configuration(self, tmp_path: Path): from flowx.translator.engine import _pipeline_to_dict pipeline = Pipeline(name="p", tasks=[_delta_copy()]) report_path = tmp_path / "report.json" report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) - answers_path = tmp_path / "answers.json" - answers_path.write_text( - json.dumps( - { - "copy_activity_paradigm": "sdp", - "non_databricks_task_compute": "classic", - "use_lakeflow_connectors": "lakeflow_connect", - } - ) - ) out_path = tmp_path / "modified.json" - exit_code = adapter_cli_main(["modify", str(report_path), str(answers_path), "--out", str(out_path)]) + exit_code = adapter_cli_main( + [ + "modify", + str(report_path), + "--answer", + "copy_activity_paradigm=sdp", + "--answer", + "non_databricks_task_compute=classic", + "--answer", + "use_lakeflow_connectors=lakeflow_connect", + "--out", + str(out_path), + "--config-out", + str(tmp_path / "configuration.json"), + ] + ) assert exit_code == 0 modified = json.loads(out_path.read_text()) - assert modified["translation_preferences"]["copy_activity_paradigm"] == "sdp" + # The collected answers are persisted verbatim as configuration.json. + config = json.loads((tmp_path / "configuration.json").read_text()) + assert config["copy_activity_paradigm"] == "sdp" + assert modified["translation_configuration"]["copy_activity_paradigm"] == "sdp" copy_task = next(task for task in modified["tasks"] if task["task_key"] == "copy_to_delta") assert copy_task["target_format"] == "sdp" assert copy_task["compute_mode"] == COMPUTE_MODE_CLASSIC_MULTI_NODE @@ -735,28 +741,23 @@ def test_modify_threads_lookup_values_into_metadata_driven_motif(self, tmp_path: pipeline = Pipeline(name="p", tasks=[motif]) report_path = tmp_path / "report.json" report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) - answers_path = tmp_path / "answers.json" - answers_path.write_text( - json.dumps( - { - "metadata_driven_consolidate": "consolidate", - "metadata_driven_access": "yes", - "metadata_driven_size": "small", - } - ) - ) - lookup_values_path = tmp_path / "lookup_values.json" - lookup_values_path.write_text(json.dumps([{"source_table": "orders"}])) out_path = tmp_path / "modified.json" exit_code = adapter_cli_main( [ "modify", str(report_path), - str(answers_path), - "--lookup-values", - str(lookup_values_path), + "--answer", + "metadata_driven_consolidate=consolidate", + "--answer", + "metadata_driven_access=yes", + "--answer", + "metadata_driven_size=small", + "--lookup-csv", + "source_table\norders", "--out", str(out_path), + "--config-out", + str(tmp_path / "configuration.json"), ] ) assert exit_code == 0 @@ -771,12 +772,112 @@ def test_modify_rejects_invalid_answer(self, tmp_path: Path): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) report_path = tmp_path / "report.json" report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) - answers_path = tmp_path / "answers.json" - answers_path.write_text(json.dumps({"copy_activity_paradigm": "yaml"})) out_path = tmp_path / "modified.json" - exit_code = adapter_cli_main(["modify", str(report_path), str(answers_path), "--out", str(out_path)]) + exit_code = adapter_cli_main( + ["modify", str(report_path), "--answer", "copy_activity_paradigm=yaml", "--out", str(out_path)] + ) + assert exit_code == 2 + + def test_modify_output_dir_convention_writes_work_and_metadata(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + report_path = tmp_path / ".work" / "translation_report.json" + report_path.parent.mkdir(parents=True) + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + exit_code = adapter_cli_main( + ["modify", str(report_path), "--output-dir", str(tmp_path), "--answer", "copy_activity_paradigm=sdp"] + ) + assert exit_code == 0 + # Stamped IR lands in the transient .work/, configuration.json in metadata/. + assert (tmp_path / ".work" / "translation_report.stamped.json").exists() + config = json.loads((tmp_path / "metadata" / "configuration.json").read_text()) + assert config == {"copy_activity_paradigm": "sdp"} + + def test_modify_requires_output_dir_or_out(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + exit_code = adapter_cli_main(["modify", str(report_path), "--answer", "copy_activity_paradigm=sdp"]) assert exit_code == 2 + def test_inspect_emits_full_schema_with_show_when(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): + """inspect returns the whole option tree at once; follow-ups carry a show_when condition + the agent evaluates locally (no per-follow-up round trip).""" + from flowx.models.ir import CopyActivity, Dependency, WebActivity + from flowx.translator.engine import _pipeline_to_dict + + copy = CopyActivity(name="Load", task_key="load") + notify = WebActivity( + name="Notify", + task_key="notify", + url="https://x", + method="POST", + depends_on=[Dependency(task_key="load", outcome="Failed")], + ) + pipeline = Pipeline(name="p", tasks=[copy, notify]) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + + assert adapter_cli_main(["inspect", str(report_path)]) == 0 + options = {o["option_id"]: o for o in json.loads(capsys.readouterr().out)["pipelines"][0]["options"]} + + # The full chain is present up front, not gated behind an answer. + assert "notify_destination" in options + assert "notify_email_recipients" in options + # The destination question is unconditional; the email follow-up is gated by show_when. + assert options["notify_destination"]["show_when"] == [] + assert options["notify_email_recipients"]["show_when"] == [{"option_id": "notify_destination", "in": ["email"]}] + # Free-text follow-up vs. enum option. + assert options["notify_email_recipients"]["free_text"] is True + assert [c["value"] for c in options["notify_destination"]["choices"]][0] == "keep" + + def test_inspect_rejects_malformed_answer(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + # Missing '=' -> validation error -> exit 2. + assert adapter_cli_main(["inspect", str(report_path), "--answer", "no_equals_sign"]) == 2 + + def test_record_results_subcommand(self, tmp_path: Path, monkeypatch, capsys: pytest.CaptureFixture[str]): + import flowx.reporting.results as rr + + md = tmp_path / "metadata" + md.mkdir() + (md / "inventory.json").write_text("{}") + monkeypatch.setattr(rr, "write_results", lambda *a, **k: ("run-xyz", 3)) + rc = adapter_cli_main(["record-results", "--output-dir", str(tmp_path), "--results-table", "c.s.t"]) + assert rc == 0 + out = capsys.readouterr().out + assert "run-xyz" in out and "3 pipeline" in out + + def test_record_results_requires_inventory(self, tmp_path: Path): + rc = adapter_cli_main(["record-results", "--output-dir", str(tmp_path), "--results-table", "c.s.t"]) + assert rc == 1 # no metadata/inventory.json + + def test_install_dashboard_subcommand(self, monkeypatch, capsys: pytest.CaptureFixture[str]): + import flowx.reporting.dashboard as dd + + monkeypatch.setattr(dd, "install_dashboard", lambda *a, **k: ("dash-1", "https://x/sql/dashboardsv3/dash-1")) + rc = adapter_cli_main(["install-dashboard", "--results-table", "c.s.t"]) + assert rc == 0 + out = capsys.readouterr().out + assert "dash-1" in out + + def test_install_dashboard_failure_returns_1(self, monkeypatch): + import flowx.reporting.dashboard as dd + + def _boom(*a, **k): + raise RuntimeError("no auth") + + monkeypatch.setattr(dd, "install_dashboard", _boom) + rc = adapter_cli_main(["install-dashboard", "--results-table", "c.s.t"]) + assert rc == 1 + class TestBundleOutput: def test_classic_copy_compute_emits_two_node_multi_node_cluster(self, tmp_path: Path): @@ -786,8 +887,8 @@ def test_classic_copy_compute_emits_two_node_multi_node_cluster(self, tmp_path: from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - prefs = TranslationPreferences(non_databricks_task_compute="classic") - stamped = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(non_databricks_task_compute="classic") + stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) @@ -805,8 +906,8 @@ def test_classic_single_node_cluster_uses_is_single_node_flag(self, tmp_path: Pa from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[WaitActivity(**_make_base("w"), wait_time_seconds=1)]) - prefs = TranslationPreferences(non_databricks_task_compute="classic") - stamped = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(non_databricks_task_compute="classic") + stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) @@ -825,7 +926,7 @@ def test_serverless_default_emits_no_job_clusters(self, tmp_path: Path): from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - stamped = apply_preferences(pipeline, TranslationPreferences()) + stamped = apply_configuration(pipeline, TranslationConfiguration()) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) @@ -836,7 +937,7 @@ def test_sdp_copy_emits_pyspark_pipelines_table_scaffold(self, tmp_path: Path): from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - stamped = apply_preferences(pipeline, TranslationPreferences(copy_activity_paradigm="sdp")) + stamped = apply_configuration(pipeline, TranslationConfiguration(copy_activity_paradigm="sdp")) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) notebook_path = tmp_path / "src" / "notebooks" / "copy_a.py" @@ -852,7 +953,7 @@ def test_lakeflow_connect_emits_pipeline_resource_and_no_notebook(self, tmp_path from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - stamped = apply_preferences(pipeline, TranslationPreferences(use_lakeflow_connectors="lakeflow_connect")) + stamped = apply_configuration(pipeline, TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect")) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) assert not (tmp_path / "src" / "notebooks" / "copy_a.py").exists() @@ -861,7 +962,7 @@ def test_lakeflow_connect_emits_pipeline_resource_and_no_notebook(self, tmp_path resource = yaml.safe_load(pipeline_yml.read_text()) lfc = resource["resources"]["pipelines"]["copy_a_lfc"] assert lfc["name"] == "copy_a_lfc" - assert lfc["ingestion_definition"]["connection_name"] == "orchestra_copy_a_connection" + assert lfc["ingestion_definition"]["connection_name"] == "flowx_copy_a_connection" objects = lfc["ingestion_definition"]["objects"] assert objects[0]["table"]["destination_table"] == "raw.events" @@ -872,7 +973,7 @@ def test_lakeflow_connect_job_task_references_pipeline(self, tmp_path: Path): from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - stamped = apply_preferences(pipeline, TranslationPreferences(use_lakeflow_connectors="lakeflow_connect")) + stamped = apply_configuration(pipeline, TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect")) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) @@ -880,79 +981,77 @@ def test_lakeflow_connect_job_task_references_pipeline(self, tmp_path: Path): assert "notebook_task" not in task assert task["pipeline_task"]["pipeline_id"] == "${resources.pipelines.copy_a_lfc.id}" - def test_metadata_driven_consolidate_question_surfaces_for_motif(self): + def test_metadata_driven_consolidate_option_surfaces_for_motif(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) - ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_METADATA_DRIVEN_CONSOLIDATE in ids + ids = {q.option_id for q in gather_options(pipeline).options} + assert OPTION_METADATA_DRIVEN_CONSOLIDATE in ids - def test_metadata_driven_followup_questions_gated_on_consolidate(self): + def test_metadata_driven_followup_options_gated_on_consolidate(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) - first_pass = gather_questions(pipeline).questions - ids = {q.question_id for q in first_pass} - assert QUESTION_METADATA_DRIVEN_CONSOLIDATE in ids - assert QUESTION_METADATA_DRIVEN_ACCESS not in ids - assert QUESTION_METADATA_DRIVEN_SIZE not in ids - assert QUESTION_METADATA_DRIVEN_LOOKUP_TOOL not in ids - - keep_pass = gather_questions(pipeline, answers={QUESTION_METADATA_DRIVEN_CONSOLIDATE: "keep"}).questions - keep_ids = {q.question_id for q in keep_pass} - assert QUESTION_METADATA_DRIVEN_ACCESS not in keep_ids - - consolidate_pass = gather_questions( - pipeline, answers={QUESTION_METADATA_DRIVEN_CONSOLIDATE: "consolidate"} - ).questions - consolidate_ids = {q.question_id for q in consolidate_pass} - assert QUESTION_METADATA_DRIVEN_ACCESS in consolidate_ids - assert QUESTION_METADATA_DRIVEN_SIZE in consolidate_ids - assert QUESTION_METADATA_DRIVEN_LOOKUP_TOOL not in consolidate_ids - - def test_metadata_driven_lookup_tool_question_gated_on_access(self): + first_pass = gather_options(pipeline).options + ids = {q.option_id for q in first_pass} + assert OPTION_METADATA_DRIVEN_CONSOLIDATE in ids + assert OPTION_METADATA_DRIVEN_ACCESS not in ids + assert OPTION_METADATA_DRIVEN_SIZE not in ids + assert OPTION_METADATA_DRIVEN_LOOKUP_TOOL not in ids + + keep_pass = gather_options(pipeline, answers={OPTION_METADATA_DRIVEN_CONSOLIDATE: "keep"}).options + keep_ids = {q.option_id for q in keep_pass} + assert OPTION_METADATA_DRIVEN_ACCESS not in keep_ids + + consolidate_pass = gather_options(pipeline, answers={OPTION_METADATA_DRIVEN_CONSOLIDATE: "consolidate"}).options + consolidate_ids = {q.option_id for q in consolidate_pass} + assert OPTION_METADATA_DRIVEN_ACCESS in consolidate_ids + assert OPTION_METADATA_DRIVEN_SIZE in consolidate_ids + assert OPTION_METADATA_DRIVEN_LOOKUP_TOOL not in consolidate_ids + + def test_metadata_driven_lookup_tool_option_gated_on_access(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) answers = { - QUESTION_METADATA_DRIVEN_CONSOLIDATE: "consolidate", - QUESTION_METADATA_DRIVEN_ACCESS: "yes", + OPTION_METADATA_DRIVEN_CONSOLIDATE: "consolidate", + OPTION_METADATA_DRIVEN_ACCESS: "yes", } - pending = gather_questions(pipeline, answers=answers).questions - ids = {q.question_id for q in pending} - assert QUESTION_METADATA_DRIVEN_LOOKUP_TOOL in ids + pending = gather_options(pipeline, answers=answers).options + ids = {q.option_id for q in pending} + assert OPTION_METADATA_DRIVEN_LOOKUP_TOOL in ids def test_modifier_consolidates_metadata_driven_when_size_is_small(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( metadata_driven_consolidate="consolidate", metadata_driven_access="yes", metadata_driven_size="small", ) - modified = apply_preferences(pipeline, prefs) + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].consolidate_metadata_driven is True def test_modifier_does_not_consolidate_when_size_is_large(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( metadata_driven_consolidate="consolidate", metadata_driven_access="yes", metadata_driven_size="large", ) - modified = apply_preferences(pipeline, prefs) + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].consolidate_metadata_driven is False def test_modifier_does_not_consolidate_when_access_is_no(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( metadata_driven_consolidate="consolidate", metadata_driven_access="no", metadata_driven_size="small", ) - modified = apply_preferences(pipeline, prefs) + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].consolidate_metadata_driven is False - def test_lakeflow_connector_type_question_suppressed_when_only_query_copies(self): + def test_lakeflow_connector_type_option_suppressed_when_only_query_copies(self): pipeline = Pipeline(name="p", tasks=[_query_delta_copy("copy_q")]) - ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids - assert QUESTION_LAKEFLOW_CONNECTOR_TYPE not in ids + ids = {q.option_id for q in gather_options(pipeline).options} + assert OPTION_USE_LAKEFLOW_CONNECTORS in ids + assert OPTION_LAKEFLOW_CONNECTOR_TYPE not in ids - def test_lakeflow_connector_type_question_suppressed_per_copy_eligibility(self): + def test_lakeflow_connector_type_option_suppressed_per_copy_eligibility(self): """Per-Copy eligibility determines connector type with no overlap. Table-based reads can only use CDC (no cursor column) and queries @@ -960,21 +1059,21 @@ def test_lakeflow_connector_type_question_suppressed_per_copy_eligibility(self): eligible connector per Copy and the prompt is suppressed. """ pipeline = Pipeline(name="p", tasks=[_delta_copy("copy_a")]) - ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_LAKEFLOW_CONNECTOR_TYPE not in ids + ids = {q.option_id for q in gather_options(pipeline).options} + assert OPTION_LAKEFLOW_CONNECTOR_TYPE not in ids - def test_query_copy_routes_to_query_based_connector_regardless_of_preference(self, tmp_path: Path): + def test_query_copy_routes_to_query_based_connector_regardless_of_configuration(self, tmp_path: Path): import yaml from flowx.bundler.dab_writer import write_bundle from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_query_delta_copy("copy_q")]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( use_lakeflow_connectors="lakeflow_connect", lakeflow_connector_type="cdc", ) - stamped = apply_preferences(pipeline, prefs) + stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_q_lfc.yml").read_text()) @@ -994,8 +1093,8 @@ def test_table_copy_uses_cdc_connector_by_default(self, tmp_path: Path): from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - stamped = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_a_lfc.yml").read_text()) @@ -1003,14 +1102,14 @@ def test_table_copy_uses_cdc_connector_by_default(self, tmp_path: Path): assert "table" in objects[0] assert "table_configuration" not in objects[0] - def test_table_copy_with_query_based_preference_routes_to_cdc(self, tmp_path: Path): + def test_table_copy_with_query_based_configuration_routes_to_cdc(self, tmp_path: Path): """LFC query-based requires a cursor column. Table-based Copies have none. Per the Lakeflow Connect query-based-overview docs, the connector requires a cursor column to drive incremental ingestion. When the user prefers query_based but the Copy is table-based (no query, no cursor candidate), the modifier honours the - eligibility rules over the preference and routes to CDC. + eligibility rules over the configuration and routes to CDC. """ import yaml @@ -1018,11 +1117,11 @@ def test_table_copy_with_query_based_preference_routes_to_cdc(self, tmp_path: Pa from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( use_lakeflow_connectors="lakeflow_connect", lakeflow_connector_type="query_based", ) - stamped = apply_preferences(pipeline, prefs) + stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_a_lfc.yml").read_text()) @@ -1040,12 +1139,12 @@ def test_consolidated_metadata_driven_motif_emits_single_pipeline(self, tmp_path motif = _metadata_driven_motif() pipeline = Pipeline(name="job", tasks=[motif]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( metadata_driven_consolidate="consolidate", metadata_driven_access="yes", metadata_driven_size="medium", ) - stamped = apply_preferences(pipeline, prefs) + stamped = apply_configuration(pipeline, prefs) consolidated_motif = dataclasses.replace( stamped.tasks[0], lookup_values=[ @@ -1065,7 +1164,7 @@ def test_consolidated_metadata_driven_motif_emits_single_pipeline(self, tmp_path assert objects[0]["table"]["source_table"] == "orders" assert objects[1]["table"]["source_table"] == "customers" - def test_table_based_copy_with_query_based_preference_falls_back_to_cdc(self, tmp_path: Path): + def test_table_based_copy_with_query_based_configuration_falls_back_to_cdc(self, tmp_path: Path): """Table-based reads have no cursor column, so query-based isn't eligible. When the user prefers query_based but the only eligible LFC connector @@ -1090,11 +1189,11 @@ def test_table_based_copy_with_query_based_preference_falls_back_to_cdc(self, tm "connection": {"host": "flowx-test-sql.database.windows.net", "port": 1433}, }, ) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( use_lakeflow_connectors="lakeflow_connect", lakeflow_connector_type="query_based", ) - stamped = apply_preferences(Pipeline(name="job", tasks=[copy]), prefs) + stamped = apply_configuration(Pipeline(name="job", tasks=[copy]), prefs) write_bundle(prepare_workflow(stamped), tmp_path) resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_customers_lfc.yml").read_text()) obj = resource["resources"]["pipelines"]["copy_customers_lfc"]["ingestion_definition"]["objects"][0] @@ -1116,8 +1215,8 @@ def test_lakeflow_connect_uses_resolved_host_from_linked_service(self, tmp_path: "connection": {"host": "flowx-test-sql.database.windows.net", "port": 1433}, }, ) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - stamped = apply_preferences(Pipeline(name="job", tasks=[copy]), prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + stamped = apply_configuration(Pipeline(name="job", tasks=[copy]), prefs) write_bundle(prepare_workflow(stamped), tmp_path) body = (tmp_path / "src" / "setup" / "create_connections.py").read_text() assert "flowx-test-sql.database.windows.net" in body @@ -1151,17 +1250,17 @@ def test_lakeflow_connect_dedupes_connection_across_copies(self, tmp_path: Path) sink_properties={"table": "orders"}, source_properties={**shared_source, "source_table": "orders"}, ) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - stamped = apply_preferences(Pipeline(name="job", tasks=[copy_a, copy_b]), prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + stamped = apply_configuration(Pipeline(name="job", tasks=[copy_a, copy_b]), prefs) write_bundle(prepare_workflow(stamped), tmp_path) body = (tmp_path / "src" / "setup" / "create_connections.py").read_text() assert body.count("CREATE CONNECTION IF NOT EXISTS") == 1 - assert body.count("orchestra_LS_AzureSqlDb_connection") >= 1 + assert body.count("flowx_LS_AzureSqlDb_connection") >= 1 for pipeline_file in ("copy_a_lfc.yml", "copy_b_lfc.yml"): resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / pipeline_file).read_text()) key = pipeline_file.replace(".yml", "") assert resource["resources"]["pipelines"][key]["ingestion_definition"]["connection_name"] == ( - "orchestra_LS_AzureSqlDb_connection" + "flowx_LS_AzureSqlDb_connection" ) def test_lakeflow_connect_emits_connection_setup_notebook(self, tmp_path: Path): @@ -1169,13 +1268,13 @@ def test_lakeflow_connect_emits_connection_setup_notebook(self, tmp_path: Path): from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - stamped = apply_preferences(pipeline, TranslationPreferences(use_lakeflow_connectors="lakeflow_connect")) + stamped = apply_configuration(pipeline, TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect")) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) setup_notebook = tmp_path / "src" / "setup" / "create_connections.py" assert setup_notebook.exists() body = setup_notebook.read_text() - assert "orchestra_copy_a_connection" in body + assert "flowx_copy_a_connection" in body assert "SQLSERVER" in body def test_existing_default_binds_to_default_cluster(self, tmp_path: Path): @@ -1188,7 +1287,7 @@ def test_existing_default_binds_to_default_cluster(self, tmp_path: Path): name="job", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/existing")], ) - stamped = apply_preferences(pipeline, TranslationPreferences()) + stamped = apply_configuration(pipeline, TranslationConfiguration()) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) diff --git a/tests/unit/test_adf_loader.py b/tests/unit/test_adf_loader.py index 678c24a..f56b4d8 100644 --- a/tests/unit/test_adf_loader.py +++ b/tests/unit/test_adf_loader.py @@ -13,6 +13,7 @@ _parse_pipeline_json, build_inventory, classify_activity, + clear_stale_outputs, load_adf_definitions, ) @@ -302,3 +303,52 @@ def test_normalize_arm_passthrough(self): data = {"name": "simple", "properties": {"activities": []}} result = _normalize_arm(data) assert result is data + + +# --------------------------------------------------------------------------- +# clear_stale_outputs +# --------------------------------------------------------------------------- + + +class TestClearStaleOutputs: + """Discover must reset a reused output_dir so prior runs don't leak into the bundle.""" + + def test_removes_prior_run_artifacts(self, tmp_path): + """Stale per-pipeline metadata and a prior generated bundle are removed.""" + (tmp_path / "metadata").mkdir() + (tmp_path / "metadata" / "OldPipeline.arm.json").write_text("{}", encoding="utf-8") + (tmp_path / "resources").mkdir() + (tmp_path / "resources" / "old_pipeline.yml").write_text("name: old", encoding="utf-8") + (tmp_path / "src" / "notebooks").mkdir(parents=True) + (tmp_path / "src" / "notebooks" / "old.py").write_text("print('old')", encoding="utf-8") + (tmp_path / ".work").mkdir() + (tmp_path / ".work" / "translation_report.json").write_text("{}", encoding="utf-8") + (tmp_path / "databricks.yml").write_text("bundle: old", encoding="utf-8") + (tmp_path / "SETUP.md").write_text("# old", encoding="utf-8") + (tmp_path / "WARNINGS.md").write_text("# old", encoding="utf-8") + + clear_stale_outputs(tmp_path) + + assert not (tmp_path / "metadata").exists() + assert not (tmp_path / "resources").exists() + assert not (tmp_path / "src").exists() + assert not (tmp_path / ".work").exists() + assert not (tmp_path / "databricks.yml").exists() + assert not (tmp_path / "SETUP.md").exists() + assert not (tmp_path / "WARNINGS.md").exists() + + def test_preserves_unrelated_files(self, tmp_path): + """Only flowx-managed entries are removed; unrelated files stay put.""" + (tmp_path / "notes.txt").write_text("keep me", encoding="utf-8") + (tmp_path / "user_data").mkdir() + (tmp_path / "user_data" / "keep.csv").write_text("a,b", encoding="utf-8") + + clear_stale_outputs(tmp_path) + + assert (tmp_path / "notes.txt").read_text(encoding="utf-8") == "keep me" + assert (tmp_path / "user_data" / "keep.csv").exists() + + def test_idempotent_on_empty_dir(self, tmp_path): + """Clearing a directory with no flowx artifacts is a no-op (no error).""" + clear_stale_outputs(tmp_path) + assert list(tmp_path.iterdir()) == [] diff --git a/tests/unit/test_bundle_invariants.py b/tests/unit/test_bundle_invariants.py new file mode 100644 index 0000000..aed8ebc --- /dev/null +++ b/tests/unit/test_bundle_invariants.py @@ -0,0 +1,61 @@ +"""Unit tests for bundle structural-invariant checks (guard #2).""" + +from __future__ import annotations + +from flowx.validate.bundle_invariants import check_job, check_resource_text + + +def _codes(findings) -> set[str]: + return {f.code for f in findings} + + +def test_clean_job_has_no_findings(): + job = { + "name": "p", + "parameters": [{"name": "region", "default": "us"}], + "tasks": [ + { + "task_key": "a", + "notebook_task": {"notebook_path": "/n", "base_parameters": {"region": "{{job.parameters.region}}"}}, + }, + {"task_key": "b", "depends_on": [{"task_key": "a"}], "notebook_task": {"notebook_path": "/n"}}, + ], + } + assert check_job("p", job) == [] + + +def test_duplicate_job_parameter_flagged(): + job = {"parameters": [{"name": "region", "default": "us"}, {"name": "region", "default": "us"}], "tasks": []} + assert "duplicate_job_parameter" in _codes(check_job("p", job)) + + +def test_duplicate_task_key_flagged(): + job = {"tasks": [{"task_key": "a"}, {"task_key": "a"}]} + assert "duplicate_task_key" in _codes(check_job("p", job)) + + +def test_undeclared_job_parameter_reference_flagged(): + job = { + "parameters": [{"name": "region"}], + "tasks": [{"task_key": "a", "notebook_task": {"base_parameters": {"env": "{{job.parameters.env}}"}}}], + } + codes = _codes(check_job("p", job)) + assert "undeclared_job_parameter" in codes # env is referenced but not declared + + +def test_dangling_depends_on_flagged(): + job = {"tasks": [{"task_key": "a", "depends_on": [{"task_key": "ghost"}]}]} + assert "dangling_depends_on" in _codes(check_job("p", job)) + + +def test_yaml_anchor_smell_flagged(): + # The exact shape PyYAML emits when the same object is in a list twice. + text = ( + "resources:\n jobs:\n p:\n name: p\n tasks: []\n" + " parameters:\n - &id001\n name: region\n default: us\n - *id001\n" + ) + findings = check_resource_text(text, filename="p.yml") + codes = _codes(findings) + assert "yaml_anchor" in codes + # and the parsed structure also trips the duplicate-parameter invariant + assert "duplicate_job_parameter" in codes diff --git a/tests/unit/test_dag_equivalence.py b/tests/unit/test_dag_equivalence.py new file mode 100644 index 0000000..4957c9b --- /dev/null +++ b/tests/unit/test_dag_equivalence.py @@ -0,0 +1,220 @@ +"""Tests for the motif-aware Tier-0 DAG equivalence check.""" + +from __future__ import annotations + +from flowx.models.adf_ast import AdfActivity, AdfDependency, AdfPipeline +from flowx.models.ir import Activity, Dependency, Pipeline +from flowx.models.motifs import ( + MOTIF_ACTIVITY_AND_NOTIFY, + MOTIF_METADATA_DRIVEN_BULK_COPY, + DetectedMotif, +) +from flowx.motifs.collapser import collapse_motifs +from flowx.utils import normalize_task_key +from flowx.validate import check_dag_equivalence, format_result + + +def _adf(name: str, deps: dict[str, list[str]] | None = None, adf_type: str = "Copy") -> AdfActivity: + """ADF activity; *deps* maps upstream name -> dependency conditions.""" + depends = [AdfDependency(activity=u, dependency_conditions=c) for u, c in (deps or {}).items()] + return AdfActivity(name=name, type=adf_type, depends_on=depends or None) + + +def _task(name: str, deps: list[tuple[str, str]] | None = None) -> Activity: + """IR leaf task; *deps* is a list of (upstream_name, outcome).""" + edges = [Dependency(task_key=normalize_task_key(u), outcome=o) for u, o in (deps or [])] + return Activity(name=name, task_key=normalize_task_key(name), depends_on=edges or None) + + +def _codes(result) -> set[str]: + return {f.code for f in result.findings} + + +# --------------------------------------------------------------------------- +# Identity (no motifs) +# --------------------------------------------------------------------------- + + +def test_identity_dag_is_equivalent(): + adf = AdfPipeline( + name="p", + activities=[_adf("A"), _adf("B", {"A": ["Succeeded"]}), _adf("C", {"B": ["Succeeded"]})], + ) + ir = Pipeline( + name="p", + tasks=[_task("A"), _task("B", [("A", "Succeeded")]), _task("C", [("B", "Succeeded")])], + ) + result = check_dag_equivalence(adf, ir) + assert result.equivalent + assert not result.violations + assert not result.warnings + + +# --------------------------------------------------------------------------- +# Motif collapse (convex) -- differences tolerated +# --------------------------------------------------------------------------- + + +def test_convex_motif_collapse_is_tolerated(): + # ADF: Lookup -> ForEach -> Sink. Motif collapses {Lookup, ForEach}. + adf = AdfPipeline( + name="p", + activities=[ + _adf("Lookup", adf_type="Lookup"), + _adf("ForEach", {"Lookup": ["Succeeded"]}, adf_type="ForEach"), + _adf("Sink", {"ForEach": ["Succeeded"]}), + ], + ) + pre = Pipeline( + name="p", + tasks=[_task("Lookup"), _task("ForEach", [("Lookup", "Succeeded")]), _task("Sink", [("ForEach", "Succeeded")])], + ) + motif = DetectedMotif(definition=MOTIF_METADATA_DRIVEN_BULK_COPY, matched_activities=["Lookup", "ForEach"]) + collapsed = collapse_motifs(pre, [motif]) + + result = check_dag_equivalence(adf, collapsed) + assert result.equivalent + assert not result.violations + # The Lookup -> ForEach internal edge was absorbed, not reported as a loss. + assert "collapsed_internal_edges" in _codes(result) + assert "missing_edge" not in _codes(result) + + +def test_activity_and_notify_collapse_is_equivalent(): + # ADF: Copy -> {NotifySuccess, NotifyFailure}. All three collapse. + adf = AdfPipeline( + name="p", + activities=[ + _adf("Copy"), + _adf("NotifySuccess", {"Copy": ["Succeeded"]}, adf_type="WebActivity"), + _adf("NotifyFailure", {"Copy": ["Failed"]}, adf_type="WebActivity"), + ], + ) + pre = Pipeline( + name="p", + tasks=[ + _task("Copy"), + _task("NotifySuccess", [("Copy", "Succeeded")]), + _task("NotifyFailure", [("Copy", "Failed")]), + ], + ) + motif = DetectedMotif( + definition=MOTIF_ACTIVITY_AND_NOTIFY, + matched_activities=["Copy", "NotifySuccess", "NotifyFailure"], + ) + collapsed = collapse_motifs(pre, [motif]) + + result = check_dag_equivalence(adf, collapsed) + assert result.equivalent + assert not result.violations + + +# --------------------------------------------------------------------------- +# Non-convex collapse -- the invariant violation +# --------------------------------------------------------------------------- + + +def test_non_convex_motif_is_a_violation(): + # ADF: a1 -> w -> a2, but the motif tries to collapse {a1, a2} with the + # external w sandwiched between them. Collapsing would reorder w. + adf = AdfPipeline( + name="p", + activities=[ + _adf("a1"), + _adf("w", {"a1": ["Succeeded"]}), + _adf("a2", {"w": ["Succeeded"]}), + ], + ) + pre = Pipeline( + name="p", + tasks=[_task("a1"), _task("w", [("a1", "Succeeded")]), _task("a2", [("w", "Succeeded")])], + ) + motif = DetectedMotif(definition=MOTIF_METADATA_DRIVEN_BULK_COPY, matched_activities=["a1", "a2"]) + collapsed = collapse_motifs(pre, [motif]) + + result = check_dag_equivalence(adf, collapsed) + assert not result.equivalent + assert "non_convex_motif" in _codes(result) + non_convex = next(f for f in result.violations if f.code == "non_convex_motif") + assert "w" in non_convex.nodes + + +# --------------------------------------------------------------------------- +# Dropped cross-boundary edge -- ordering constraint lost +# --------------------------------------------------------------------------- + + +def test_dropped_ordering_edge_is_a_violation(): + adf = AdfPipeline( + name="p", + activities=[_adf("A"), _adf("B", {"A": ["Succeeded"]}), _adf("C", {"B": ["Succeeded"]})], + ) + # IR forgot the B -> C edge. + ir = Pipeline(name="p", tasks=[_task("A"), _task("B", [("A", "Succeeded")]), _task("C")]) + result = check_dag_equivalence(adf, ir) + assert not result.equivalent + assert "missing_edge" in _codes(result) + + +# --------------------------------------------------------------------------- +# Synthesised init task -- IR-only, tolerated +# --------------------------------------------------------------------------- + + +def test_synthesised_init_task_is_tolerated(): + adf = AdfPipeline(name="p", activities=[_adf("A"), _adf("B", {"A": ["Succeeded"]})]) + init = Activity(name="_init_flag", task_key="_init_flag") + ir = Pipeline(name="p", tasks=[init, _task("A"), _task("B", [("A", "Succeeded")])]) + result = check_dag_equivalence(adf, ir) + assert result.equivalent + assert "synthesised_task" in _codes(result) + assert "unmapped_ir_task" not in _codes(result) + + +# --------------------------------------------------------------------------- +# Merged dependency outcomes -- lossy collapse, warned (not blocking) +# --------------------------------------------------------------------------- + + +def test_merged_outcome_is_warned_not_blocking(): + # Both motif members depend on E, but with different conditions; the + # collapser keeps only one, which we surface as a warning. + adf = AdfPipeline( + name="p", + activities=[ + _adf("E"), + _adf("a1", {"E": ["Succeeded"]}), + _adf("a2", {"E": ["Failed"]}), + ], + ) + pre = Pipeline( + name="p", + tasks=[_task("E"), _task("a1", [("E", "Succeeded")]), _task("a2", [("E", "Failed")])], + ) + motif = DetectedMotif(definition=MOTIF_ACTIVITY_AND_NOTIFY, matched_activities=["a1", "a2"]) + collapsed = collapse_motifs(pre, [motif]) + + result = check_dag_equivalence(adf, collapsed) + assert result.equivalent # warning, not violation + assert "merged_outcome" in _codes(result) + + +# --------------------------------------------------------------------------- +# Extra ordering edge -- over-constraining, warned +# --------------------------------------------------------------------------- + + +def test_extra_edge_is_warned(): + adf = AdfPipeline(name="p", activities=[_adf("A"), _adf("B")]) # A and B independent + ir = Pipeline(name="p", tasks=[_task("A"), _task("B", [("A", "Succeeded")])]) # IR adds A -> B + result = check_dag_equivalence(adf, ir) + assert result.equivalent + assert "extra_edge" in _codes(result) + + +def test_format_result_reports_status(): + adf = AdfPipeline(name="p", activities=[_adf("A"), _adf("B", {"A": ["Succeeded"]})]) + ir = Pipeline(name="p", tasks=[_task("A"), _task("B")]) # missing edge + rendered = format_result(check_dag_equivalence(adf, ir)) + assert "NOT EQUIVALENT" in rendered + assert "missing_edge" in rendered diff --git a/tests/unit/test_mcp_migrate.py b/tests/unit/test_mcp_migrate.py new file mode 100644 index 0000000..cf38589 --- /dev/null +++ b/tests/unit/test_mcp_migrate.py @@ -0,0 +1,117 @@ +"""Tests for the MCP server's agent-driven interactive ``migrate`` flow. + +The server returns the full option schema once (``needs_input``); the agent walks the chain locally +and re-calls ``migrate`` once with the complete answers, which applies and packages. These tests +guard that one-shot contract. Skipped where the optional ``mcp`` dependency is absent. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +pytest.importorskip("mcp") + +from flowx.mcp import runner, server # noqa: E402 + + +class _FakeResult: + def __init__(self, *, ok: bool = True, stdout: str = "", stderr: str = "") -> None: + self.ok = ok + self.stdout = stdout + self.stderr = stderr + self.returncode = 0 if ok else 1 + + def as_dict(self) -> dict[str, object]: + return {"returncode": self.returncode, "stdout": self.stdout, "stderr": self.stderr} + + +_SCHEMA = { + "pipelines": [ + { + "pipeline_name": "p", + "options": [ + { + "option_id": "notify_destination", + "prompt": "Route notifications?", + "rationale": "...", + "choices": [{"value": "keep", "label": "Keep", "description": ""}], + "free_text": False, + "default": "keep", + "affected_task_keys": ["load"], + "show_when": [], + }, + { + "option_id": "notify_slack_url", + "prompt": "Slack URL?", + "rationale": "...", + "choices": [], + "free_text": True, + "default": "", + "affected_task_keys": ["load"], + "show_when": [{"option_id": "notify_destination", "in": ["slack"]}], + }, + ], + } + ] +} + + +@pytest.fixture +def stub_adapter(monkeypatch): + """Stubs the adapter subprocess + artifact readers; records which subcommands ran.""" + calls: list[str] = [] + + def fake_run_adapter(args): + calls.append(args[0]) + if args[0] == "inspect": + return _FakeResult(stdout=json.dumps(_SCHEMA)) + return _FakeResult() + + monkeypatch.setattr(runner, "run_adapter", fake_run_adapter) + monkeypatch.setattr(runner, "summarize_inventory", lambda out: {"pipeline_count": 1}) + monkeypatch.setattr(runner, "summarize_translation", lambda out: {"translated": 1}) + monkeypatch.setattr(runner, "list_tree", lambda out: ["databricks.yml"]) + monkeypatch.setattr(runner, "read_tree", lambda out: {"files": {}, "truncated": []}) + return calls + + +def test_first_call_returns_full_schema_without_packaging(stub_adapter, tmp_path: Path): + result = server._cmd_migrate({"adf_source_path": str(tmp_path / "adf"), "output_dir": str(tmp_path / "out")}) + assert result["status"] == "needs_input" + # The whole tree (including the conditional slack follow-up) is returned up front. + option_ids = {o["option_id"] for pipe in result["pending_options"] for o in pipe["options"]} + assert {"notify_destination", "notify_slack_url"} <= option_ids + all_options = [o for pipe in result["pending_options"] for o in pipe["options"]] + slack = next(o for o in all_options if o["option_id"] == "notify_slack_url") + assert slack["show_when"] == [{"option_id": "notify_destination", "in": ["slack"]}] + # discover + convert ran, but NOT package (we paused for input). + assert stub_adapter == ["discover", "convert", "inspect"] + + +def test_resume_with_answers_applies_and_packages_once(stub_adapter, tmp_path: Path): + out = tmp_path / "out" + (out / ".work").mkdir(parents=True) + (out / ".work" / "translation_report.json").write_text("{}") # prior convert output -> resume path + + result = server._cmd_migrate( + { + "adf_source_path": str(tmp_path / "adf"), + "output_dir": str(out), + "answers": ["notify_destination=slack", "notify_slack_url=https://hooks.slack.com/x"], + } + ) + assert result["status"] == "completed" + # Resume skips discover/convert and does not re-inspect; it applies the answers then packages. + assert stub_adapter == ["modify", "package"] + assert "apply_answers" in result["steps"] and "package" in result["steps"] + + +def test_interactive_false_skips_prompt_and_packages(stub_adapter, tmp_path: Path): + result = server._cmd_migrate( + {"adf_source_path": str(tmp_path / "adf"), "output_dir": str(tmp_path / "out"), "interactive": False} + ) + assert result["status"] == "completed" + assert stub_adapter == ["discover", "convert", "package"] # no inspect, no pause diff --git a/tests/unit/test_merge_agentic.py b/tests/unit/test_merge_agentic.py new file mode 100644 index 0000000..9723cbe --- /dev/null +++ b/tests/unit/test_merge_agentic.py @@ -0,0 +1,100 @@ +"""Tests for engine --merge-agentic (folding agent results into a translation report).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from flowx.translator.engine import merge_agentic_results + + +def _write(path: Path, obj: object) -> None: + path.write_text(json.dumps(obj), encoding="utf-8") + + +def test_merge_replaces_nested_placeholder_and_preserves_edges(tmp_path: Path): + report = tmp_path / "translation_report.json" + _write( + report, + { + "name": "p", + "tasks": [ + { + "name": "Gate", + "type": "IfConditionActivity", + "task_key": "gate", + "if_true_activities": [ + { + "name": "Wait", + "type": "PlaceholderActivity", + "task_key": "wait", + "original_type": "Until", + "depends_on": [{"task_key": "upstream", "outcome": "Succeeded"}], + }, + ], + }, + ], + }, + ) + results = tmp_path / "agentic_results" + results.mkdir() + _write( + results / "wait.json", + { + "activity_name": "Wait", + "task": { + "type": "NotebookActivity", + "name": "Wait", + "task_key": "wait", + "notebook_path": "/Workspace/Shared/until_wait", + }, + }, + ) + + merged, unmatched = merge_agentic_results(report, results) + assert (merged, unmatched) == (1, 0) + out = json.loads(report.read_text()) + task = out["tasks"][0]["if_true_activities"][0] + assert task["type"] == "NotebookActivity" + assert task["notebook_path"] == "/Workspace/Shared/until_wait" + # depends_on carried over from the placeholder + assert task["depends_on"] == [{"task_key": "upstream", "outcome": "Succeeded"}] + + +def test_merge_unmatched_when_activity_absent(tmp_path: Path): + report = tmp_path / "r.json" + _write(report, {"name": "p", "tasks": [{"name": "A", "type": "NotebookActivity", "task_key": "a"}]}) + results = tmp_path / "res" + results.mkdir() + _write(results / "x.json", {"activity_name": "Nope", "task": {"type": "NotebookActivity", "name": "Nope"}}) + + merged, unmatched = merge_agentic_results(report, results) + assert (merged, unmatched) == (0, 1) + + +def test_merge_multi_pipeline_disambiguates_by_name(tmp_path: Path): + report = tmp_path / "r.json" + _write( + report, + { + "pipelines": [ + {"name": "p1", "tasks": [{"name": "U", "type": "PlaceholderActivity", "task_key": "u1"}]}, + {"name": "p2", "tasks": [{"name": "U", "type": "PlaceholderActivity", "task_key": "u2"}]}, + ] + }, + ) + results = tmp_path / "res" + results.mkdir() + _write( + results / "u.json", + { + "pipeline": "p2", + "activity_name": "U", + "task": {"type": "NotebookActivity", "name": "U", "task_key": "u2", "notebook_path": "/x"}, + }, + ) + merged, unmatched = merge_agentic_results(report, results) + assert (merged, unmatched) == (1, 0) + out = json.loads(report.read_text()) + assert out["pipelines"][0]["tasks"][0]["type"] == "PlaceholderActivity" # p1 untouched + assert out["pipelines"][1]["tasks"][0]["type"] == "NotebookActivity" # p2 merged diff --git a/tests/unit/test_motifs.py b/tests/unit/test_motifs.py index 1be2e9b..0ca9639 100644 --- a/tests/unit/test_motifs.py +++ b/tests/unit/test_motifs.py @@ -110,7 +110,7 @@ def test_detects_copy_then_web_notification(self): ) motifs = detect_motifs(pipeline, _EMPTY_DEFS) assert len(motifs) == 1 - assert motifs[0].definition.motif_id == "copy_and_notify" + assert motifs[0].definition.motif_id == "activity_and_notify" class TestDetectorParentChild: diff --git a/tests/unit/test_notify.py b/tests/unit/test_notify.py new file mode 100644 index 0000000..ea8cc88 --- /dev/null +++ b/tests/unit/test_notify.py @@ -0,0 +1,397 @@ +"""Tests for the activity_and_notify -> Databricks notification destination feature.""" + +from __future__ import annotations + +from flowx.adapter.models import TranslationConfiguration +from flowx.adapter.operations import ( + apply_configuration, + collect_notify_args, + gather_options, + provision_notification_destinations, +) +from flowx.models.ir import ( + CopyActivity, + Dependency, + LookupActivity, + NotebookActivity, + Pipeline, + WebActivity, +) +from flowx.preparer.notifications import resolve_task_notifications + + +def _pipeline_with_upstream_notify(upstream) -> Pipeline: + """A non-Copy upstream activity followed by success/failure notify Web activities.""" + notify_ok = WebActivity( + name="Notify Success", + task_key="notify_success", + url="https://x", + method="POST", + depends_on=[Dependency(task_key=upstream.task_key, outcome="Succeeded")], + ) + notify_fail = WebActivity( + name="Notify Failure", + task_key="notify_failure", + url="https://x", + method="POST", + depends_on=[Dependency(task_key=upstream.task_key, outcome="Failed")], + ) + return Pipeline(name="p", tasks=[upstream, notify_ok, notify_fail]) + + +def test_notebook_upstream_surfaces_and_collapses(): + """A Notebook (not a Copy) followed by notify Web activities is offered and collapses.""" + p = _pipeline_with_upstream_notify(NotebookActivity(name="Transform", task_key="transform", notebook_path="/t")) + assert "notify_destination" in {o.option_id for o in gather_options(p, []).options} + + cfg = TranslationConfiguration(notify_destination="email", notify_args={"addresses": "a@x.com"}) + out = apply_configuration(p, cfg) + names = {t.name for t in out.tasks} + assert "Notify Success" not in names and "Notify Failure" not in names + transform = next(t for t in out.tasks if t.task_key == "transform") + assert transform.notifications["destination"] == "email" + assert set(transform.notifications["events"]) == {"on_success", "on_failure"} + + +def test_lookup_upstream_collapses(): + """A Lookup followed by a failure-notify Web collapses onto the Lookup task.""" + lookup = LookupActivity(name="Read Control", task_key="read_control") + notify = WebActivity( + name="Alert", + task_key="alert", + url="https://x", + method="POST", + depends_on=[Dependency(task_key="read_control", outcome="Failed")], + ) + p = Pipeline(name="p", tasks=[lookup, notify]) + out = apply_configuration(p, TranslationConfiguration(notify_destination="email", notify_args={"addresses": "a@x"})) + assert "Alert" not in {t.name for t in out.tasks} + read = next(t for t in out.tasks if t.task_key == "read_control") + assert read.notifications["destination"] == "email" + assert read.notifications["events"] == ["on_failure"] + + +def test_generic_preparer_wires_notifications_on_non_copy_task(): + """prepare_activity wires a stamped notification spec into the task for any type, not just Copy.""" + from flowx.preparer.workflow_preparer import prepare_activity + + notebook = NotebookActivity( + name="Transform", + task_key="transform", + notebook_path="/t", + notifications={"destination": "email", "args": {"addresses": ["a@x.com"]}, "events": ["on_failure"]}, + ) + prepared = prepare_activity(notebook) + assert prepared.task["email_notifications"] == {"on_failure": ["a@x.com"]} + + +def test_web_upstream_is_not_a_notify_target(): + """A WebActivity following another WebActivity is not treated as a notify group (web->web).""" + work = WebActivity(name="Call API", task_key="call_api", url="https://api", method="POST") + notify = WebActivity( + name="Notify", + task_key="notify", + url="https://x", + method="POST", + depends_on=[Dependency(task_key="call_api", outcome="Succeeded")], + ) + p = Pipeline(name="p", tasks=[work, notify]) + assert "notify_destination" not in {o.option_id for o in gather_options(p, []).options} + + +def _pipeline_with_notify() -> Pipeline: + copy = CopyActivity(name="Load Curated", task_key="load_curated") + notify_ok = WebActivity( + name="Notify Success", + task_key="notify_success", + url="https://x", + method="POST", + depends_on=[Dependency(task_key="load_curated", outcome="Succeeded")], + ) + notify_fail = WebActivity( + name="Notify Failure", + task_key="notify_failure", + url="https://x", + method="POST", + depends_on=[Dependency(task_key="load_curated", outcome="Failed")], + ) + downstream = NotebookActivity( + name="After", + task_key="after", + notebook_path="/n", + depends_on=[Dependency(task_key="notify_success", outcome="Succeeded")], + ) + return Pipeline(name="p", tasks=[copy, notify_ok, notify_fail, downstream]) + + +def test_option_surfaces_and_followups_are_gated_by_answer(): + p = _pipeline_with_notify() + ids = {o.option_id for o in gather_options(p, []).options} + assert "notify_destination" in ids + # follow-ups not shown until a destination is chosen + assert "notify_email_recipients" not in ids + assert "notify_slack_url" not in ids + + email_ids = {o.option_id for o in gather_options(p, [], answers={"notify_destination": "email"}).options} + assert "notify_email_recipients" in email_ids + assert "notify_slack_url" not in email_ids + + slack_ids = {o.option_id for o in gather_options(p, [], answers={"notify_destination": "slack"}).options} + assert "notify_slack_url" in slack_ids + assert "notify_destination_name" in slack_ids + assert "notify_email_recipients" not in slack_ids + + +def test_chain_surfaces_every_sdk_field_for_destination(): + """Each SDK field of the chosen destination becomes its own follow-up option, + in registry order (required first), so the agent can prompt sequentially.""" + p = _pipeline_with_notify() + webhook_ids = [o.option_id for o in gather_options(p, [], answers={"notify_destination": "webhook"}).options] + # SDK fields surface in registry order (required url first), then name + events + webhook_fields = [i for i in webhook_ids if i.startswith("notify_webhook")] + assert webhook_fields == [ + "notify_webhook_url", + "notify_webhook_username", + "notify_webhook_password", + ] + assert "notify_destination_name" in webhook_ids + assert "notify_events" in webhook_ids + + slack_ids = [o.option_id for o in gather_options(p, [], answers={"notify_destination": "slack"}).options] + slack_fields = [i for i in slack_ids if i.startswith("notify_slack")] + assert slack_fields == [ + "notify_slack_url", + "notify_slack_channel_id", + "notify_slack_oauth_token", + ] + + +def test_answered_field_drops_out_of_the_chain(): + """Already-answered follow-ups are filtered, so the chain advances field by field.""" + p = _pipeline_with_notify() + answers = {"notify_destination": "slack", "notify_slack_url": "https://hooks.slack.com/x"} + ids = {o.option_id for o in gather_options(p, [], answers=answers).options} + assert "notify_slack_url" not in ids # answered -> gone + assert "notify_slack_channel_id" in ids # still pending + + +def test_collect_notify_args_reads_only_chosen_destination_fields(): + answers = { + "notify_destination": "webhook", + "notify_webhook_url": "https://hooks.example.com", + "notify_webhook_username": "svc", + "notify_webhook_password": "", # blank -> omitted + "notify_slack_url": "https://leftover.slack", # belongs to a different dest -> ignored + } + args = collect_notify_args(answers) + assert args == {"url": "https://hooks.example.com", "username": "svc"} + + +def test_keep_default_does_not_collapse(): + p = _pipeline_with_notify() + out = apply_configuration(p, TranslationConfiguration()) # default = keep + names = {t.name for t in out.tasks} + assert {"Notify Success", "Notify Failure"} <= names # still present + + +def test_email_collapse_drops_notifies_and_stamps_copy(): + p = _pipeline_with_notify() + cfg = TranslationConfiguration( + notify_destination="email", + notify_args={"addresses": "a@x.com, b@x.com"}, + notify_events="both", + ) + out = apply_configuration(p, cfg) + names = {t.name for t in out.tasks} + assert "Notify Success" not in names and "Notify Failure" not in names + copy = next(t for t in out.tasks if t.task_key == "load_curated") + assert copy.notifications["destination"] == "email" + assert copy.notifications["args"]["addresses"] == ["a@x.com", "b@x.com"] + assert set(copy.notifications["events"]) == {"on_success", "on_failure"} + # downstream task rewired off the dropped notify onto the copy + after = next(t for t in out.tasks if t.task_key == "after") + assert any(d.task_key == "load_curated" for d in (after.depends_on or [])) + + +def test_webhook_collapse_stamps_resolved_args(): + p = _pipeline_with_notify() + cfg = TranslationConfiguration( + notify_destination="webhook", + notify_args={"url": "https://hooks.example.com", "username": "svc"}, + notify_events="both", + ) + out = apply_configuration(p, cfg) + copy = next(t for t in out.tasks if t.task_key == "load_curated") + assert copy.notifications["destination"] == "webhook" + assert copy.notifications["args"] == {"url": "https://hooks.example.com", "username": "svc"} + + +def test_events_restriction_to_failure_only(): + p = _pipeline_with_notify() + cfg = TranslationConfiguration( + notify_destination="email", + notify_args={"addresses": "a@x.com"}, + notify_events="on_failure", + ) + out = apply_configuration(p, cfg) + copy = next(t for t in out.tasks if t.task_key == "load_curated") + assert copy.notifications["events"] == ["on_failure"] + + +def test_resolve_email_notifications(): + keys, setup = resolve_task_notifications( + {"destination": "email", "args": {"addresses": ["a@x.com"]}, "events": ["on_failure", "on_success"]} + ) + assert keys == {"email_notifications": {"on_failure": ["a@x.com"], "on_success": ["a@x.com"]}} + assert setup == [] + + +def test_resolve_webhook_without_workspace_falls_back_to_setup_task(monkeypatch): + # Force the SDK create path to fail -> graceful fallback to a setup task. + import flowx.preparer.notifications as nm + + monkeypatch.setattr(nm, "_ensure_destination", lambda *a, **k: None) + keys, setup = resolve_task_notifications( + { + "destination": "slack", + "args": {"url": "https://hooks"}, + "destination_name": "flowx-slack", + "events": ["on_failure"], + } + ) + assert keys == {} + assert len(setup) == 1 and setup[0].type == "notification_destination" + assert setup[0].config["url"] == "https://hooks" + + +def test_build_destination_config_passes_only_supplied_optional_fields(): + """Optional SDK kwargs are omitted when blank so the SDK applies its defaults.""" + import flowx.preparer.notifications as nm + + class _FakeSlackConfig: + def __init__(self, **kwargs): + self.kwargs = kwargs + + class _FakeConfig: + def __init__(self, slack=None): + self.slack = slack + + class _FakeSettings: + Config = _FakeConfig + SlackConfig = _FakeSlackConfig + + cfg = nm._build_destination_config(_FakeSettings, "slack", {"url": "https://hooks", "channel_id": ""}) + assert cfg.slack.kwargs == {"url": "https://hooks"} # blank channel_id dropped + + +def test_validate_answer_accepts_free_text_notify_options(): + """Regression: free-text notify follow-ups must validate (were rejected + as 'Unknown option_id', so email recipients never reached the config).""" + import pytest + + from flowx.adapter.operations import validate_answer + + # free-text options accept any value + assert validate_answer("notify_email_recipients", "a@x.com, b@x.com") == "a@x.com, b@x.com" + assert validate_answer("notify_webhook_url", "https://hooks.example.com") == "https://hooks.example.com" + assert validate_answer("notify_slack_oauth_token", "xoxb-123") == "xoxb-123" + assert validate_answer("notify_pagerduty_integration_key", "abc123") == "abc123" + assert validate_answer("notify_destination_name", "flowx-oncall") == "flowx-oncall" + # enum-backed options still validate against their enum + assert validate_answer("notify_destination", "email") == "email" + with pytest.raises(ValueError): + validate_answer("notify_destination", "carrier_pigeon") + # genuinely unknown ids are still rejected + with pytest.raises(ValueError): + validate_answer("totally_unknown_option", "x") + + +def test_provision_destination_creates_at_prompt_time(monkeypatch): + """Non-email destinations are created (SDK) at prompt time and the resolved + id is stamped onto the spec.""" + import flowx.preparer.notifications as nm + + monkeypatch.setattr(nm, "_ensure_destination", lambda dest, name, args: "dest-abc-1") + spec = {"destination": "slack", "destination_name": "flowx-slack", "args": {"url": "https://h"}} + new_spec, message = nm.provision_destination(spec) + assert new_spec["destination_id"] == "dest-abc-1" + assert "Created" in message or "reused" in message.lower() + + +def test_provision_destination_email_is_passthrough(monkeypatch): + """Email needs no destination -- provision is a no-op and never calls the SDK.""" + import flowx.preparer.notifications as nm + + def _boom(*a, **k): + raise AssertionError("SDK must not be called for email") + + monkeypatch.setattr(nm, "_ensure_destination", _boom) + spec = {"destination": "email", "args": {"addresses": ["a@x.com"]}} + new_spec, message = nm.provision_destination(spec) + assert new_spec is spec + assert message == "" + + +def test_provision_destination_failure_keeps_spec(monkeypatch): + """When creation fails at prompt time the spec is unchanged (args retained) so + prepare can retry / emit a setup task, and a warning is surfaced.""" + import flowx.preparer.notifications as nm + + monkeypatch.setattr(nm, "_ensure_destination", lambda *a, **k: None) + spec = {"destination": "webhook", "args": {"url": "https://h"}} + new_spec, message = nm.provision_destination(spec) + assert "destination_id" not in new_spec + assert new_spec is spec + assert message.startswith("WARNING") + + +def test_provision_notification_destinations_walk(monkeypatch): + """The adapter modify-phase walk stamps resolved ids onto non-email copy tasks.""" + import flowx.preparer.notifications as nm + + monkeypatch.setattr(nm, "_ensure_destination", lambda dest, name, args: "dest-xyz-9") + p = _pipeline_with_notify() + cfg = TranslationConfiguration( + notify_destination="slack", + notify_args={"url": "https://hooks.slack.com/x"}, + notify_events="both", + ) + stamped = apply_configuration(p, cfg) + provisioned, messages = provision_notification_destinations(stamped) + copy = next(t for t in provisioned.tasks if t.task_key == "load_curated") + assert copy.notifications["destination_id"] == "dest-xyz-9" + assert len(messages) == 1 + + +def test_provision_walk_skips_email_and_keep(monkeypatch): + """Email collapse produces no destination; the walk makes no SDK call and stamps no id.""" + import flowx.preparer.notifications as nm + + def _boom(*a, **k): + raise AssertionError("SDK must not be called for email") + + monkeypatch.setattr(nm, "_ensure_destination", _boom) + p = _pipeline_with_notify() + cfg = TranslationConfiguration( + notify_destination="email", notify_args={"addresses": "a@x.com"}, notify_events="both" + ) + provisioned, messages = provision_notification_destinations(apply_configuration(p, cfg)) + copy = next(t for t in provisioned.tasks if t.task_key == "load_curated") + assert "destination_id" not in copy.notifications + assert messages == [] + + +def test_resolve_uses_pre_resolved_id_without_sdk(monkeypatch): + """At prepare time a spec carrying a prompt-time destination_id wires directly, + with no further SDK call.""" + import flowx.preparer.notifications as nm + + def _boom(*a, **k): + raise AssertionError("prepare must reuse the prompt-time id, not call the SDK") + + monkeypatch.setattr(nm, "_ensure_destination", _boom) + keys, setup = resolve_task_notifications( + {"destination": "slack", "destination_id": "dest-pre-7", "events": ["on_failure"]} + ) + assert keys == {"webhook_notifications": {"on_failure": [{"id": "dest-pre-7"}]}} + assert setup == [] diff --git a/tests/unit/test_param_dedup.py b/tests/unit/test_param_dedup.py new file mode 100644 index 0000000..2c5680c --- /dev/null +++ b/tests/unit/test_param_dedup.py @@ -0,0 +1,36 @@ +"""Regression: job parameters must not be duplicated through the CLI report path.""" + +from __future__ import annotations + +from flowx.bundler.dab_writer import _build_job_resource, _pipeline_dict_to_workflow + + +def _report(default="us"): + return { + "name": "pipeline_simple", + "parameters": [{"name": "region", "type": "String", "default": default}], + "tasks": [ + { + "name": "Ingest Bronze", + "type": "NotebookActivity", + "task_key": "ingest_bronze", + "notebook_path": "/Shared/ETL/01_ingest_bronze", + "base_parameters": {"region": "@pipeline().parameters.region"}, + }, + ], + } + + +def test_report_path_does_not_duplicate_parameters(): + wf = _pipeline_dict_to_workflow(_report()) + names = [p.get("name") for p in wf.parameters] + assert names == ["region"], f"expected one region parameter, got {names}" + + +def test_build_job_resource_dedupes_parameters(): + wf = _pipeline_dict_to_workflow(_report()) + # even if a caller double-added, the emitted job declares region once + wf.parameters = wf.parameters + wf.parameters + job = _build_job_resource(wf, "pipeline_simple")["resources"]["jobs"]["pipeline_simple"] + names = [p["name"] for p in job["parameters"]] + assert names == ["region"] diff --git a/tests/unit/test_preparers.py b/tests/unit/test_preparers.py index 0be332a..f206c73 100644 --- a/tests/unit/test_preparers.py +++ b/tests/unit/test_preparers.py @@ -138,9 +138,9 @@ def test_prepare_notebook_no_params(self): # No placeholder for an absolute workspace path. assert prepared.notebooks == [] - def test_prepare_notebook_vendors_downloaded_workspace_notebook(self, monkeypatch): + def test_prepare_notebook_downloads_downloaded_workspace_notebook(self, monkeypatch): """When downloads are enabled and the SDK returns content, the workspace notebook - is vendored into src/notebooks/ under the workspace basename, and the task is + is downloaded into src/notebooks/ under the workspace basename, and the task is bound to the default cluster.""" monkeypatch.setattr(workspace_downloader, "_downloads_enabled", True) monkeypatch.setattr( @@ -160,7 +160,7 @@ def test_prepare_notebook_vendors_downloaded_workspace_notebook(self, monkeypatc # skips ../src/ paths because flowx-generated notebooks are # serverless-only; downloaded notebooks need classic compute). assert prepared.task["job_cluster_key"] == "default_cluster" - # Notebook vendored under the workspace basename + # Notebook downloaded under the workspace basename assert len(prepared.notebooks) == 1 assert prepared.notebooks[0].relative_path == "notebooks/transform.py" assert "from /Shared/ETL/transform" in prepared.notebooks[0].content @@ -184,7 +184,7 @@ def test_prepare_notebook_preserves_workspace_basename_verbatim(self, monkeypatc def test_prepare_notebook_falls_back_to_in_place_when_download_fails(self, monkeypatch): """If downloads are enabled but the SDK returns None, behavior matches the - legacy in-place reference (no vendor, no cluster bind in the preparer).""" + legacy in-place reference (no download, no cluster bind in the preparer).""" monkeypatch.setattr(workspace_downloader, "_downloads_enabled", True) monkeypatch.setattr( "flowx.preparer.activity_preparers.notebook.download_notebook", @@ -943,10 +943,50 @@ def test_motif_preparer_registered(self): motif_config={"sink_table": "raw.{schema_name}_{table_name}"}, ) prepared = prepare_activity(activity) - assert "notebook_task" in prepared.task - assert prepared.task["notebook_task"]["notebook_path"].endswith("bulk_ingest.py") - assert len(prepared.notebooks) == 1 - assert "metadata_driven_bulk_copy" in prepared.notebooks[0].content + # Default (non-consolidated) metadata-driven bulk copy now becomes a for_each_task that runs + # one Spark JDBC read per source table. With no resolved lookup_values, a runtime + # control-table lookup task seeds the iteration inputs. + assert "for_each_task" in prepared.task + for_each = prepared.task["for_each_task"] + assert for_each["inputs"] == "{{tasks.bulk_ingest_control_lookup.values.items}}" + assert for_each["task"]["notebook_task"]["base_parameters"]["item"] == "{{input}}" + assert prepared.task["depends_on"] == [{"task_key": "bulk_ingest_control_lookup"}] + assert [t["task_key"] for t in prepared.extra_tasks] == ["bulk_ingest_control_lookup"] + # inner per-item read notebook + the control-lookup seed notebook + assert {nb.relative_path for nb in prepared.notebooks} == { + "notebooks/bulk_ingest_ingest.py", + "notebooks/bulk_ingest_control_lookup.py", + } + + def test_metadata_driven_for_each_uses_static_inputs_when_lookup_values_resolved(self): + """Resolved control rows are inlined as the for_each_task inputs -- no runtime lookup task.""" + import json + + from flowx.models.ir import MotifActivity + + rows = [{"schema_name": "dbo", "table_name": "orders"}, {"schema_name": "dbo", "table_name": "customers"}] + activity = MotifActivity( + **_make_base("Bulk Ingest", "bulk_ingest"), + motif_id="metadata_driven_bulk_copy", + display_name="Metadata-driven bulk copy", + databricks_replacement="for_each_ingestion", + matched_activity_names=["GetTableList", "ForEachTable", "CopyTable"], + source_type_hint="database", + motif_config={"sink_table": "raw.{schema_name}_{table_name}", "copy_scope": "src_db"}, + lookup_values=rows, + ) + prepared = prepare_activity(activity) + for_each = prepared.task["for_each_task"] + assert json.loads(for_each["inputs"]) == rows # inlined literal JSON array + assert prepared.extra_tasks == [] # no control-lookup seed task + assert "depends_on" not in prepared.task or all( + d["task_key"] != "bulk_ingest_control_lookup" for d in prepared.task.get("depends_on", []) + ) + assert [nb.relative_path for nb in prepared.notebooks] == ["notebooks/bulk_ingest_ingest.py"] + # inner read notebook targets the configured sink + secret scope + body = prepared.notebooks[0].content + assert 'dbutils.secrets.get(scope="src_db"' in body + assert "raw.{schema_name}_{table_name}" in body class TestSwitchPreparer: diff --git a/tests/unit/test_profile_report.py b/tests/unit/test_profile_report.py new file mode 100644 index 0000000..e7cc7ef --- /dev/null +++ b/tests/unit/test_profile_report.py @@ -0,0 +1,107 @@ +"""Tests for the profile-phase complexity report (CSV + T-shirt sizing + ARM export).""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +from flowx.models.adf_ast import ( + AdfActivity, + AdfDataset, + AdfDatasetReference, + AdfDefinitions, + AdfLinkedService, + AdfLinkedServiceReference, + AdfPipeline, +) +from flowx.parser.adf_loader import ( + _activity_category, + _complexity_score, + _tshirt_size, + build_profile_rows, + write_pipeline_arm, + write_profile_csv, +) + + +def test_activity_category_ordering(): + assert _activity_category("DatabricksNotebook") == "databricks" + assert _activity_category("ForEach") == "control" + assert _activity_category("SetVariable") == "control" + assert _activity_category("Copy") == "other" + assert _activity_category("WebActivity") == "other" + + +def test_complexity_score_weights_other_highest(): + # 1 native vs 1 other: other must score higher. + native = _complexity_score({"databricks": 1, "control": 0, "other": 0}, 0, 0, 0) + control = _complexity_score({"databricks": 0, "control": 1, "other": 0}, 0, 0, 0) + other = _complexity_score({"databricks": 0, "control": 0, "other": 1}, 0, 0, 0) + assert native < control < other + + +def test_tshirt_size_buckets(): + assert _tshirt_size(2) == "S" + assert _tshirt_size(10) == "M" + assert _tshirt_size(25) == "L" + assert _tshirt_size(40) == "XL" + + +def _pipeline_with_copy() -> AdfDefinitions: + copy = AdfActivity( + name="Load", + type="Copy", + inputs=[AdfDatasetReference(reference_name="src_ds")], + outputs=[AdfDatasetReference(reference_name="dst_ds")], + ) + notebook = AdfActivity( + name="Run", + type="DatabricksNotebook", + linked_service_name=AdfLinkedServiceReference(reference_name="adb_ls"), + ) + pipeline = AdfPipeline(name="p1", activities=[copy, notebook], raw={"name": "p1", "properties": {}}) + return AdfDefinitions( + pipelines=[pipeline], + datasets={ + "src_ds": AdfDataset(name="src_ds", type="AzureSqlTable", properties={}, linked_service_name="sql_ls"), + "dst_ds": AdfDataset(name="dst_ds", type="DelimitedText", properties={}, linked_service_name="adls_ls"), + }, + linked_services={ + "sql_ls": AdfLinkedService(name="sql_ls", type="AzureSqlDatabase", properties={}), + "adls_ls": AdfLinkedService(name="adls_ls", type="AzureBlobFS", properties={}), + "adb_ls": AdfLinkedService(name="adb_ls", type="AzureDatabricks", properties={}), + }, + ) + + +def test_build_profile_rows_counts_datasets_and_linked_services(): + rows = build_profile_rows(_pipeline_with_copy()) + assert len(rows) == 1 + row = rows[0] + assert row["pipeline"] == "p1" + assert row["activities"] == 2 + assert row["datasets"] == 2 # src_ds + dst_ds + # 2 from datasets (sql_ls, adls_ls) + 1 activity-level (adb_ls) + assert row["linked_services"] == 3 + assert row["databricks_native_activities"] == 1 + assert row["other_activities"] == 1 + assert row["complexity_size"] in {"S", "M", "L", "XL"} + + +def test_write_profile_csv_roundtrip(tmp_path: Path): + rows = build_profile_rows(_pipeline_with_copy()) + csv_path = tmp_path / "profile_report.csv" + write_profile_csv(rows, csv_path) + with csv_path.open() as handle: + read_rows = list(csv.DictReader(handle)) + assert read_rows[0]["pipeline"] == "p1" + assert read_rows[0]["activities"] == "2" + + +def test_write_pipeline_arm_emits_verbatim_source(tmp_path: Path): + definitions = _pipeline_with_copy() + written = write_pipeline_arm(definitions, tmp_path) + assert len(written) == 1 + arm = json.loads(written[0].read_text()) + assert arm == {"name": "p1", "properties": {}} diff --git a/tests/unit/test_reporting_coverage.py b/tests/unit/test_reporting_coverage.py new file mode 100644 index 0000000..5c1a8f9 --- /dev/null +++ b/tests/unit/test_reporting_coverage.py @@ -0,0 +1,90 @@ +"""Tests for building per-pipeline coverage rows from migration metadata.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +from flowx.reporting.coverage import build_coverage_rows + + +def _write_metadata(tmp_path: Path) -> Path: + md = tmp_path / "metadata" + md.mkdir() + inventory = { + "pipelines": [ + { + "name": "p_alpha", + "activities": [ + {"name": "a1", "type": "DatabricksNotebook", "strategy": "deterministic"}, + {"name": "a2", "type": "Copy", "strategy": "deterministic"}, + {"name": "a3", "type": "ExecuteDataFlow", "strategy": "agentic"}, + {"name": "a4", "type": "Custom", "strategy": "unsupported"}, + ], + }, + { + "name": "p_beta", + "activities": [ + {"name": "b1", "type": "DatabricksNotebook", "strategy": "deterministic"}, + ], + }, + ], + "summary": {"pipeline_count": 2}, + } + (md / "inventory.json").write_text(json.dumps(inventory), encoding="utf-8") + rows = [ + { + "pipeline": "p_alpha", + "activities": 4, + "datasets": 2, + "linked_services": 1, + "collapsible_patterns": 1, + "databricks_native_activities": 1, + "control_flow_activities": 0, + "other_activities": 3, + "complexity_score": 12, + "complexity_size": "M", + }, + { + "pipeline": "p_beta", + "activities": 1, + "datasets": 0, + "linked_services": 1, + "collapsible_patterns": 0, + "databricks_native_activities": 1, + "control_flow_activities": 0, + "other_activities": 0, + "complexity_score": 2, + "complexity_size": "S", + }, + ] + with (md / "profile_report.csv").open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + return md + + +def test_build_coverage_rows_joins_inventory_and_csv(tmp_path: Path): + rows = build_coverage_rows(_write_metadata(tmp_path)) + assert [r["pipeline"] for r in rows] == ["p_alpha", "p_beta"] # sorted by name + alpha = rows[0] + assert alpha["activities"] == 4 + assert alpha["deterministic_activities"] == 2 + assert alpha["agentic_activities"] == 1 + assert alpha["unsupported_activities"] == 1 + # coverage = (det + agentic) / total = 3/4 = 75.0 + assert alpha["coverage_pct"] == 75.0 + # complexity columns come from the CSV + assert alpha["datasets"] == 2 and alpha["linked_services"] == 1 + assert alpha["collapsible_patterns"] == 1 and alpha["complexity_size"] == "M" + + +def test_build_coverage_rows_full_coverage_and_missing_csv(tmp_path: Path): + md = _write_metadata(tmp_path) + (md / "profile_report.csv").unlink() # CSV optional -> complexity columns default + rows = {r["pipeline"]: r for r in build_coverage_rows(md)} + beta = rows["p_beta"] + assert beta["coverage_pct"] == 100.0 # 1/1 deterministic + assert beta["datasets"] == 0 and beta["complexity_size"] == "" # defaulted, no CSV diff --git a/tests/unit/test_reporting_dashboard.py b/tests/unit/test_reporting_dashboard.py new file mode 100644 index 0000000..319e75d --- /dev/null +++ b/tests/unit/test_reporting_dashboard.py @@ -0,0 +1,101 @@ +"""Tests for the coverage dashboard builder + installer.""" + +from __future__ import annotations + +import json + +import pytest + +from flowx.reporting import dashboard as D + + +def test_build_serialized_dashboard_injects_table_and_is_valid_json(): + serialized = D.build_serialized_dashboard("cat.sch.results") + spec = json.loads(serialized) + assert "{{RESULTS_TABLE}}" not in serialized + # every dataset query references the fully-qualified table + joined = " ".join(line for ds in spec["datasets"] for line in ds["queryLines"]) + assert "cat.sch.results" in joined + assert spec["pages"][0]["pageType"] == "PAGE_TYPE_CANVAS" + # widget field names match their dataset fields (counter references a real column) + widget_names = {w["widget"]["name"] for w in spec["pages"][0]["layout"]} + assert {"kpi-coverage", "by-size", "coverage-trend", "pipeline-table"} <= widget_names + + +def test_build_serialized_dashboard_requires_table(): + with pytest.raises(ValueError): + D.build_serialized_dashboard("") + + +class _Created: + dashboard_id = "dash-123" + + +class _FakeLakeview: + def __init__(self): + self.created = None + self.published = None + + def create(self, dashboard): + self.created = dashboard + return _Created() + + def publish(self, dashboard_id, warehouse_id): + self.published = (dashboard_id, warehouse_id) + + +class _FakeWarehousesAPI: + def list(self): + class _W: + id = "wh1" + name = "wh1" + state = "RUNNING" + enable_serverless_compute = True + warehouse_type = "PRO" + + return [_W()] + + +class _Me: + user_name = "greg@databricks.com" + + +class _FakeCurrentUser: + def me(self): + return _Me() + + +class _FakeConfig: + host = "https://example.cloud.databricks.com" + + +class _FakeClient: + def __init__(self): + self.lakeview = _FakeLakeview() + self.warehouses = _FakeWarehousesAPI() + self.current_user = _FakeCurrentUser() + self.config = _FakeConfig() + + +def test_install_dashboard_creates_and_publishes(): + client = _FakeClient() + dashboard_id, url = D.install_dashboard("cat.sch.results", client=client) + assert dashboard_id == "dash-123" + assert url == "https://example.cloud.databricks.com/sql/dashboardsv3/dash-123" + # created with resolved warehouse, table-bound spec, and default parent path = user home + created = client.lakeview.created + assert created.warehouse_id == "wh1" + assert created.parent_path == "/Workspace/Users/greg@databricks.com" + assert "cat.sch.results" in created.serialized_dashboard + assert client.lakeview.published == ("dash-123", "wh1") + + +def test_install_dashboard_respects_overrides(): + client = _FakeClient() + D.install_dashboard( + "cat.sch.results", warehouse_id="whX", display_name="My Dash", parent_path="/Workspace/Shared", client=client + ) + created = client.lakeview.created + assert created.warehouse_id == "whX" + assert created.display_name == "My Dash" + assert created.parent_path == "/Workspace/Shared" diff --git a/tests/unit/test_reporting_results.py b/tests/unit/test_reporting_results.py new file mode 100644 index 0000000..6351ecf --- /dev/null +++ b/tests/unit/test_reporting_results.py @@ -0,0 +1,180 @@ +"""Tests for the UC-table results writer (SQL builders, warehouse resolution, write).""" + +from __future__ import annotations + +import csv +import json +import uuid +from pathlib import Path + +import pytest + +from flowx.reporting import results as R + + +def test_create_table_sql_has_run_metadata_and_all_columns(): + sql = R.build_create_table_sql("cat.sch.tbl") + assert sql.startswith("CREATE TABLE IF NOT EXISTS cat.sch.tbl") + assert "run_id STRING" in sql + assert "run_date TIMESTAMP" in sql + assert "run_by STRING" in sql + assert "coverage_pct DOUBLE" in sql + assert "complexity_size STRING" in sql + + +def test_insert_sql_stamps_run_metadata_and_escapes(): + rows = [ + { + "pipeline": "p1", + "activities": 3, + "datasets": 1, + "linked_services": 0, + "collapsible_patterns": 0, + "databricks_native_activities": 1, + "control_flow_activities": 0, + "other_activities": 2, + "deterministic_activities": 2, + "agentic_activities": 1, + "unsupported_activities": 0, + "coverage_pct": 100.0, + "complexity_score": 7, + "complexity_size": "M", + }, + { + "pipeline": "O'Brien's pipe", + "activities": 1, + "datasets": 0, + "linked_services": 0, + "collapsible_patterns": 0, + "databricks_native_activities": 0, + "control_flow_activities": 0, + "other_activities": 1, + "deterministic_activities": 0, + "agentic_activities": 0, + "unsupported_activities": 1, + "coverage_pct": 0.0, + "complexity_score": 3, + "complexity_size": "S", + }, + ] + run_id = "abc-123" + sql = R.build_insert_sql("cat.sch.tbl", rows, run_id) + assert "INSERT INTO cat.sch.tbl (run_id, run_date, run_by," in sql + # run metadata: literal run_id + SQL functions on every row + assert sql.count("'abc-123'") == 2 + assert sql.count("CURRENT_TIMESTAMP()") == 2 + assert sql.count("CURRENT_USER()") == 2 + # apostrophe escaped by doubling + assert "'O''Brien''s pipe'" in sql + # numeric + float rendered unquoted + assert "100.0" in sql + + +class _FakeWarehouse: + def __init__(self, id, state, serverless=False): + self.id = id + self.name = id + self.state = state + self.enable_serverless_compute = serverless + self.warehouse_type = "PRO" + + +class _FakeWarehousesAPI: + def __init__(self, items): + self._items = items + + def list(self): + return list(self._items) + + +class _FakeStmtAPI: + def __init__(self): + self.statements = [] + + def execute_statement(self, statement, warehouse_id, wait_timeout=None): + self.statements.append((warehouse_id, statement)) + + class _Resp: + class status: + state = "SUCCEEDED" + + return _Resp() + + +class _FakeClient: + def __init__(self, warehouses): + self.warehouses = _FakeWarehousesAPI(warehouses) + self.statement_execution = _FakeStmtAPI() + + +def test_resolve_warehouse_prefers_running_serverless(): + client = _FakeClient( + [ + _FakeWarehouse("w_stopped", "STOPPED", serverless=True), + _FakeWarehouse("w_running_classic", "RUNNING", serverless=False), + _FakeWarehouse("w_running_serverless", "RUNNING", serverless=True), + ] + ) + assert R.resolve_warehouse_id(client) == "w_running_serverless" + # explicit id passes through + assert R.resolve_warehouse_id(client, "explicit") == "explicit" + + +def test_resolve_warehouse_none_raises(): + with pytest.raises(RuntimeError): + R.resolve_warehouse_id(_FakeClient([])) + + +def _metadata(tmp_path: Path) -> Path: + md = tmp_path / "metadata" + md.mkdir() + inv = { + "pipelines": [ + {"name": "p1", "activities": [{"name": "a", "type": "DatabricksNotebook", "strategy": "deterministic"}]} + ] + } + (md / "inventory.json").write_text(json.dumps(inv)) + with (md / "profile_report.csv").open("w", newline="") as fh: + w = csv.DictWriter( + fh, + fieldnames=[ + "pipeline", + "activities", + "datasets", + "linked_services", + "collapsible_patterns", + "databricks_native_activities", + "control_flow_activities", + "other_activities", + "complexity_score", + "complexity_size", + ], + ) + w.writeheader() + w.writerow( + { + "pipeline": "p1", + "activities": 1, + "datasets": 0, + "linked_services": 1, + "collapsible_patterns": 0, + "databricks_native_activities": 1, + "control_flow_activities": 0, + "other_activities": 0, + "complexity_score": 2, + "complexity_size": "S", + } + ) + return md + + +def test_write_results_executes_create_then_insert(tmp_path: Path): + client = _FakeClient([_FakeWarehouse("wh1", "RUNNING", serverless=True)]) + run_id, rows = R.write_results(_metadata(tmp_path), "cat.sch.tbl", client=client) + assert rows == 1 + uuid.UUID(run_id) # valid uuid + stmts = client.statement_execution.statements + assert len(stmts) == 2 + assert stmts[0][0] == "wh1" and stmts[0][1].startswith("CREATE TABLE IF NOT EXISTS") + assert stmts[1][1].startswith("INSERT INTO cat.sch.tbl") + assert run_id in stmts[1][1] diff --git a/tests/unit/test_until_agentic_handler.py b/tests/unit/test_until_agentic_handler.py new file mode 100644 index 0000000..aa97a05 --- /dev/null +++ b/tests/unit/test_until_agentic_handler.py @@ -0,0 +1,71 @@ +"""#1: Until (incl. nested) must surface as an agentic gap carrying the full ARM JSON.""" + +from __future__ import annotations + +from flowx.models.adf_ast import AdfDefinitions +from flowx.models.ir import PlaceholderActivity +from flowx.parser.adf_loader import _parse_pipeline_json +from flowx.translator.engine import translate_pipeline + +_DEFS = AdfDefinitions(pipelines=[], datasets={}, linked_services={}, triggers=[]) + +_PIPELINE = { + "name": "p", + "properties": { + "activities": [ + { + "name": "Gate", + "type": "IfCondition", + "typeProperties": { + "expression": {"value": "@equals(1, 1)", "type": "Expression"}, + "ifTrueActivities": [ + { + "name": "Poll Until Ready", + "type": "Until", + "typeProperties": { + "expression": {"value": "@equals(variables('s'), 'done')", "type": "Expression"}, + "timeout": "0.01:00:00", + "activities": [ + {"name": "Wait A Bit", "type": "Wait", "typeProperties": {"waitTimeInSeconds": 5}} + ], + }, + } + ], + }, + } + ] + }, +} + + +def test_nested_until_gap_carries_full_arm_json(): + adf = _parse_pipeline_json(_PIPELINE, fallback_name="p") + report = translate_pipeline(adf, _DEFS) + until_gaps = [g for g in report.gaps if g.activity_type == "Until"] + assert len(until_gaps) == 1, "nested Until must be reported as a gap" + raw = until_gaps[0].raw_definition + assert raw is not None and raw.get("type") == "Until" + # full ARM JSON, not just typeProperties: name + nested loop body present + assert raw.get("name") == "Poll Until Ready" + assert raw["typeProperties"]["activities"][0]["name"] == "Wait A Bit" + assert until_gaps[0].recommended_skill == "adf-to-databricks:adf-pipeline-converter" + + +def test_until_placeholder_ir_node_carries_arm_json(): + adf = _parse_pipeline_json(_PIPELINE, fallback_name="p") + report = translate_pipeline(adf, _DEFS) + + def _find(tasks): + for t in tasks: + if isinstance(t, PlaceholderActivity) and t.original_type == "Until": + return t + for attr in ("inner_activities", "if_true_activities", "if_false_activities"): + found = _find(getattr(t, attr, []) or []) + if found: + return found + return None + + ph = _find(report.pipeline.tasks) + assert ph is not None and ph.raw_definition is not None + assert ph.raw_definition.get("type") == "Until" + assert ph.agentic_skill == "adf-to-databricks:adf-pipeline-converter" diff --git a/tests/unit/test_web_body_and_param_defaults.py b/tests/unit/test_web_body_and_param_defaults.py new file mode 100644 index 0000000..48e1e48 --- /dev/null +++ b/tests/unit/test_web_body_and_param_defaults.py @@ -0,0 +1,71 @@ +"""Regression tests for #2 parsing fixes: web-activity body expressions and +@utcNow pipeline-parameter defaults.""" + +from __future__ import annotations + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions, AdfParameter, AdfPipeline +from flowx.models.ir import TranslationContext +from flowx.preparer.code_generator import generate_web_activity_notebook +from flowx.translator.activity_translators import web_activity +from flowx.translator.engine import translate_pipeline + +_DEFS = AdfDefinitions(pipelines=[], datasets={}, linked_services={}, triggers=[]) + + +def _web(type_properties: dict, context: TranslationContext): + activity = AdfActivity(name="Notify", type="WebActivity", type_properties=type_properties) + return web_activity.translate(activity, {"name": "Notify", "task_key": "notify"}, context, _DEFS) + + +def test_nested_concat_variables_body_is_lowered_to_python(): + ctx = TranslationContext().with_variable("batchId", "_init_batchId") + ir = _web( + { + "method": "POST", + "url": "https://example.com/hook", + "body": {"text": {"value": "@concat('batch ', variables('batchId'))", "type": "Expression"}}, + }, + ctx, + ) + assert ir.body_code is not None + nb = generate_web_activity_notebook(ir) + assert "@concat" not in nb and "@variables" not in nb + assert "dbutils.widgets.get('batchId')" in nb + assert "'batch ' +" in nb # concatenation, not raw token + + +def test_bare_variables_body_reads_from_widget_and_binds_it(): + ctx = TranslationContext().with_variable("statusMessage", "set_msg") + ir = _web( + { + "method": "POST", + "url": "https://example.com/hook", + "body": {"text": {"value": "@variables('statusMessage')", "type": "Expression"}}, + }, + ctx, + ) + nb = generate_web_activity_notebook(ir) + assert 'dbutils.widgets.get("statusMessage")' in nb + # the dab ref is threaded so the preparer can bind it in base_parameters + assert ir.body_required_parameters.get("statusMessage") == "{{tasks.set_msg.values.statusMessage}}" + + +def test_literal_body_unchanged(): + ir = _web( + {"method": "POST", "url": "https://x", "body": {"status": "completed"}}, + TranslationContext(), + ) + assert ir.body_code is None # pure literal -> generator renders directly + nb = generate_web_activity_notebook(ir) + assert "completed" in nb + + +def test_utcnow_parameter_default_resolves_to_dab_ref(): + pipeline = AdfPipeline( + name="p", + activities=[AdfActivity(name="W", type="Wait", type_properties={"waitTimeInSeconds": 1})], + parameters={"runDate": AdfParameter(type="String", default_value="@utcNow('yyyy-MM-dd')")}, + ) + report = translate_pipeline(pipeline, _DEFS) + run_date = next(p for p in report.pipeline.parameters if p["name"] == "runDate") + assert run_date["default"] == "{{job.start_time.iso_date}}" diff --git a/tests/unit/test_workspace_downloader.py b/tests/unit/test_workspace_downloader.py index da35d7f..69d3483 100644 --- a/tests/unit/test_workspace_downloader.py +++ b/tests/unit/test_workspace_downloader.py @@ -90,10 +90,38 @@ def test_returns_true_when_host_and_token_set(self, monkeypatch): monkeypatch.setenv("DATABRICKS_TOKEN", "dapi-abc") assert auth_available() is True + def test_returns_true_when_oauth_m2m_env_set(self, monkeypatch): + # The MCP path: flowx hosted as a Databricks App injects the service + # principal's OAuth client id/secret (no PAT, no profile). + monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False) + monkeypatch.delenv("DATABRICKS_TOKEN", raising=False) + monkeypatch.setenv("DATABRICKS_HOST", "https://example.cloud.databricks.com") + monkeypatch.setenv("DATABRICKS_CLIENT_ID", "sp-client-id") + monkeypatch.setenv("DATABRICKS_CLIENT_SECRET", "sp-secret") + monkeypatch.setattr(workspace_downloader, "_local_workspace_accessible", lambda: False) + monkeypatch.setattr(workspace_downloader, "_list_profiles", lambda: []) + assert auth_available() is True + def test_returns_false_when_no_env_and_no_profiles(self, monkeypatch): monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False) monkeypatch.delenv("DATABRICKS_HOST", raising=False) monkeypatch.delenv("DATABRICKS_TOKEN", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + monkeypatch.setattr(workspace_downloader, "_local_workspace_accessible", lambda: False) + monkeypatch.setattr(workspace_downloader, "_is_databricks_runtime", lambda: False) + monkeypatch.setattr(workspace_downloader, "_list_profiles", lambda: []) + assert auth_available() is False + + def test_returns_false_when_host_set_without_token_or_client_creds(self, monkeypatch): + # HOST alone must not satisfy the check (incomplete credentials). + monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False) + monkeypatch.delenv("DATABRICKS_TOKEN", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + monkeypatch.setenv("DATABRICKS_HOST", "https://example.cloud.databricks.com") + monkeypatch.setattr(workspace_downloader, "_local_workspace_accessible", lambda: False) + monkeypatch.setattr(workspace_downloader, "_is_databricks_runtime", lambda: False) monkeypatch.setattr(workspace_downloader, "_list_profiles", lambda: []) assert auth_available() is False diff --git a/uv.lock b/uv.lock index 6104941..ed81fa7 100644 --- a/uv.lock +++ b/uv.lock @@ -2,22 +2,53 @@ version = 1 revision = 3 requires-python = ">=3.12" +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", upload-time = "2026-03-24T12:59:08.246Z" }, +] + [[package]] name = "argcomplete" version = "3.6.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/38/61/0b9ae6399dd4a58d8c1b1dc5a27d6f2808023d0b5dd3104bb99f45a33ff6/argcomplete-3.6.3.tar.gz", hash = "sha256:62e8ed4fd6a45864acc8235409461b72c9a28ee785a2011cc5eb78318786c89c", size = 73754, upload-time = "2025-10-20T03:33:34.741Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/61/0b9ae6399dd4a58d8c1b1dc5a27d6f2808023d0b5dd3104bb99f45a33ff6/argcomplete-3.6.3.tar.gz", hash = "sha256:62e8ed4fd6a45864acc8235409461b72c9a28ee785a2011cc5eb78318786c89c", upload-time = "2025-10-20T03:33:34.741Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/74/f5/9373290775639cb67a2fce7f629a1c240dce9f12fe927bc32b2736e16dfc/argcomplete-3.6.3-py3-none-any.whl", hash = "sha256:f5007b3a600ccac5d25bbce33089211dfd49eab4a7718da3f10e3082525a92ce", size = 43846, upload-time = "2025-10-20T03:33:33.021Z" }, + { url = "https://files.pythonhosted.org/packages/74/f5/9373290775639cb67a2fce7f629a1c240dce9f12fe927bc32b2736e16dfc/argcomplete-3.6.3-py3-none-any.whl", hash = "sha256:f5007b3a600ccac5d25bbce33089211dfd49eab4a7718da3f10e3082525a92ce", upload-time = "2025-10-20T03:33:33.021Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", upload-time = "2026-03-19T14:22:23.645Z" }, ] [[package]] name = "certifi" version = "2026.5.20" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", upload-time = "2026-05-20T11:46:50.073Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", upload-time = "2026-05-20T11:46:48.578Z" }, ] [[package]] @@ -27,220 +58,232 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, - { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, - { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, - { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, - { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, - { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, - { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, - { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, - { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, - { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, - { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, - { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, - { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, - { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, - { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, - { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, - { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, - { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, - { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, - { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, - { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, - { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, - { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, - { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, - { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, - { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, - { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, - { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, - { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, - { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, - { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, - { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, - { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, - { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, - { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, - { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, - { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, - { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, - { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, - { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, - { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, - { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, - { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", upload-time = "2025-09-08T23:22:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", upload-time = "2025-09-08T23:23:02.231Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", upload-time = "2025-09-08T23:23:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", upload-time = "2025-09-08T23:23:04.792Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", upload-time = "2025-09-08T23:23:06.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", upload-time = "2025-09-08T23:23:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", upload-time = "2025-09-08T23:23:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", upload-time = "2025-09-08T23:23:10.928Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", upload-time = "2025-09-08T23:23:12.42Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", upload-time = "2025-09-08T23:23:18.087Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", upload-time = "2025-09-08T23:23:19.622Z" }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", upload-time = "2025-09-08T23:23:20.853Z" }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", upload-time = "2025-09-08T23:23:22.08Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", upload-time = "2025-09-08T23:23:23.314Z" }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", upload-time = "2025-09-08T23:23:24.541Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", upload-time = "2025-09-08T23:23:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", upload-time = "2025-09-08T23:23:27.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", upload-time = "2025-09-08T23:23:29.347Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", upload-time = "2025-09-08T23:23:30.63Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", upload-time = "2025-09-08T23:23:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", upload-time = "2025-09-08T23:23:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", upload-time = "2025-09-08T23:23:34.495Z" }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", upload-time = "2025-09-08T23:23:36.096Z" }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", upload-time = "2025-09-08T23:23:37.328Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", upload-time = "2025-09-08T23:23:38.945Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", upload-time = "2025-09-08T23:23:43.004Z" }, ] [[package]] name = "charset-normalizer" version = "3.4.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, - { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, - { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, - { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, - { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, - { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, - { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, - { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, - { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, - { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, - { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, - { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, - { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, - { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, - { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, - { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, - { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, - { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, - { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, - { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, - { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, - { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, - { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, - { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, - { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, - { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, - { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, - { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, - { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, - { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, - { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, - { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, - { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, - { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, - { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, - { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, - { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, - { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, - { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, - { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, - { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, - { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, - { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, - { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, - { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, - { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, - { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, - { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, - { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, - { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, - { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, - { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, - { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, - { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, - { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, - { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, - { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, - { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, - { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, - { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, - { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", upload-time = "2026-04-02T09:26:24.331Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", upload-time = "2026-04-02T09:26:25.568Z" }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", upload-time = "2026-04-02T09:26:26.865Z" }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", upload-time = "2026-04-02T09:26:28.044Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", upload-time = "2026-04-02T09:26:29.239Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", upload-time = "2026-04-02T09:26:30.5Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", upload-time = "2026-04-02T09:26:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", upload-time = "2026-04-02T09:26:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", upload-time = "2026-04-02T09:26:34.845Z" }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", upload-time = "2026-04-02T09:26:36.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", upload-time = "2026-04-02T09:26:37.672Z" }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", upload-time = "2026-04-02T09:26:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", upload-time = "2026-04-02T09:26:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", upload-time = "2026-04-02T09:26:41.416Z" }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", upload-time = "2026-04-02T09:26:42.554Z" }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", upload-time = "2026-04-02T09:26:44.075Z" }, + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", upload-time = "2026-04-02T09:26:45.198Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", upload-time = "2026-04-02T09:26:46.824Z" }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", upload-time = "2026-04-02T09:26:48.397Z" }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", upload-time = "2026-04-02T09:26:49.684Z" }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", upload-time = "2026-04-02T09:26:50.915Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", upload-time = "2026-04-02T09:26:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", upload-time = "2026-04-02T09:26:53.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", upload-time = "2026-04-02T09:26:54.975Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", upload-time = "2026-04-02T09:26:56.303Z" }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", upload-time = "2026-04-02T09:26:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", upload-time = "2026-04-02T09:26:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", upload-time = "2026-04-02T09:27:00.437Z" }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", upload-time = "2026-04-02T09:27:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", upload-time = "2026-04-02T09:27:03.192Z" }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", upload-time = "2026-04-02T09:27:04.454Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", upload-time = "2026-04-02T09:27:05.971Z" }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", upload-time = "2026-04-02T09:27:07.194Z" }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", upload-time = "2026-04-02T09:27:08.749Z" }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", upload-time = "2026-04-02T09:27:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", upload-time = "2026-04-02T09:27:11.175Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", upload-time = "2026-04-02T09:27:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", upload-time = "2026-04-02T09:27:13.721Z" }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", upload-time = "2026-04-02T09:27:15.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", upload-time = "2026-04-02T09:27:16.834Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", upload-time = "2026-04-02T09:27:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", upload-time = "2026-04-02T09:27:19.445Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", upload-time = "2026-04-02T09:27:20.79Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", upload-time = "2026-04-02T09:27:22.063Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", upload-time = "2026-04-02T09:27:23.486Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", upload-time = "2026-04-02T09:27:25.146Z" }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", upload-time = "2026-04-02T09:27:26.642Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", upload-time = "2026-04-02T09:27:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", upload-time = "2026-04-02T09:27:29.474Z" }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", upload-time = "2026-04-02T09:27:30.793Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", upload-time = "2026-04-02T09:27:32.44Z" }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", upload-time = "2026-04-02T09:27:34.03Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", upload-time = "2026-04-02T09:27:35.369Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", upload-time = "2026-04-02T09:27:36.661Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", upload-time = "2026-04-02T09:27:38.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", upload-time = "2026-04-02T09:27:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", upload-time = "2026-04-02T09:27:40.722Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", upload-time = "2026-04-02T09:27:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", upload-time = "2026-04-02T09:27:43.924Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", upload-time = "2026-04-02T09:27:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", upload-time = "2026-04-02T09:27:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", upload-time = "2026-04-02T09:27:48.053Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", upload-time = "2026-04-02T09:27:49.795Z" }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", upload-time = "2026-04-02T09:27:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", upload-time = "2026-04-02T09:28:37.794Z" }, +] + +[[package]] +name = "click" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", upload-time = "2026-05-22T04:08:37.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", upload-time = "2026-05-22T04:08:35.26Z" }, ] [[package]] name = "colorama" version = "0.4.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", upload-time = "2022-10-25T02:36:20.889Z" }, ] [[package]] name = "coverage" version = "7.13.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", size = 219554, upload-time = "2026-03-17T10:30:42.208Z" }, - { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", size = 219908, upload-time = "2026-03-17T10:30:43.906Z" }, - { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", size = 251419, upload-time = "2026-03-17T10:30:45.545Z" }, - { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", size = 254159, upload-time = "2026-03-17T10:30:47.204Z" }, - { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", size = 255270, upload-time = "2026-03-17T10:30:48.812Z" }, - { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", size = 257538, upload-time = "2026-03-17T10:30:50.77Z" }, - { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", size = 251821, upload-time = "2026-03-17T10:30:52.5Z" }, - { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", size = 253191, upload-time = "2026-03-17T10:30:54.543Z" }, - { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", size = 251337, upload-time = "2026-03-17T10:30:56.663Z" }, - { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", size = 255404, upload-time = "2026-03-17T10:30:58.427Z" }, - { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", size = 250903, upload-time = "2026-03-17T10:31:00.093Z" }, - { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", size = 252780, upload-time = "2026-03-17T10:31:01.916Z" }, - { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", size = 222093, upload-time = "2026-03-17T10:31:03.642Z" }, - { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", size = 222900, upload-time = "2026-03-17T10:31:05.651Z" }, - { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", size = 221515, upload-time = "2026-03-17T10:31:07.293Z" }, - { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" }, - { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" }, - { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" }, - { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" }, - { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" }, - { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" }, - { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" }, - { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" }, - { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" }, - { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" }, - { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" }, - { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" }, - { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" }, - { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" }, - { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" }, - { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" }, - { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" }, - { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" }, - { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" }, - { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" }, - { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" }, - { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" }, - { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" }, - { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" }, - { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" }, - { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" }, - { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" }, - { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" }, - { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" }, - { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" }, - { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" }, - { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" }, - { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" }, - { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" }, - { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" }, - { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" }, - { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" }, - { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" }, - { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" }, - { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" }, - { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" }, - { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" }, - { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" }, - { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" }, - { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" }, - { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" }, - { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" }, - { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" }, - { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" }, - { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" }, - { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" }, - { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" }, - { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" }, - { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" }, - { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" }, - { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" }, - { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", upload-time = "2026-03-17T10:33:18.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", upload-time = "2026-03-17T10:30:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", upload-time = "2026-03-17T10:30:43.906Z" }, + { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", upload-time = "2026-03-17T10:30:45.545Z" }, + { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", upload-time = "2026-03-17T10:30:47.204Z" }, + { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", upload-time = "2026-03-17T10:30:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", upload-time = "2026-03-17T10:30:50.77Z" }, + { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", upload-time = "2026-03-17T10:30:52.5Z" }, + { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", upload-time = "2026-03-17T10:30:54.543Z" }, + { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", upload-time = "2026-03-17T10:30:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", upload-time = "2026-03-17T10:30:58.427Z" }, + { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", upload-time = "2026-03-17T10:31:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", upload-time = "2026-03-17T10:31:01.916Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", upload-time = "2026-03-17T10:31:03.642Z" }, + { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", upload-time = "2026-03-17T10:31:05.651Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", upload-time = "2026-03-17T10:31:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", upload-time = "2026-03-17T10:31:09.045Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", upload-time = "2026-03-17T10:31:10.708Z" }, + { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", upload-time = "2026-03-17T10:31:12.392Z" }, + { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", upload-time = "2026-03-17T10:31:14.247Z" }, + { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", upload-time = "2026-03-17T10:31:16.193Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", upload-time = "2026-03-17T10:31:17.89Z" }, + { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", upload-time = "2026-03-17T10:31:19.605Z" }, + { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", upload-time = "2026-03-17T10:31:21.312Z" }, + { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", upload-time = "2026-03-17T10:31:23.565Z" }, + { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", upload-time = "2026-03-17T10:31:25.58Z" }, + { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", upload-time = "2026-03-17T10:31:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", upload-time = "2026-03-17T10:31:29.472Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", upload-time = "2026-03-17T10:31:31.526Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", upload-time = "2026-03-17T10:31:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", upload-time = "2026-03-17T10:31:35.445Z" }, + { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", upload-time = "2026-03-17T10:31:37.184Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", upload-time = "2026-03-17T10:31:39.245Z" }, + { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", upload-time = "2026-03-17T10:31:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", upload-time = "2026-03-17T10:31:43.724Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", upload-time = "2026-03-17T10:31:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", upload-time = "2026-03-17T10:31:48.293Z" }, + { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", upload-time = "2026-03-17T10:31:50.125Z" }, + { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", upload-time = "2026-03-17T10:31:51.961Z" }, + { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", upload-time = "2026-03-17T10:31:53.872Z" }, + { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", upload-time = "2026-03-17T10:31:55.997Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", upload-time = "2026-03-17T10:31:58.308Z" }, + { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", upload-time = "2026-03-17T10:32:00.141Z" }, + { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", upload-time = "2026-03-17T10:32:02.246Z" }, + { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", upload-time = "2026-03-17T10:32:04.416Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", upload-time = "2026-03-17T10:32:06.65Z" }, + { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", upload-time = "2026-03-17T10:32:08.589Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", upload-time = "2026-03-17T10:32:10.507Z" }, + { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", upload-time = "2026-03-17T10:32:12.41Z" }, + { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", upload-time = "2026-03-17T10:32:14.449Z" }, + { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", upload-time = "2026-03-17T10:32:16.56Z" }, + { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", upload-time = "2026-03-17T10:32:19.004Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", upload-time = "2026-03-17T10:32:21.344Z" }, + { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", upload-time = "2026-03-17T10:32:23.506Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", upload-time = "2026-03-17T10:32:25.516Z" }, + { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", upload-time = "2026-03-17T10:32:27.944Z" }, + { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", upload-time = "2026-03-17T10:32:29.914Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", upload-time = "2026-03-17T10:32:31.981Z" }, + { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", upload-time = "2026-03-17T10:32:34.233Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", upload-time = "2026-03-17T10:32:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", upload-time = "2026-03-17T10:32:38.736Z" }, + { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", upload-time = "2026-03-17T10:32:41.196Z" }, + { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", upload-time = "2026-03-17T10:32:43.204Z" }, + { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", upload-time = "2026-03-17T10:32:45.514Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", upload-time = "2026-03-17T10:32:47.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", upload-time = "2026-03-17T10:32:50.1Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", upload-time = "2026-03-17T10:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", upload-time = "2026-03-17T10:32:54.544Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", upload-time = "2026-03-17T10:32:56.803Z" }, + { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", upload-time = "2026-03-17T10:32:59.466Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", upload-time = "2026-03-17T10:33:01.847Z" }, + { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", upload-time = "2026-03-17T10:33:03.945Z" }, + { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", upload-time = "2026-03-17T10:33:06.285Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", upload-time = "2026-03-17T10:33:08.756Z" }, + { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", upload-time = "2026-03-17T10:33:11.174Z" }, + { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", upload-time = "2026-03-17T10:33:13.466Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", upload-time = "2026-03-17T10:33:15.691Z" }, ] [[package]] @@ -250,51 +293,101 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", size = 832984, upload-time = "2026-05-04T22:59:38.133Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", size = 8001587, upload-time = "2026-05-04T22:57:36.803Z" }, - { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", size = 4690433, upload-time = "2026-05-04T22:57:40.373Z" }, - { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", size = 4710620, upload-time = "2026-05-04T22:57:42.935Z" }, - { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", size = 4696283, upload-time = "2026-05-04T22:57:45.294Z" }, - { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", size = 5296573, upload-time = "2026-05-04T22:57:47.933Z" }, - { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", size = 4743677, upload-time = "2026-05-04T22:57:50.067Z" }, - { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", size = 4330808, upload-time = "2026-05-04T22:57:52.301Z" }, - { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", size = 4695941, upload-time = "2026-05-04T22:57:54.603Z" }, - { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", size = 5252579, upload-time = "2026-05-04T22:57:57.207Z" }, - { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", size = 4743326, upload-time = "2026-05-04T22:57:59.535Z" }, - { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", size = 4826672, upload-time = "2026-05-04T22:58:01.996Z" }, - { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", size = 4972574, upload-time = "2026-05-04T22:58:03.968Z" }, - { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", size = 3294868, upload-time = "2026-05-04T22:58:06.467Z" }, - { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", size = 3817107, upload-time = "2026-05-04T22:58:08.845Z" }, - { url = "https://files.pythonhosted.org/packages/6b/84/70e3feea9feea87fd7cbe77efb2712ae1e3e6edf10749dc6e95f4e60e455/cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec", size = 7986556, upload-time = "2026-05-04T22:58:11.172Z" }, - { url = "https://files.pythonhosted.org/packages/89/6e/18e07a618bb5442ba10cf4df16e99c071365528aa570dfcb8c02e25a303b/cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18", size = 4684776, upload-time = "2026-05-04T22:58:13.712Z" }, - { url = "https://files.pythonhosted.org/packages/be/6a/4ea3b4c6c6759794d5ee2103c304a5076dc4b19ae1f9fe47dba439e159e9/cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20", size = 4698121, upload-time = "2026-05-04T22:58:16.448Z" }, - { url = "https://files.pythonhosted.org/packages/2f/59/6ff6ad6cae03bb887da2a5860b2c9805f8dac969ef01ce563336c49bd1d1/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff", size = 4690042, upload-time = "2026-05-04T22:58:18.544Z" }, - { url = "https://files.pythonhosted.org/packages/ca/b4/fc334ed8cfd705aca282fe4d8f5ae64a8e0f74932e9feecb344610cf6e4d/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c", size = 5282526, upload-time = "2026-05-04T22:58:20.75Z" }, - { url = "https://files.pythonhosted.org/packages/11/08/9f8c5386cc4cd90d8255c7cdd0f5baf459a08502a09de30dc51f553d38dc/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db", size = 4733116, upload-time = "2026-05-04T22:58:23.627Z" }, - { url = "https://files.pythonhosted.org/packages/b8/77/99307d7574045699f8805aa500fa0fb83422d115b5400a064ddd306d7750/cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741", size = 4316030, upload-time = "2026-05-04T22:58:25.581Z" }, - { url = "https://files.pythonhosted.org/packages/fd/36/a608b98337af3cb2aff4818e406649d30572b7031918b04c87d979495348/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166", size = 4689640, upload-time = "2026-05-04T22:58:27.747Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a6/825010a291b4438aecc1f568bc428189fc1175515223632477c07dc0a6df/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336", size = 5237657, upload-time = "2026-05-04T22:58:29.848Z" }, - { url = "https://files.pythonhosted.org/packages/b9/09/4e76a09b4caa29aad535ddc806f5d4c5d01885bd978bd984fbc6ca032cae/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057", size = 4732362, upload-time = "2026-05-04T22:58:32.009Z" }, - { url = "https://files.pythonhosted.org/packages/18/78/444fa04a77d0cb95f417dda20d450e13c56ba8e5220fc892a1658f44f882/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae", size = 4819580, upload-time = "2026-05-04T22:58:34.254Z" }, - { url = "https://files.pythonhosted.org/packages/38/85/ea67067c70a1fd4be2c63d35eeed82658023021affccc7b17705f8527dd2/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c", size = 4963283, upload-time = "2026-05-04T22:58:36.376Z" }, - { url = "https://files.pythonhosted.org/packages/75/54/cc6d0f3deac3e81c7f847e8a189a12b6cdd65059b43dad25d4316abd849a/cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f", size = 3270954, upload-time = "2026-05-04T22:58:38.791Z" }, - { url = "https://files.pythonhosted.org/packages/49/67/cc947e288c0758a4e5473d1dcb743037ab7785541265a969240b8885441a/cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12", size = 3797313, upload-time = "2026-05-04T22:58:40.746Z" }, - { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", size = 7983482, upload-time = "2026-05-04T22:58:43.769Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", size = 4683266, upload-time = "2026-05-04T22:58:46.064Z" }, - { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", size = 4696228, upload-time = "2026-05-04T22:58:48.22Z" }, - { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", size = 4689097, upload-time = "2026-05-04T22:58:50.9Z" }, - { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", size = 5283582, upload-time = "2026-05-04T22:58:53.017Z" }, - { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", size = 4730479, upload-time = "2026-05-04T22:58:55.611Z" }, - { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", size = 4326481, upload-time = "2026-05-04T22:58:57.607Z" }, - { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", size = 4688713, upload-time = "2026-05-04T22:59:00.077Z" }, - { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", size = 5238165, upload-time = "2026-05-04T22:59:02.317Z" }, - { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", size = 4729947, upload-time = "2026-05-04T22:59:05.255Z" }, - { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", size = 4822059, upload-time = "2026-05-04T22:59:07.802Z" }, - { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", size = 4960575, upload-time = "2026-05-04T22:59:09.851Z" }, - { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", size = 3279117, upload-time = "2026-05-04T22:59:12.019Z" }, - { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", size = 3792100, upload-time = "2026-05-04T22:59:14.884Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", upload-time = "2026-05-04T22:59:38.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", upload-time = "2026-05-04T22:57:36.803Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", upload-time = "2026-05-04T22:57:40.373Z" }, + { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", upload-time = "2026-05-04T22:57:42.935Z" }, + { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", upload-time = "2026-05-04T22:57:45.294Z" }, + { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", upload-time = "2026-05-04T22:57:47.933Z" }, + { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", upload-time = "2026-05-04T22:57:50.067Z" }, + { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", upload-time = "2026-05-04T22:57:52.301Z" }, + { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", upload-time = "2026-05-04T22:57:54.603Z" }, + { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", upload-time = "2026-05-04T22:57:57.207Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", upload-time = "2026-05-04T22:57:59.535Z" }, + { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", upload-time = "2026-05-04T22:58:01.996Z" }, + { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", upload-time = "2026-05-04T22:58:03.968Z" }, + { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", upload-time = "2026-05-04T22:58:06.467Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", upload-time = "2026-05-04T22:58:08.845Z" }, + { url = "https://files.pythonhosted.org/packages/6b/84/70e3feea9feea87fd7cbe77efb2712ae1e3e6edf10749dc6e95f4e60e455/cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec", upload-time = "2026-05-04T22:58:11.172Z" }, + { url = "https://files.pythonhosted.org/packages/89/6e/18e07a618bb5442ba10cf4df16e99c071365528aa570dfcb8c02e25a303b/cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18", upload-time = "2026-05-04T22:58:13.712Z" }, + { url = "https://files.pythonhosted.org/packages/be/6a/4ea3b4c6c6759794d5ee2103c304a5076dc4b19ae1f9fe47dba439e159e9/cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20", upload-time = "2026-05-04T22:58:16.448Z" }, + { url = "https://files.pythonhosted.org/packages/2f/59/6ff6ad6cae03bb887da2a5860b2c9805f8dac969ef01ce563336c49bd1d1/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff", upload-time = "2026-05-04T22:58:18.544Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b4/fc334ed8cfd705aca282fe4d8f5ae64a8e0f74932e9feecb344610cf6e4d/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c", upload-time = "2026-05-04T22:58:20.75Z" }, + { url = "https://files.pythonhosted.org/packages/11/08/9f8c5386cc4cd90d8255c7cdd0f5baf459a08502a09de30dc51f553d38dc/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db", upload-time = "2026-05-04T22:58:23.627Z" }, + { url = "https://files.pythonhosted.org/packages/b8/77/99307d7574045699f8805aa500fa0fb83422d115b5400a064ddd306d7750/cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741", upload-time = "2026-05-04T22:58:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/fd/36/a608b98337af3cb2aff4818e406649d30572b7031918b04c87d979495348/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166", upload-time = "2026-05-04T22:58:27.747Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a6/825010a291b4438aecc1f568bc428189fc1175515223632477c07dc0a6df/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336", upload-time = "2026-05-04T22:58:29.848Z" }, + { url = "https://files.pythonhosted.org/packages/b9/09/4e76a09b4caa29aad535ddc806f5d4c5d01885bd978bd984fbc6ca032cae/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057", upload-time = "2026-05-04T22:58:32.009Z" }, + { url = "https://files.pythonhosted.org/packages/18/78/444fa04a77d0cb95f417dda20d450e13c56ba8e5220fc892a1658f44f882/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae", upload-time = "2026-05-04T22:58:34.254Z" }, + { url = "https://files.pythonhosted.org/packages/38/85/ea67067c70a1fd4be2c63d35eeed82658023021affccc7b17705f8527dd2/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c", upload-time = "2026-05-04T22:58:36.376Z" }, + { url = "https://files.pythonhosted.org/packages/75/54/cc6d0f3deac3e81c7f847e8a189a12b6cdd65059b43dad25d4316abd849a/cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f", upload-time = "2026-05-04T22:58:38.791Z" }, + { url = "https://files.pythonhosted.org/packages/49/67/cc947e288c0758a4e5473d1dcb743037ab7785541265a969240b8885441a/cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12", upload-time = "2026-05-04T22:58:40.746Z" }, + { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", upload-time = "2026-05-04T22:58:43.769Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", upload-time = "2026-05-04T22:58:46.064Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", upload-time = "2026-05-04T22:58:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", upload-time = "2026-05-04T22:58:50.9Z" }, + { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", upload-time = "2026-05-04T22:58:53.017Z" }, + { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", upload-time = "2026-05-04T22:58:55.611Z" }, + { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", upload-time = "2026-05-04T22:58:57.607Z" }, + { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", upload-time = "2026-05-04T22:59:00.077Z" }, + { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", upload-time = "2026-05-04T22:59:02.317Z" }, + { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", upload-time = "2026-05-04T22:59:05.255Z" }, + { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", upload-time = "2026-05-04T22:59:07.802Z" }, + { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", upload-time = "2026-05-04T22:59:09.851Z" }, + { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", upload-time = "2026-05-04T22:59:12.019Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", upload-time = "2026-05-04T22:59:14.884Z" }, +] + +[[package]] +name = "databricks-flowx" +version = "0.2.0" +source = { editable = "." } +dependencies = [ + { name = "databricks-sdk" }, + { name = "pyyaml" }, + { name = "sqlglot" }, +] + +[package.optional-dependencies] +mcp = [ + { name = "mcp" }, + { name = "starlette" }, + { name = "uvicorn" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage" }, + { name = "mypy" }, + { name = "pytest" }, + { name = "ruff" }, + { name = "types-pyyaml" }, +] +yq = [ + { name = "yq" }, +] + +[package.metadata] +requires-dist = [ + { name = "databricks-sdk", specifier = ">=0.40" }, + { name = "mcp", marker = "extra == 'mcp'", specifier = ">=1.12" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=25.0" }, + { name = "starlette", marker = "extra == 'mcp'", specifier = ">=0.40" }, + { name = "uvicorn", marker = "extra == 'mcp'", specifier = ">=0.30" }, +] +provides-extras = ["mcp"] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", specifier = ">=7.6.1,<8" }, + { name = "mypy", specifier = ">=1.18.2,<2" }, + { name = "pytest", specifier = ">=8.3.3,<9" }, + { name = "ruff", specifier = ">=0.14.0,<1" }, + { name = "types-pyyaml", specifier = ">=6.0.12.20250915,<7" }, ] +yq = [{ name = "yq", specifier = "~=3.4.3" }] [[package]] name = "databricks-sdk" @@ -305,9 +398,9 @@ dependencies = [ { name = "protobuf" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/0f/488d61ece084f70a6d4d0ab8b5e38b0902e0b9029d0b72cde99e3f2c6b4a/databricks_sdk-0.110.0.tar.gz", hash = "sha256:b62d806982b37f8160f700d657c37b3bd586c649eb5c8c4c1216090d888c5820", size = 945261, upload-time = "2026-05-19T09:18:46.23Z" } +sdist = { url = "https://files.pythonhosted.org/packages/07/0f/488d61ece084f70a6d4d0ab8b5e38b0902e0b9029d0b72cde99e3f2c6b4a/databricks_sdk-0.110.0.tar.gz", hash = "sha256:b62d806982b37f8160f700d657c37b3bd586c649eb5c8c4c1216090d888c5820", upload-time = "2026-05-19T09:18:46.23Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/23/7c2a827890ab120ac349847ec17ab5a37eb4e3bf8f1d0989fd9eec0c1e6a/databricks_sdk-0.110.0-py3-none-any.whl", hash = "sha256:8a23db05be7a304bea43b4fa78b437051ed0f3755b19594429c649ee4159b546", size = 892096, upload-time = "2026-05-19T09:18:44.313Z" }, + { url = "https://files.pythonhosted.org/packages/9d/23/7c2a827890ab120ac349847ec17ab5a37eb4e3bf8f1d0989fd9eec0c1e6a/databricks_sdk-0.110.0-py3-none-any.whl", hash = "sha256:8a23db05be7a304bea43b4fa78b437051ed0f3755b19594429c649ee4159b546", upload-time = "2026-05-19T09:18:44.313Z" }, ] [[package]] @@ -318,87 +411,185 @@ dependencies = [ { name = "cryptography" }, { name = "pyasn1-modules" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c6/ad/ff781329bbbdc0974a098d996e89c9e1f7024262f9e3eec442fbb9ad1ac6/google_auth-2.53.0.tar.gz", hash = "sha256:e7e6aa16f6bee7b2b264830fd04f08087a1d5a836df516251a5d15327b246c9c", size = 335844, upload-time = "2026-05-15T20:53:07.928Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/ad/ff781329bbbdc0974a098d996e89c9e1f7024262f9e3eec442fbb9ad1ac6/google_auth-2.53.0.tar.gz", hash = "sha256:e7e6aa16f6bee7b2b264830fd04f08087a1d5a836df516251a5d15327b246c9c", upload-time = "2026-05-15T20:53:07.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/c9/db44165ba7c581268c6d46017ef63339110378305062830104fc7fa144cb/google_auth-2.53.0-py3-none-any.whl", hash = "sha256:6e7449917c599b35126a99ec268ec6880301f2fea41dce198fe8fd83ff642b68", upload-time = "2026-05-15T20:53:05.609Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "httpx-sse" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", upload-time = "2025-10-10T21:48:22.271Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/c9/db44165ba7c581268c6d46017ef63339110378305062830104fc7fa144cb/google_auth-2.53.0-py3-none-any.whl", hash = "sha256:6e7449917c599b35126a99ec268ec6880301f2fea41dce198fe8fd83ff642b68", size = 246071, upload-time = "2026-05-15T20:53:05.609Z" }, + { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", upload-time = "2025-10-10T21:48:21.158Z" }, ] [[package]] name = "idna" version = "3.15" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", upload-time = "2026-05-12T22:45:57.011Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, + { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", upload-time = "2026-05-12T22:45:55.733Z" }, ] [[package]] name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", upload-time = "2026-01-07T13:41:07.246Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", upload-time = "2025-09-08T01:34:57.871Z" }, ] [[package]] name = "librt" version = "0.8.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/56/9c/b4b0c54d84da4a94b37bd44151e46d5e583c9534c7e02250b961b1b6d8a8/librt-0.8.1.tar.gz", hash = "sha256:be46a14693955b3bd96014ccbdb8339ee8c9346fbe11c1b78901b55125f14c73", size = 177471, upload-time = "2026-02-17T16:13:06.101Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/21/d39b0a87ac52fc98f621fb6f8060efb017a767ebbbac2f99fbcbc9ddc0d7/librt-0.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a28f2612ab566b17f3698b0da021ff9960610301607c9a5e8eaca62f5e1c350a", size = 66516, upload-time = "2026-02-17T16:11:41.604Z" }, - { url = "https://files.pythonhosted.org/packages/69/f1/46375e71441c43e8ae335905e069f1c54febee63a146278bcee8782c84fd/librt-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:60a78b694c9aee2a0f1aaeaa7d101cf713e92e8423a941d2897f4fa37908dab9", size = 68634, upload-time = "2026-02-17T16:11:43.268Z" }, - { url = "https://files.pythonhosted.org/packages/0a/33/c510de7f93bf1fa19e13423a606d8189a02624a800710f6e6a0a0f0784b3/librt-0.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:758509ea3f1eba2a57558e7e98f4659d0ea7670bff49673b0dde18a3c7e6c0eb", size = 198941, upload-time = "2026-02-17T16:11:44.28Z" }, - { url = "https://files.pythonhosted.org/packages/dd/36/e725903416409a533d92398e88ce665476f275081d0d7d42f9c4951999e5/librt-0.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:039b9f2c506bd0ab0f8725aa5ba339c6f0cd19d3b514b50d134789809c24285d", size = 209991, upload-time = "2026-02-17T16:11:45.462Z" }, - { url = "https://files.pythonhosted.org/packages/30/7a/8d908a152e1875c9f8eac96c97a480df425e657cdb47854b9efaa4998889/librt-0.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bb54f1205a3a6ab41a6fd71dfcdcbd278670d3a90ca502a30d9da583105b6f7", size = 224476, upload-time = "2026-02-17T16:11:46.542Z" }, - { url = "https://files.pythonhosted.org/packages/a8/b8/a22c34f2c485b8903a06f3fe3315341fe6876ef3599792344669db98fcff/librt-0.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:05bd41cdee35b0c59c259f870f6da532a2c5ca57db95b5f23689fcb5c9e42440", size = 217518, upload-time = "2026-02-17T16:11:47.746Z" }, - { url = "https://files.pythonhosted.org/packages/79/6f/5c6fea00357e4f82ba44f81dbfb027921f1ab10e320d4a64e1c408d035d9/librt-0.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adfab487facf03f0d0857b8710cf82d0704a309d8ffc33b03d9302b4c64e91a9", size = 225116, upload-time = "2026-02-17T16:11:49.298Z" }, - { url = "https://files.pythonhosted.org/packages/f2/a0/95ced4e7b1267fe1e2720a111685bcddf0e781f7e9e0ce59d751c44dcfe5/librt-0.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:153188fe98a72f206042be10a2c6026139852805215ed9539186312d50a8e972", size = 217751, upload-time = "2026-02-17T16:11:50.49Z" }, - { url = "https://files.pythonhosted.org/packages/93/c2/0517281cb4d4101c27ab59472924e67f55e375bc46bedae94ac6dc6e1902/librt-0.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:dd3c41254ee98604b08bd5b3af5bf0a89740d4ee0711de95b65166bf44091921", size = 218378, upload-time = "2026-02-17T16:11:51.783Z" }, - { url = "https://files.pythonhosted.org/packages/43/e8/37b3ac108e8976888e559a7b227d0ceac03c384cfd3e7a1c2ee248dbae79/librt-0.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e0d138c7ae532908cbb342162b2611dbd4d90c941cd25ab82084aaf71d2c0bd0", size = 241199, upload-time = "2026-02-17T16:11:53.561Z" }, - { url = "https://files.pythonhosted.org/packages/4b/5b/35812d041c53967fedf551a39399271bbe4257e681236a2cf1a69c8e7fa1/librt-0.8.1-cp312-cp312-win32.whl", hash = "sha256:43353b943613c5d9c49a25aaffdba46f888ec354e71e3529a00cca3f04d66a7a", size = 54917, upload-time = "2026-02-17T16:11:54.758Z" }, - { url = "https://files.pythonhosted.org/packages/de/d1/fa5d5331b862b9775aaf2a100f5ef86854e5d4407f71bddf102f4421e034/librt-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:ff8baf1f8d3f4b6b7257fcb75a501f2a5499d0dda57645baa09d4d0d34b19444", size = 62017, upload-time = "2026-02-17T16:11:55.748Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7c/c614252f9acda59b01a66e2ddfd243ed1c7e1deab0293332dfbccf862808/librt-0.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:0f2ae3725904f7377e11cc37722d5d401e8b3d5851fb9273d7f4fe04f6b3d37d", size = 52441, upload-time = "2026-02-17T16:11:56.801Z" }, - { url = "https://files.pythonhosted.org/packages/c5/3c/f614c8e4eaac7cbf2bbdf9528790b21d89e277ee20d57dc6e559c626105f/librt-0.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e6bad1cd94f6764e1e21950542f818a09316645337fd5ab9a7acc45d99a8f35", size = 66529, upload-time = "2026-02-17T16:11:57.809Z" }, - { url = "https://files.pythonhosted.org/packages/ab/96/5836544a45100ae411eda07d29e3d99448e5258b6e9c8059deb92945f5c2/librt-0.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cf450f498c30af55551ba4f66b9123b7185362ec8b625a773b3d39aa1a717583", size = 68669, upload-time = "2026-02-17T16:11:58.843Z" }, - { url = "https://files.pythonhosted.org/packages/06/53/f0b992b57af6d5531bf4677d75c44f095f2366a1741fb695ee462ae04b05/librt-0.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:eca45e982fa074090057132e30585a7e8674e9e885d402eae85633e9f449ce6c", size = 199279, upload-time = "2026-02-17T16:11:59.862Z" }, - { url = "https://files.pythonhosted.org/packages/f3/ad/4848cc16e268d14280d8168aee4f31cea92bbd2b79ce33d3e166f2b4e4fc/librt-0.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c3811485fccfda840861905b8c70bba5ec094e02825598bb9d4ca3936857a04", size = 210288, upload-time = "2026-02-17T16:12:00.954Z" }, - { url = "https://files.pythonhosted.org/packages/52/05/27fdc2e95de26273d83b96742d8d3b7345f2ea2bdbd2405cc504644f2096/librt-0.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e4af413908f77294605e28cfd98063f54b2c790561383971d2f52d113d9c363", size = 224809, upload-time = "2026-02-17T16:12:02.108Z" }, - { url = "https://files.pythonhosted.org/packages/7a/d0/78200a45ba3240cb042bc597d6f2accba9193a2c57d0356268cbbe2d0925/librt-0.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5212a5bd7fae98dae95710032902edcd2ec4dc994e883294f75c857b83f9aba0", size = 218075, upload-time = "2026-02-17T16:12:03.631Z" }, - { url = "https://files.pythonhosted.org/packages/af/72/a210839fa74c90474897124c064ffca07f8d4b347b6574d309686aae7ca6/librt-0.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e692aa2d1d604e6ca12d35e51fdc36f4cda6345e28e36374579f7ef3611b3012", size = 225486, upload-time = "2026-02-17T16:12:04.725Z" }, - { url = "https://files.pythonhosted.org/packages/a3/c1/a03cc63722339ddbf087485f253493e2b013039f5b707e8e6016141130fa/librt-0.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4be2a5c926b9770c9e08e717f05737a269b9d0ebc5d2f0060f0fe3fe9ce47acb", size = 218219, upload-time = "2026-02-17T16:12:05.828Z" }, - { url = "https://files.pythonhosted.org/packages/58/f5/fff6108af0acf941c6f274a946aea0e484bd10cd2dc37610287ce49388c5/librt-0.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fd1a720332ea335ceb544cf0a03f81df92abd4bb887679fd1e460976b0e6214b", size = 218750, upload-time = "2026-02-17T16:12:07.09Z" }, - { url = "https://files.pythonhosted.org/packages/71/67/5a387bfef30ec1e4b4f30562c8586566faf87e47d696768c19feb49e3646/librt-0.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2af9e01e0ef80d95ae3c720be101227edae5f2fe7e3dc63d8857fadfc5a1d", size = 241624, upload-time = "2026-02-17T16:12:08.43Z" }, - { url = "https://files.pythonhosted.org/packages/d4/be/24f8502db11d405232ac1162eb98069ca49c3306c1d75c6ccc61d9af8789/librt-0.8.1-cp313-cp313-win32.whl", hash = "sha256:086a32dbb71336627e78cc1d6ee305a68d038ef7d4c39aaff41ae8c9aa46e91a", size = 54969, upload-time = "2026-02-17T16:12:09.633Z" }, - { url = "https://files.pythonhosted.org/packages/5c/73/c9fdf6cb2a529c1a092ce769a12d88c8cca991194dfe641b6af12fa964d2/librt-0.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:e11769a1dbda4da7b00a76cfffa67aa47cfa66921d2724539eee4b9ede780b79", size = 62000, upload-time = "2026-02-17T16:12:10.632Z" }, - { url = "https://files.pythonhosted.org/packages/d3/97/68f80ca3ac4924f250cdfa6e20142a803e5e50fca96ef5148c52ee8c10ea/librt-0.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:924817ab3141aca17893386ee13261f1d100d1ef410d70afe4389f2359fea4f0", size = 52495, upload-time = "2026-02-17T16:12:11.633Z" }, - { url = "https://files.pythonhosted.org/packages/c9/6a/907ef6800f7bca71b525a05f1839b21f708c09043b1c6aa77b6b827b3996/librt-0.8.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6cfa7fe54fd4d1f47130017351a959fe5804bda7a0bc7e07a2cdbc3fdd28d34f", size = 66081, upload-time = "2026-02-17T16:12:12.766Z" }, - { url = "https://files.pythonhosted.org/packages/1b/18/25e991cd5640c9fb0f8d91b18797b29066b792f17bf8493da183bf5caabe/librt-0.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:228c2409c079f8c11fb2e5d7b277077f694cb93443eb760e00b3b83cb8b3176c", size = 68309, upload-time = "2026-02-17T16:12:13.756Z" }, - { url = "https://files.pythonhosted.org/packages/a4/36/46820d03f058cfb5a9de5940640ba03165ed8aded69e0733c417bb04df34/librt-0.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7aae78ab5e3206181780e56912d1b9bb9f90a7249ce12f0e8bf531d0462dd0fc", size = 196804, upload-time = "2026-02-17T16:12:14.818Z" }, - { url = "https://files.pythonhosted.org/packages/59/18/5dd0d3b87b8ff9c061849fbdb347758d1f724b9a82241aa908e0ec54ccd0/librt-0.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:172d57ec04346b047ca6af181e1ea4858086c80bdf455f61994c4aa6fc3f866c", size = 206907, upload-time = "2026-02-17T16:12:16.513Z" }, - { url = "https://files.pythonhosted.org/packages/d1/96/ef04902aad1424fd7299b62d1890e803e6ab4018c3044dca5922319c4b97/librt-0.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b1977c4ea97ce5eb7755a78fae68d87e4102e4aaf54985e8b56806849cc06a3", size = 221217, upload-time = "2026-02-17T16:12:17.906Z" }, - { url = "https://files.pythonhosted.org/packages/6d/ff/7e01f2dda84a8f5d280637a2e5827210a8acca9a567a54507ef1c75b342d/librt-0.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10c42e1f6fd06733ef65ae7bebce2872bcafd8d6e6b0a08fe0a05a23b044fb14", size = 214622, upload-time = "2026-02-17T16:12:19.108Z" }, - { url = "https://files.pythonhosted.org/packages/1e/8c/5b093d08a13946034fed57619742f790faf77058558b14ca36a6e331161e/librt-0.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4c8dfa264b9193c4ee19113c985c95f876fae5e51f731494fc4e0cf594990ba7", size = 221987, upload-time = "2026-02-17T16:12:20.331Z" }, - { url = "https://files.pythonhosted.org/packages/d3/cc/86b0b3b151d40920ad45a94ce0171dec1aebba8a9d72bb3fa00c73ab25dd/librt-0.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:01170b6729a438f0dedc4a26ed342e3dc4f02d1000b4b19f980e1877f0c297e6", size = 215132, upload-time = "2026-02-17T16:12:21.54Z" }, - { url = "https://files.pythonhosted.org/packages/fc/be/8588164a46edf1e69858d952654e216a9a91174688eeefb9efbb38a9c799/librt-0.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7b02679a0d783bdae30d443025b94465d8c3dc512f32f5b5031f93f57ac32071", size = 215195, upload-time = "2026-02-17T16:12:23.073Z" }, - { url = "https://files.pythonhosted.org/packages/f5/f2/0b9279bea735c734d69344ecfe056c1ba211694a72df10f568745c899c76/librt-0.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:190b109bb69592a3401fe1ffdea41a2e73370ace2ffdc4a0e8e2b39cdea81b78", size = 237946, upload-time = "2026-02-17T16:12:24.275Z" }, - { url = "https://files.pythonhosted.org/packages/e9/cc/5f2a34fbc8aeb35314a3641f9956fa9051a947424652fad9882be7a97949/librt-0.8.1-cp314-cp314-win32.whl", hash = "sha256:e70a57ecf89a0f64c24e37f38d3fe217a58169d2fe6ed6d70554964042474023", size = 50689, upload-time = "2026-02-17T16:12:25.766Z" }, - { url = "https://files.pythonhosted.org/packages/a0/76/cd4d010ab2147339ca2b93e959c3686e964edc6de66ddacc935c325883d7/librt-0.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:7e2f3edca35664499fbb36e4770650c4bd4a08abc1f4458eab9df4ec56389730", size = 57875, upload-time = "2026-02-17T16:12:27.465Z" }, - { url = "https://files.pythonhosted.org/packages/84/0f/2143cb3c3ca48bd3379dcd11817163ca50781927c4537345d608b5045998/librt-0.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:0d2f82168e55ddefd27c01c654ce52379c0750ddc31ee86b4b266bcf4d65f2a3", size = 48058, upload-time = "2026-02-17T16:12:28.556Z" }, - { url = "https://files.pythonhosted.org/packages/d2/0e/9b23a87e37baf00311c3efe6b48d6b6c168c29902dfc3f04c338372fd7db/librt-0.8.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2c74a2da57a094bd48d03fa5d196da83d2815678385d2978657499063709abe1", size = 68313, upload-time = "2026-02-17T16:12:29.659Z" }, - { url = "https://files.pythonhosted.org/packages/db/9a/859c41e5a4f1c84200a7d2b92f586aa27133c8243b6cac9926f6e54d01b9/librt-0.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a355d99c4c0d8e5b770313b8b247411ed40949ca44e33e46a4789b9293a907ee", size = 70994, upload-time = "2026-02-17T16:12:31.516Z" }, - { url = "https://files.pythonhosted.org/packages/4c/28/10605366ee599ed34223ac2bf66404c6fb59399f47108215d16d5ad751a8/librt-0.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2eb345e8b33fb748227409c9f1233d4df354d6e54091f0e8fc53acdb2ffedeb7", size = 220770, upload-time = "2026-02-17T16:12:33.294Z" }, - { url = "https://files.pythonhosted.org/packages/af/8d/16ed8fd452dafae9c48d17a6bc1ee3e818fd40ef718d149a8eff2c9f4ea2/librt-0.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9be2f15e53ce4e83cc08adc29b26fb5978db62ef2a366fbdf716c8a6c8901040", size = 235409, upload-time = "2026-02-17T16:12:35.443Z" }, - { url = "https://files.pythonhosted.org/packages/89/1b/7bdf3e49349c134b25db816e4a3db6b94a47ac69d7d46b1e682c2c4949be/librt-0.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:785ae29c1f5c6e7c2cde2c7c0e148147f4503da3abc5d44d482068da5322fd9e", size = 246473, upload-time = "2026-02-17T16:12:36.656Z" }, - { url = "https://files.pythonhosted.org/packages/4e/8a/91fab8e4fd2a24930a17188c7af5380eb27b203d72101c9cc000dbdfd95a/librt-0.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d3a7da44baf692f0c6aeb5b2a09c5e6fc7a703bca9ffa337ddd2e2da53f7732", size = 238866, upload-time = "2026-02-17T16:12:37.849Z" }, - { url = "https://files.pythonhosted.org/packages/b9/e0/c45a098843fc7c07e18a7f8a24ca8496aecbf7bdcd54980c6ca1aaa79a8e/librt-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5fc48998000cbc39ec0d5311312dda93ecf92b39aaf184c5e817d5d440b29624", size = 250248, upload-time = "2026-02-17T16:12:39.445Z" }, - { url = "https://files.pythonhosted.org/packages/82/30/07627de23036640c952cce0c1fe78972e77d7d2f8fd54fa5ef4554ff4a56/librt-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e96baa6820280077a78244b2e06e416480ed859bbd8e5d641cf5742919d8beb4", size = 240629, upload-time = "2026-02-17T16:12:40.889Z" }, - { url = "https://files.pythonhosted.org/packages/fb/c1/55bfe1ee3542eba055616f9098eaf6eddb966efb0ca0f44eaa4aba327307/librt-0.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31362dbfe297b23590530007062c32c6f6176f6099646bb2c95ab1b00a57c382", size = 239615, upload-time = "2026-02-17T16:12:42.446Z" }, - { url = "https://files.pythonhosted.org/packages/2b/39/191d3d28abc26c9099b19852e6c99f7f6d400b82fa5a4e80291bd3803e19/librt-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc3656283d11540ab0ea01978378e73e10002145117055e03722417aeab30994", size = 263001, upload-time = "2026-02-17T16:12:43.627Z" }, - { url = "https://files.pythonhosted.org/packages/b9/eb/7697f60fbe7042ab4e88f4ee6af496b7f222fffb0a4e3593ef1f29f81652/librt-0.8.1-cp314-cp314t-win32.whl", hash = "sha256:738f08021b3142c2918c03692608baed43bc51144c29e35807682f8070ee2a3a", size = 51328, upload-time = "2026-02-17T16:12:45.148Z" }, - { url = "https://files.pythonhosted.org/packages/7c/72/34bf2eb7a15414a23e5e70ecb9440c1d3179f393d9349338a91e2781c0fb/librt-0.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:89815a22daf9c51884fb5dbe4f1ef65ee6a146e0b6a8df05f753e2e4a9359bf4", size = 58722, upload-time = "2026-02-17T16:12:46.85Z" }, - { url = "https://files.pythonhosted.org/packages/b2/c8/d148e041732d631fc76036f8b30fae4e77b027a1e95b7a84bb522481a940/librt-0.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:bf512a71a23504ed08103a13c941f763db13fb11177beb3d9244c98c29fb4a61", size = 48755, upload-time = "2026-02-17T16:12:47.943Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/56/9c/b4b0c54d84da4a94b37bd44151e46d5e583c9534c7e02250b961b1b6d8a8/librt-0.8.1.tar.gz", hash = "sha256:be46a14693955b3bd96014ccbdb8339ee8c9346fbe11c1b78901b55125f14c73", upload-time = "2026-02-17T16:13:06.101Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/21/d39b0a87ac52fc98f621fb6f8060efb017a767ebbbac2f99fbcbc9ddc0d7/librt-0.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a28f2612ab566b17f3698b0da021ff9960610301607c9a5e8eaca62f5e1c350a", upload-time = "2026-02-17T16:11:41.604Z" }, + { url = "https://files.pythonhosted.org/packages/69/f1/46375e71441c43e8ae335905e069f1c54febee63a146278bcee8782c84fd/librt-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:60a78b694c9aee2a0f1aaeaa7d101cf713e92e8423a941d2897f4fa37908dab9", upload-time = "2026-02-17T16:11:43.268Z" }, + { url = "https://files.pythonhosted.org/packages/0a/33/c510de7f93bf1fa19e13423a606d8189a02624a800710f6e6a0a0f0784b3/librt-0.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:758509ea3f1eba2a57558e7e98f4659d0ea7670bff49673b0dde18a3c7e6c0eb", upload-time = "2026-02-17T16:11:44.28Z" }, + { url = "https://files.pythonhosted.org/packages/dd/36/e725903416409a533d92398e88ce665476f275081d0d7d42f9c4951999e5/librt-0.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:039b9f2c506bd0ab0f8725aa5ba339c6f0cd19d3b514b50d134789809c24285d", upload-time = "2026-02-17T16:11:45.462Z" }, + { url = "https://files.pythonhosted.org/packages/30/7a/8d908a152e1875c9f8eac96c97a480df425e657cdb47854b9efaa4998889/librt-0.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bb54f1205a3a6ab41a6fd71dfcdcbd278670d3a90ca502a30d9da583105b6f7", upload-time = "2026-02-17T16:11:46.542Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b8/a22c34f2c485b8903a06f3fe3315341fe6876ef3599792344669db98fcff/librt-0.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:05bd41cdee35b0c59c259f870f6da532a2c5ca57db95b5f23689fcb5c9e42440", upload-time = "2026-02-17T16:11:47.746Z" }, + { url = "https://files.pythonhosted.org/packages/79/6f/5c6fea00357e4f82ba44f81dbfb027921f1ab10e320d4a64e1c408d035d9/librt-0.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adfab487facf03f0d0857b8710cf82d0704a309d8ffc33b03d9302b4c64e91a9", upload-time = "2026-02-17T16:11:49.298Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a0/95ced4e7b1267fe1e2720a111685bcddf0e781f7e9e0ce59d751c44dcfe5/librt-0.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:153188fe98a72f206042be10a2c6026139852805215ed9539186312d50a8e972", upload-time = "2026-02-17T16:11:50.49Z" }, + { url = "https://files.pythonhosted.org/packages/93/c2/0517281cb4d4101c27ab59472924e67f55e375bc46bedae94ac6dc6e1902/librt-0.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:dd3c41254ee98604b08bd5b3af5bf0a89740d4ee0711de95b65166bf44091921", upload-time = "2026-02-17T16:11:51.783Z" }, + { url = "https://files.pythonhosted.org/packages/43/e8/37b3ac108e8976888e559a7b227d0ceac03c384cfd3e7a1c2ee248dbae79/librt-0.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e0d138c7ae532908cbb342162b2611dbd4d90c941cd25ab82084aaf71d2c0bd0", upload-time = "2026-02-17T16:11:53.561Z" }, + { url = "https://files.pythonhosted.org/packages/4b/5b/35812d041c53967fedf551a39399271bbe4257e681236a2cf1a69c8e7fa1/librt-0.8.1-cp312-cp312-win32.whl", hash = "sha256:43353b943613c5d9c49a25aaffdba46f888ec354e71e3529a00cca3f04d66a7a", upload-time = "2026-02-17T16:11:54.758Z" }, + { url = "https://files.pythonhosted.org/packages/de/d1/fa5d5331b862b9775aaf2a100f5ef86854e5d4407f71bddf102f4421e034/librt-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:ff8baf1f8d3f4b6b7257fcb75a501f2a5499d0dda57645baa09d4d0d34b19444", upload-time = "2026-02-17T16:11:55.748Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7c/c614252f9acda59b01a66e2ddfd243ed1c7e1deab0293332dfbccf862808/librt-0.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:0f2ae3725904f7377e11cc37722d5d401e8b3d5851fb9273d7f4fe04f6b3d37d", upload-time = "2026-02-17T16:11:56.801Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3c/f614c8e4eaac7cbf2bbdf9528790b21d89e277ee20d57dc6e559c626105f/librt-0.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e6bad1cd94f6764e1e21950542f818a09316645337fd5ab9a7acc45d99a8f35", upload-time = "2026-02-17T16:11:57.809Z" }, + { url = "https://files.pythonhosted.org/packages/ab/96/5836544a45100ae411eda07d29e3d99448e5258b6e9c8059deb92945f5c2/librt-0.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cf450f498c30af55551ba4f66b9123b7185362ec8b625a773b3d39aa1a717583", upload-time = "2026-02-17T16:11:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/06/53/f0b992b57af6d5531bf4677d75c44f095f2366a1741fb695ee462ae04b05/librt-0.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:eca45e982fa074090057132e30585a7e8674e9e885d402eae85633e9f449ce6c", upload-time = "2026-02-17T16:11:59.862Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ad/4848cc16e268d14280d8168aee4f31cea92bbd2b79ce33d3e166f2b4e4fc/librt-0.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c3811485fccfda840861905b8c70bba5ec094e02825598bb9d4ca3936857a04", upload-time = "2026-02-17T16:12:00.954Z" }, + { url = "https://files.pythonhosted.org/packages/52/05/27fdc2e95de26273d83b96742d8d3b7345f2ea2bdbd2405cc504644f2096/librt-0.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e4af413908f77294605e28cfd98063f54b2c790561383971d2f52d113d9c363", upload-time = "2026-02-17T16:12:02.108Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d0/78200a45ba3240cb042bc597d6f2accba9193a2c57d0356268cbbe2d0925/librt-0.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5212a5bd7fae98dae95710032902edcd2ec4dc994e883294f75c857b83f9aba0", upload-time = "2026-02-17T16:12:03.631Z" }, + { url = "https://files.pythonhosted.org/packages/af/72/a210839fa74c90474897124c064ffca07f8d4b347b6574d309686aae7ca6/librt-0.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e692aa2d1d604e6ca12d35e51fdc36f4cda6345e28e36374579f7ef3611b3012", upload-time = "2026-02-17T16:12:04.725Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c1/a03cc63722339ddbf087485f253493e2b013039f5b707e8e6016141130fa/librt-0.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4be2a5c926b9770c9e08e717f05737a269b9d0ebc5d2f0060f0fe3fe9ce47acb", upload-time = "2026-02-17T16:12:05.828Z" }, + { url = "https://files.pythonhosted.org/packages/58/f5/fff6108af0acf941c6f274a946aea0e484bd10cd2dc37610287ce49388c5/librt-0.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fd1a720332ea335ceb544cf0a03f81df92abd4bb887679fd1e460976b0e6214b", upload-time = "2026-02-17T16:12:07.09Z" }, + { url = "https://files.pythonhosted.org/packages/71/67/5a387bfef30ec1e4b4f30562c8586566faf87e47d696768c19feb49e3646/librt-0.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2af9e01e0ef80d95ae3c720be101227edae5f2fe7e3dc63d8857fadfc5a1d", upload-time = "2026-02-17T16:12:08.43Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/24f8502db11d405232ac1162eb98069ca49c3306c1d75c6ccc61d9af8789/librt-0.8.1-cp313-cp313-win32.whl", hash = "sha256:086a32dbb71336627e78cc1d6ee305a68d038ef7d4c39aaff41ae8c9aa46e91a", upload-time = "2026-02-17T16:12:09.633Z" }, + { url = "https://files.pythonhosted.org/packages/5c/73/c9fdf6cb2a529c1a092ce769a12d88c8cca991194dfe641b6af12fa964d2/librt-0.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:e11769a1dbda4da7b00a76cfffa67aa47cfa66921d2724539eee4b9ede780b79", upload-time = "2026-02-17T16:12:10.632Z" }, + { url = "https://files.pythonhosted.org/packages/d3/97/68f80ca3ac4924f250cdfa6e20142a803e5e50fca96ef5148c52ee8c10ea/librt-0.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:924817ab3141aca17893386ee13261f1d100d1ef410d70afe4389f2359fea4f0", upload-time = "2026-02-17T16:12:11.633Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6a/907ef6800f7bca71b525a05f1839b21f708c09043b1c6aa77b6b827b3996/librt-0.8.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6cfa7fe54fd4d1f47130017351a959fe5804bda7a0bc7e07a2cdbc3fdd28d34f", upload-time = "2026-02-17T16:12:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/1b/18/25e991cd5640c9fb0f8d91b18797b29066b792f17bf8493da183bf5caabe/librt-0.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:228c2409c079f8c11fb2e5d7b277077f694cb93443eb760e00b3b83cb8b3176c", upload-time = "2026-02-17T16:12:13.756Z" }, + { url = "https://files.pythonhosted.org/packages/a4/36/46820d03f058cfb5a9de5940640ba03165ed8aded69e0733c417bb04df34/librt-0.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7aae78ab5e3206181780e56912d1b9bb9f90a7249ce12f0e8bf531d0462dd0fc", upload-time = "2026-02-17T16:12:14.818Z" }, + { url = "https://files.pythonhosted.org/packages/59/18/5dd0d3b87b8ff9c061849fbdb347758d1f724b9a82241aa908e0ec54ccd0/librt-0.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:172d57ec04346b047ca6af181e1ea4858086c80bdf455f61994c4aa6fc3f866c", upload-time = "2026-02-17T16:12:16.513Z" }, + { url = "https://files.pythonhosted.org/packages/d1/96/ef04902aad1424fd7299b62d1890e803e6ab4018c3044dca5922319c4b97/librt-0.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b1977c4ea97ce5eb7755a78fae68d87e4102e4aaf54985e8b56806849cc06a3", upload-time = "2026-02-17T16:12:17.906Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ff/7e01f2dda84a8f5d280637a2e5827210a8acca9a567a54507ef1c75b342d/librt-0.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10c42e1f6fd06733ef65ae7bebce2872bcafd8d6e6b0a08fe0a05a23b044fb14", upload-time = "2026-02-17T16:12:19.108Z" }, + { url = "https://files.pythonhosted.org/packages/1e/8c/5b093d08a13946034fed57619742f790faf77058558b14ca36a6e331161e/librt-0.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4c8dfa264b9193c4ee19113c985c95f876fae5e51f731494fc4e0cf594990ba7", upload-time = "2026-02-17T16:12:20.331Z" }, + { url = "https://files.pythonhosted.org/packages/d3/cc/86b0b3b151d40920ad45a94ce0171dec1aebba8a9d72bb3fa00c73ab25dd/librt-0.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:01170b6729a438f0dedc4a26ed342e3dc4f02d1000b4b19f980e1877f0c297e6", upload-time = "2026-02-17T16:12:21.54Z" }, + { url = "https://files.pythonhosted.org/packages/fc/be/8588164a46edf1e69858d952654e216a9a91174688eeefb9efbb38a9c799/librt-0.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7b02679a0d783bdae30d443025b94465d8c3dc512f32f5b5031f93f57ac32071", upload-time = "2026-02-17T16:12:23.073Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f2/0b9279bea735c734d69344ecfe056c1ba211694a72df10f568745c899c76/librt-0.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:190b109bb69592a3401fe1ffdea41a2e73370ace2ffdc4a0e8e2b39cdea81b78", upload-time = "2026-02-17T16:12:24.275Z" }, + { url = "https://files.pythonhosted.org/packages/e9/cc/5f2a34fbc8aeb35314a3641f9956fa9051a947424652fad9882be7a97949/librt-0.8.1-cp314-cp314-win32.whl", hash = "sha256:e70a57ecf89a0f64c24e37f38d3fe217a58169d2fe6ed6d70554964042474023", upload-time = "2026-02-17T16:12:25.766Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/cd4d010ab2147339ca2b93e959c3686e964edc6de66ddacc935c325883d7/librt-0.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:7e2f3edca35664499fbb36e4770650c4bd4a08abc1f4458eab9df4ec56389730", upload-time = "2026-02-17T16:12:27.465Z" }, + { url = "https://files.pythonhosted.org/packages/84/0f/2143cb3c3ca48bd3379dcd11817163ca50781927c4537345d608b5045998/librt-0.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:0d2f82168e55ddefd27c01c654ce52379c0750ddc31ee86b4b266bcf4d65f2a3", upload-time = "2026-02-17T16:12:28.556Z" }, + { url = "https://files.pythonhosted.org/packages/d2/0e/9b23a87e37baf00311c3efe6b48d6b6c168c29902dfc3f04c338372fd7db/librt-0.8.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2c74a2da57a094bd48d03fa5d196da83d2815678385d2978657499063709abe1", upload-time = "2026-02-17T16:12:29.659Z" }, + { url = "https://files.pythonhosted.org/packages/db/9a/859c41e5a4f1c84200a7d2b92f586aa27133c8243b6cac9926f6e54d01b9/librt-0.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a355d99c4c0d8e5b770313b8b247411ed40949ca44e33e46a4789b9293a907ee", upload-time = "2026-02-17T16:12:31.516Z" }, + { url = "https://files.pythonhosted.org/packages/4c/28/10605366ee599ed34223ac2bf66404c6fb59399f47108215d16d5ad751a8/librt-0.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2eb345e8b33fb748227409c9f1233d4df354d6e54091f0e8fc53acdb2ffedeb7", upload-time = "2026-02-17T16:12:33.294Z" }, + { url = "https://files.pythonhosted.org/packages/af/8d/16ed8fd452dafae9c48d17a6bc1ee3e818fd40ef718d149a8eff2c9f4ea2/librt-0.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9be2f15e53ce4e83cc08adc29b26fb5978db62ef2a366fbdf716c8a6c8901040", upload-time = "2026-02-17T16:12:35.443Z" }, + { url = "https://files.pythonhosted.org/packages/89/1b/7bdf3e49349c134b25db816e4a3db6b94a47ac69d7d46b1e682c2c4949be/librt-0.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:785ae29c1f5c6e7c2cde2c7c0e148147f4503da3abc5d44d482068da5322fd9e", upload-time = "2026-02-17T16:12:36.656Z" }, + { url = "https://files.pythonhosted.org/packages/4e/8a/91fab8e4fd2a24930a17188c7af5380eb27b203d72101c9cc000dbdfd95a/librt-0.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d3a7da44baf692f0c6aeb5b2a09c5e6fc7a703bca9ffa337ddd2e2da53f7732", upload-time = "2026-02-17T16:12:37.849Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e0/c45a098843fc7c07e18a7f8a24ca8496aecbf7bdcd54980c6ca1aaa79a8e/librt-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5fc48998000cbc39ec0d5311312dda93ecf92b39aaf184c5e817d5d440b29624", upload-time = "2026-02-17T16:12:39.445Z" }, + { url = "https://files.pythonhosted.org/packages/82/30/07627de23036640c952cce0c1fe78972e77d7d2f8fd54fa5ef4554ff4a56/librt-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e96baa6820280077a78244b2e06e416480ed859bbd8e5d641cf5742919d8beb4", upload-time = "2026-02-17T16:12:40.889Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c1/55bfe1ee3542eba055616f9098eaf6eddb966efb0ca0f44eaa4aba327307/librt-0.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31362dbfe297b23590530007062c32c6f6176f6099646bb2c95ab1b00a57c382", upload-time = "2026-02-17T16:12:42.446Z" }, + { url = "https://files.pythonhosted.org/packages/2b/39/191d3d28abc26c9099b19852e6c99f7f6d400b82fa5a4e80291bd3803e19/librt-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc3656283d11540ab0ea01978378e73e10002145117055e03722417aeab30994", upload-time = "2026-02-17T16:12:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/b9/eb/7697f60fbe7042ab4e88f4ee6af496b7f222fffb0a4e3593ef1f29f81652/librt-0.8.1-cp314-cp314t-win32.whl", hash = "sha256:738f08021b3142c2918c03692608baed43bc51144c29e35807682f8070ee2a3a", upload-time = "2026-02-17T16:12:45.148Z" }, + { url = "https://files.pythonhosted.org/packages/7c/72/34bf2eb7a15414a23e5e70ecb9440c1d3179f393d9349338a91e2781c0fb/librt-0.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:89815a22daf9c51884fb5dbe4f1ef65ee6a146e0b6a8df05f753e2e4a9359bf4", upload-time = "2026-02-17T16:12:46.85Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c8/d148e041732d631fc76036f8b30fae4e77b027a1e95b7a84bb522481a940/librt-0.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:bf512a71a23504ed08103a13c941f763db13fb11177beb3d9244c98c29fb4a61", upload-time = "2026-02-17T16:12:47.943Z" }, +] + +[[package]] +name = "mcp" +version = "1.27.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "jsonschema" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/3c/347cf965d313f5d41764e7d46bea6ffe7d9ef13b983cc429b0340962a082/mcp-1.27.2.tar.gz", hash = "sha256:8e02db104096d1c25b28e64bde29a5c32b31bc241710213e12fd4d84985bdfef", upload-time = "2026-05-29T17:16:04.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/11/252c6f971dc4f16af1d98a1c469d8ba523aab00d1bb76b4d3bc1ff32eacc/mcp-1.27.2-py3-none-any.whl", hash = "sha256:d6ff5160c6ca65d93013626efb3fc249de683c30b2d8570755ceddd490344de5", upload-time = "2026-05-29T17:16:02.442Z" }, ] [[package]] @@ -411,136 +602,97 @@ dependencies = [ { name = "pathspec" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f8/5c/b0089fe7fef0a994ae5ee07029ced0526082c6cfaaa4c10d40a10e33b097/mypy-1.20.0.tar.gz", hash = "sha256:eb96c84efcc33f0b5e0e04beacf00129dd963b67226b01c00b9dfc8affb464c3", size = 3815028, upload-time = "2026-03-31T16:55:14.959Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/dd/3afa29b58c2e57c79116ed55d700721c3c3b15955e2b6251dd165d377c0e/mypy-1.20.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:002b613ae19f4ac7d18b7e168ffe1cb9013b37c57f7411984abbd3b817b0a214", size = 14509525, upload-time = "2026-03-31T16:55:01.824Z" }, - { url = "https://files.pythonhosted.org/packages/54/eb/227b516ab8cad9f2a13c5e7a98d28cd6aa75e9c83e82776ae6c1c4c046c7/mypy-1.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9336b5e6712f4adaf5afc3203a99a40b379049104349d747eb3e5a3aa23ac2e", size = 13326469, upload-time = "2026-03-31T16:51:41.23Z" }, - { url = "https://files.pythonhosted.org/packages/57/d4/1ddb799860c1b5ac6117ec307b965f65deeb47044395ff01ab793248a591/mypy-1.20.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f13b3e41bce9d257eded794c0f12878af3129d80aacd8a3ee0dee51f3a978651", size = 13705953, upload-time = "2026-03-31T16:48:55.69Z" }, - { url = "https://files.pythonhosted.org/packages/c5/b7/54a720f565a87b893182a2a393370289ae7149e4715859e10e1c05e49154/mypy-1.20.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9804c3ad27f78e54e58b32e7cb532d128b43dbfb9f3f9f06262b821a0f6bd3f5", size = 14710363, upload-time = "2026-03-31T16:53:26.948Z" }, - { url = "https://files.pythonhosted.org/packages/b2/2a/74810274848d061f8a8ea4ac23aaad43bd3d8c1882457999c2e568341c57/mypy-1.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:697f102c5c1d526bdd761a69f17c6070f9892eebcb94b1a5963d679288c09e78", size = 14947005, upload-time = "2026-03-31T16:50:17.591Z" }, - { url = "https://files.pythonhosted.org/packages/77/91/21b8ba75f958bcda75690951ce6fa6b7138b03471618959529d74b8544e2/mypy-1.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ecd63f75fdd30327e4ad8b5704bd6d91fc6c1b2e029f8ee14705e1207212489", size = 10880616, upload-time = "2026-03-31T16:52:19.986Z" }, - { url = "https://files.pythonhosted.org/packages/8a/15/3d8198ef97c1ca03aea010cce4f1d4f3bc5d9849e8c0140111ca2ead9fdd/mypy-1.20.0-cp312-cp312-win_arm64.whl", hash = "sha256:f194db59657c58593a3c47c6dfd7bad4ef4ac12dbc94d01b3a95521f78177e33", size = 9813091, upload-time = "2026-03-31T16:53:44.385Z" }, - { url = "https://files.pythonhosted.org/packages/d6/a7/f64ea7bd592fa431cb597418b6dec4a47f7d0c36325fec7ac67bc8402b94/mypy-1.20.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b20c8b0fd5877abdf402e79a3af987053de07e6fb208c18df6659f708b535134", size = 14485344, upload-time = "2026-03-31T16:49:16.78Z" }, - { url = "https://files.pythonhosted.org/packages/bb/72/8927d84cfc90c6abea6e96663576e2e417589347eb538749a464c4c218a0/mypy-1.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:367e5c993ba34d5054d11937d0485ad6dfc60ba760fa326c01090fc256adf15c", size = 13327400, upload-time = "2026-03-31T16:53:08.02Z" }, - { url = "https://files.pythonhosted.org/packages/ab/4a/11ab99f9afa41aa350178d24a7d2da17043228ea10f6456523f64b5a6cf6/mypy-1.20.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f799d9db89fc00446f03281f84a221e50018fc40113a3ba9864b132895619ebe", size = 13706384, upload-time = "2026-03-31T16:52:28.577Z" }, - { url = "https://files.pythonhosted.org/packages/42/79/694ca73979cfb3535ebfe78733844cd5aff2e63304f59bf90585110d975a/mypy-1.20.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555658c611099455b2da507582ea20d2043dfdfe7f5ad0add472b1c6238b433f", size = 14700378, upload-time = "2026-03-31T16:48:45.527Z" }, - { url = "https://files.pythonhosted.org/packages/84/24/a022ccab3a46e3d2cdf2e0e260648633640eb396c7e75d5a42818a8d3971/mypy-1.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:efe8d70949c3023698c3fca1e94527e7e790a361ab8116f90d11221421cd8726", size = 14932170, upload-time = "2026-03-31T16:49:36.038Z" }, - { url = "https://files.pythonhosted.org/packages/d8/9b/549228d88f574d04117e736f55958bd4908f980f9f5700a07aeb85df005b/mypy-1.20.0-cp313-cp313-win_amd64.whl", hash = "sha256:f49590891d2c2f8a9de15614e32e459a794bcba84693c2394291a2038bbaaa69", size = 10888526, upload-time = "2026-03-31T16:50:59.827Z" }, - { url = "https://files.pythonhosted.org/packages/91/17/15095c0e54a8bc04d22d4ff06b2139d5f142c2e87520b4e39010c4862771/mypy-1.20.0-cp313-cp313-win_arm64.whl", hash = "sha256:76a70bf840495729be47510856b978f1b0ec7d08f257ca38c9d932720bf6b43e", size = 9816456, upload-time = "2026-03-31T16:49:59.537Z" }, - { url = "https://files.pythonhosted.org/packages/4e/0e/6ca4a84cbed9e62384bc0b2974c90395ece5ed672393e553996501625fc5/mypy-1.20.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0f42dfaab7ec1baff3b383ad7af562ab0de573c5f6edb44b2dab016082b89948", size = 14483331, upload-time = "2026-03-31T16:52:57.999Z" }, - { url = "https://files.pythonhosted.org/packages/7d/c5/5fe9d8a729dd9605064691816243ae6c49fde0bd28f6e5e17f6a24203c43/mypy-1.20.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:31b5dbb55293c1bd27c0fc813a0d2bb5ceef9d65ac5afa2e58f829dab7921fd5", size = 13342047, upload-time = "2026-03-31T16:54:21.555Z" }, - { url = "https://files.pythonhosted.org/packages/4c/33/e18bcfa338ca4e6b2771c85d4c5203e627d0c69d9de5c1a2cf2ba13320ba/mypy-1.20.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49d11c6f573a5a08f77fad13faff2139f6d0730ebed2cfa9b3d2702671dd7188", size = 13719585, upload-time = "2026-03-31T16:51:53.89Z" }, - { url = "https://files.pythonhosted.org/packages/6b/8d/93491ff7b79419edc7eabf95cb3b3f7490e2e574b2855c7c7e7394ff933f/mypy-1.20.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d3243c406773185144527f83be0e0aefc7bf4601b0b2b956665608bf7c98a83", size = 14685075, upload-time = "2026-03-31T16:54:04.464Z" }, - { url = "https://files.pythonhosted.org/packages/b5/9d/d924b38a4923f8d164bf2b4ec98bf13beaf6e10a5348b4b137eadae40a6e/mypy-1.20.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a79c1eba7ac4209f2d850f0edd0a2f8bba88cbfdfefe6fb76a19e9d4fe5e71a2", size = 14919141, upload-time = "2026-03-31T16:54:51.785Z" }, - { url = "https://files.pythonhosted.org/packages/59/98/1da9977016678c0b99d43afe52ed00bb3c1a0c4c995d3e6acca1a6ebb9b4/mypy-1.20.0-cp314-cp314-win_amd64.whl", hash = "sha256:00e047c74d3ec6e71a2eb88e9ea551a2edb90c21f993aefa9e0d2a898e0bb732", size = 11050925, upload-time = "2026-03-31T16:51:30.758Z" }, - { url = "https://files.pythonhosted.org/packages/5e/e3/ba0b7a3143e49a9c4f5967dde6ea4bf8e0b10ecbbcca69af84027160ee89/mypy-1.20.0-cp314-cp314-win_arm64.whl", hash = "sha256:931a7630bba591593dcf6e97224a21ff80fb357e7982628d25e3c618e7f598ef", size = 10001089, upload-time = "2026-03-31T16:49:43.632Z" }, - { url = "https://files.pythonhosted.org/packages/12/28/e617e67b3be9d213cda7277913269c874eb26472489f95d09d89765ce2d8/mypy-1.20.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:26c8b52627b6552f47ff11adb4e1509605f094e29815323e487fc0053ebe93d1", size = 15534710, upload-time = "2026-03-31T16:52:12.506Z" }, - { url = "https://files.pythonhosted.org/packages/6e/0c/3b5f2d3e45dc7169b811adce8451679d9430399d03b168f9b0489f43adaa/mypy-1.20.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:39362cdb4ba5f916e7976fccecaab1ba3a83e35f60fa68b64e9a70e221bb2436", size = 14393013, upload-time = "2026-03-31T16:54:41.186Z" }, - { url = "https://files.pythonhosted.org/packages/a3/49/edc8b0aa145cc09c1c74f7ce2858eead9329931dcbbb26e2ad40906daa4e/mypy-1.20.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34506397dbf40c15dc567635d18a21d33827e9ab29014fb83d292a8f4f8953b6", size = 15047240, upload-time = "2026-03-31T16:54:31.955Z" }, - { url = "https://files.pythonhosted.org/packages/42/37/a946bb416e37a57fa752b3100fd5ede0e28df94f92366d1716555d47c454/mypy-1.20.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555493c44a4f5a1b58d611a43333e71a9981c6dbe26270377b6f8174126a0526", size = 15858565, upload-time = "2026-03-31T16:53:36.997Z" }, - { url = "https://files.pythonhosted.org/packages/2f/99/7690b5b5b552db1bd4ff362e4c0eb3107b98d680835e65823fbe888c8b78/mypy-1.20.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2721f0ce49cb74a38f00c50da67cb7d36317b5eda38877a49614dc018e91c787", size = 16087874, upload-time = "2026-03-31T16:52:48.313Z" }, - { url = "https://files.pythonhosted.org/packages/aa/76/53e893a498138066acd28192b77495c9357e5a58cc4be753182846b43315/mypy-1.20.0-cp314-cp314t-win_amd64.whl", hash = "sha256:47781555a7aa5fedcc2d16bcd72e0dc83eb272c10dd657f9fb3f9cc08e2e6abb", size = 12572380, upload-time = "2026-03-31T16:49:52.454Z" }, - { url = "https://files.pythonhosted.org/packages/76/9c/6dbdae21f01b7aacddc2c0bbf3c5557aa547827fdf271770fe1e521e7093/mypy-1.20.0-cp314-cp314t-win_arm64.whl", hash = "sha256:c70380fe5d64010f79fb863b9081c7004dd65225d2277333c219d93a10dad4dd", size = 10381174, upload-time = "2026-03-31T16:51:20.179Z" }, - { url = "https://files.pythonhosted.org/packages/21/66/4d734961ce167f0fd8380769b3b7c06dbdd6ff54c2190f3f2ecd22528158/mypy-1.20.0-py3-none-any.whl", hash = "sha256:a6e0641147cbfa7e4e94efdb95c2dab1aff8cfc159ded13e07f308ddccc8c48e", size = 2636365, upload-time = "2026-03-31T16:51:44.911Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/f8/5c/b0089fe7fef0a994ae5ee07029ced0526082c6cfaaa4c10d40a10e33b097/mypy-1.20.0.tar.gz", hash = "sha256:eb96c84efcc33f0b5e0e04beacf00129dd963b67226b01c00b9dfc8affb464c3", upload-time = "2026-03-31T16:55:14.959Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/dd/3afa29b58c2e57c79116ed55d700721c3c3b15955e2b6251dd165d377c0e/mypy-1.20.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:002b613ae19f4ac7d18b7e168ffe1cb9013b37c57f7411984abbd3b817b0a214", upload-time = "2026-03-31T16:55:01.824Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/227b516ab8cad9f2a13c5e7a98d28cd6aa75e9c83e82776ae6c1c4c046c7/mypy-1.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9336b5e6712f4adaf5afc3203a99a40b379049104349d747eb3e5a3aa23ac2e", upload-time = "2026-03-31T16:51:41.23Z" }, + { url = "https://files.pythonhosted.org/packages/57/d4/1ddb799860c1b5ac6117ec307b965f65deeb47044395ff01ab793248a591/mypy-1.20.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f13b3e41bce9d257eded794c0f12878af3129d80aacd8a3ee0dee51f3a978651", upload-time = "2026-03-31T16:48:55.69Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b7/54a720f565a87b893182a2a393370289ae7149e4715859e10e1c05e49154/mypy-1.20.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9804c3ad27f78e54e58b32e7cb532d128b43dbfb9f3f9f06262b821a0f6bd3f5", upload-time = "2026-03-31T16:53:26.948Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2a/74810274848d061f8a8ea4ac23aaad43bd3d8c1882457999c2e568341c57/mypy-1.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:697f102c5c1d526bdd761a69f17c6070f9892eebcb94b1a5963d679288c09e78", upload-time = "2026-03-31T16:50:17.591Z" }, + { url = "https://files.pythonhosted.org/packages/77/91/21b8ba75f958bcda75690951ce6fa6b7138b03471618959529d74b8544e2/mypy-1.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ecd63f75fdd30327e4ad8b5704bd6d91fc6c1b2e029f8ee14705e1207212489", upload-time = "2026-03-31T16:52:19.986Z" }, + { url = "https://files.pythonhosted.org/packages/8a/15/3d8198ef97c1ca03aea010cce4f1d4f3bc5d9849e8c0140111ca2ead9fdd/mypy-1.20.0-cp312-cp312-win_arm64.whl", hash = "sha256:f194db59657c58593a3c47c6dfd7bad4ef4ac12dbc94d01b3a95521f78177e33", upload-time = "2026-03-31T16:53:44.385Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a7/f64ea7bd592fa431cb597418b6dec4a47f7d0c36325fec7ac67bc8402b94/mypy-1.20.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b20c8b0fd5877abdf402e79a3af987053de07e6fb208c18df6659f708b535134", upload-time = "2026-03-31T16:49:16.78Z" }, + { url = "https://files.pythonhosted.org/packages/bb/72/8927d84cfc90c6abea6e96663576e2e417589347eb538749a464c4c218a0/mypy-1.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:367e5c993ba34d5054d11937d0485ad6dfc60ba760fa326c01090fc256adf15c", upload-time = "2026-03-31T16:53:08.02Z" }, + { url = "https://files.pythonhosted.org/packages/ab/4a/11ab99f9afa41aa350178d24a7d2da17043228ea10f6456523f64b5a6cf6/mypy-1.20.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f799d9db89fc00446f03281f84a221e50018fc40113a3ba9864b132895619ebe", upload-time = "2026-03-31T16:52:28.577Z" }, + { url = "https://files.pythonhosted.org/packages/42/79/694ca73979cfb3535ebfe78733844cd5aff2e63304f59bf90585110d975a/mypy-1.20.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555658c611099455b2da507582ea20d2043dfdfe7f5ad0add472b1c6238b433f", upload-time = "2026-03-31T16:48:45.527Z" }, + { url = "https://files.pythonhosted.org/packages/84/24/a022ccab3a46e3d2cdf2e0e260648633640eb396c7e75d5a42818a8d3971/mypy-1.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:efe8d70949c3023698c3fca1e94527e7e790a361ab8116f90d11221421cd8726", upload-time = "2026-03-31T16:49:36.038Z" }, + { url = "https://files.pythonhosted.org/packages/d8/9b/549228d88f574d04117e736f55958bd4908f980f9f5700a07aeb85df005b/mypy-1.20.0-cp313-cp313-win_amd64.whl", hash = "sha256:f49590891d2c2f8a9de15614e32e459a794bcba84693c2394291a2038bbaaa69", upload-time = "2026-03-31T16:50:59.827Z" }, + { url = "https://files.pythonhosted.org/packages/91/17/15095c0e54a8bc04d22d4ff06b2139d5f142c2e87520b4e39010c4862771/mypy-1.20.0-cp313-cp313-win_arm64.whl", hash = "sha256:76a70bf840495729be47510856b978f1b0ec7d08f257ca38c9d932720bf6b43e", upload-time = "2026-03-31T16:49:59.537Z" }, + { url = "https://files.pythonhosted.org/packages/4e/0e/6ca4a84cbed9e62384bc0b2974c90395ece5ed672393e553996501625fc5/mypy-1.20.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0f42dfaab7ec1baff3b383ad7af562ab0de573c5f6edb44b2dab016082b89948", upload-time = "2026-03-31T16:52:57.999Z" }, + { url = "https://files.pythonhosted.org/packages/7d/c5/5fe9d8a729dd9605064691816243ae6c49fde0bd28f6e5e17f6a24203c43/mypy-1.20.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:31b5dbb55293c1bd27c0fc813a0d2bb5ceef9d65ac5afa2e58f829dab7921fd5", upload-time = "2026-03-31T16:54:21.555Z" }, + { url = "https://files.pythonhosted.org/packages/4c/33/e18bcfa338ca4e6b2771c85d4c5203e627d0c69d9de5c1a2cf2ba13320ba/mypy-1.20.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49d11c6f573a5a08f77fad13faff2139f6d0730ebed2cfa9b3d2702671dd7188", upload-time = "2026-03-31T16:51:53.89Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8d/93491ff7b79419edc7eabf95cb3b3f7490e2e574b2855c7c7e7394ff933f/mypy-1.20.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d3243c406773185144527f83be0e0aefc7bf4601b0b2b956665608bf7c98a83", upload-time = "2026-03-31T16:54:04.464Z" }, + { url = "https://files.pythonhosted.org/packages/b5/9d/d924b38a4923f8d164bf2b4ec98bf13beaf6e10a5348b4b137eadae40a6e/mypy-1.20.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a79c1eba7ac4209f2d850f0edd0a2f8bba88cbfdfefe6fb76a19e9d4fe5e71a2", upload-time = "2026-03-31T16:54:51.785Z" }, + { url = "https://files.pythonhosted.org/packages/59/98/1da9977016678c0b99d43afe52ed00bb3c1a0c4c995d3e6acca1a6ebb9b4/mypy-1.20.0-cp314-cp314-win_amd64.whl", hash = "sha256:00e047c74d3ec6e71a2eb88e9ea551a2edb90c21f993aefa9e0d2a898e0bb732", upload-time = "2026-03-31T16:51:30.758Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e3/ba0b7a3143e49a9c4f5967dde6ea4bf8e0b10ecbbcca69af84027160ee89/mypy-1.20.0-cp314-cp314-win_arm64.whl", hash = "sha256:931a7630bba591593dcf6e97224a21ff80fb357e7982628d25e3c618e7f598ef", upload-time = "2026-03-31T16:49:43.632Z" }, + { url = "https://files.pythonhosted.org/packages/12/28/e617e67b3be9d213cda7277913269c874eb26472489f95d09d89765ce2d8/mypy-1.20.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:26c8b52627b6552f47ff11adb4e1509605f094e29815323e487fc0053ebe93d1", upload-time = "2026-03-31T16:52:12.506Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0c/3b5f2d3e45dc7169b811adce8451679d9430399d03b168f9b0489f43adaa/mypy-1.20.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:39362cdb4ba5f916e7976fccecaab1ba3a83e35f60fa68b64e9a70e221bb2436", upload-time = "2026-03-31T16:54:41.186Z" }, + { url = "https://files.pythonhosted.org/packages/a3/49/edc8b0aa145cc09c1c74f7ce2858eead9329931dcbbb26e2ad40906daa4e/mypy-1.20.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34506397dbf40c15dc567635d18a21d33827e9ab29014fb83d292a8f4f8953b6", upload-time = "2026-03-31T16:54:31.955Z" }, + { url = "https://files.pythonhosted.org/packages/42/37/a946bb416e37a57fa752b3100fd5ede0e28df94f92366d1716555d47c454/mypy-1.20.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555493c44a4f5a1b58d611a43333e71a9981c6dbe26270377b6f8174126a0526", upload-time = "2026-03-31T16:53:36.997Z" }, + { url = "https://files.pythonhosted.org/packages/2f/99/7690b5b5b552db1bd4ff362e4c0eb3107b98d680835e65823fbe888c8b78/mypy-1.20.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2721f0ce49cb74a38f00c50da67cb7d36317b5eda38877a49614dc018e91c787", upload-time = "2026-03-31T16:52:48.313Z" }, + { url = "https://files.pythonhosted.org/packages/aa/76/53e893a498138066acd28192b77495c9357e5a58cc4be753182846b43315/mypy-1.20.0-cp314-cp314t-win_amd64.whl", hash = "sha256:47781555a7aa5fedcc2d16bcd72e0dc83eb272c10dd657f9fb3f9cc08e2e6abb", upload-time = "2026-03-31T16:49:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/76/9c/6dbdae21f01b7aacddc2c0bbf3c5557aa547827fdf271770fe1e521e7093/mypy-1.20.0-cp314-cp314t-win_arm64.whl", hash = "sha256:c70380fe5d64010f79fb863b9081c7004dd65225d2277333c219d93a10dad4dd", upload-time = "2026-03-31T16:51:20.179Z" }, + { url = "https://files.pythonhosted.org/packages/21/66/4d734961ce167f0fd8380769b3b7c06dbdd6ff54c2190f3f2ecd22528158/mypy-1.20.0-py3-none-any.whl", hash = "sha256:a6e0641147cbfa7e4e94efdb95c2dab1aff8cfc159ded13e07f308ddccc8c48e", upload-time = "2026-03-31T16:51:44.911Z" }, ] [[package]] name = "mypy-extensions" version = "1.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", upload-time = "2025-04-22T14:54:24.164Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, -] - -[[package]] -name = "flowx" -version = "0.2.0" -source = { editable = "." } -dependencies = [ - { name = "databricks-sdk" }, - { name = "pyyaml" }, - { name = "sqlglot" }, -] - -[package.dev-dependencies] -dev = [ - { name = "coverage" }, - { name = "mypy" }, - { name = "pytest" }, - { name = "ruff" }, - { name = "types-pyyaml" }, -] -yq = [ - { name = "yq" }, -] - -[package.metadata] -requires-dist = [ - { name = "databricks-sdk", specifier = ">=0.40" }, - { name = "pyyaml", specifier = ">=6.0" }, - { name = "sqlglot", specifier = ">=25.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "coverage", specifier = ">=7.6.1,<8" }, - { name = "mypy", specifier = ">=1.18.2,<2" }, - { name = "pytest", specifier = ">=8.3.3,<9" }, - { name = "ruff", specifier = ">=0.14.0,<1" }, - { name = "types-pyyaml", specifier = ">=6.0.12.20250915,<7" }, + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", upload-time = "2025-04-22T14:54:22.983Z" }, ] -yq = [{ name = "yq", specifier = "~=3.4.3" }] [[package]] name = "packaging" version = "26.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", upload-time = "2026-01-21T20:50:39.064Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", upload-time = "2026-01-21T20:50:37.788Z" }, ] [[package]] name = "pathspec" version = "1.0.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", upload-time = "2026-01-27T03:59:46.938Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", upload-time = "2026-01-27T03:59:45.137Z" }, ] [[package]] name = "pluggy" version = "1.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", upload-time = "2025-05-15T12:30:07.975Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", upload-time = "2025-05-15T12:30:06.134Z" }, ] [[package]] name = "protobuf" version = "6.33.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", upload-time = "2026-03-18T19:05:00.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, - { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, - { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, - { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, - { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, - { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", upload-time = "2026-03-18T19:04:59.826Z" }, ] [[package]] name = "pyasn1" version = "0.6.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", upload-time = "2026-03-17T01:06:53.382Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", upload-time = "2026-03-17T01:06:52.036Z" }, ] [[package]] @@ -550,27 +702,145 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyasn1" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", upload-time = "2025-03-28T02:41:22.17Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", upload-time = "2025-03-28T02:41:19.028Z" }, ] [[package]] name = "pycparser" version = "3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", upload-time = "2026-01-21T14:26:50.693Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", upload-time = "2026-05-06T13:39:29.883Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", upload-time = "2026-05-08T13:40:06.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", upload-time = "2026-05-08T13:40:04.958Z" }, ] [[package]] name = "pygments" version = "2.20.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pyjwt" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", upload-time = "2026-05-21T19:54:36.618Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", upload-time = "2026-05-21T19:54:35.362Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, ] [[package]] @@ -584,55 +854,106 @@ dependencies = [ { name = "pluggy" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "python-multipart" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", upload-time = "2026-06-04T16:18:58.647Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", upload-time = "2026-06-04T16:18:57.319Z" }, +] + +[[package]] +name = "pywin32" +version = "312" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", upload-time = "2026-06-04T07:49:28.836Z" }, + { url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", upload-time = "2026-06-04T07:49:31.85Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", upload-time = "2026-06-04T07:49:34.244Z" }, + { url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", upload-time = "2026-06-04T07:49:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", upload-time = "2026-06-04T07:49:38.876Z" }, + { url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", upload-time = "2026-06-04T07:49:41.083Z" }, + { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", upload-time = "2026-06-04T07:49:43.188Z" }, + { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", upload-time = "2026-06-04T07:49:45.34Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", upload-time = "2026-06-04T07:49:47.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", upload-time = "2026-06-04T07:49:50.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", upload-time = "2026-06-04T07:49:54.857Z" }, + { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", upload-time = "2026-06-04T07:49:57.531Z" }, ] [[package]] name = "pyyaml" version = "6.0.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, - { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, - { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, - { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, - { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, - { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, - { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, - { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, - { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, - { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, - { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, - { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, - { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, - { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, - { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, - { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, - { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, - { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, - { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, - { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, - { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, - { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, - { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, - { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, - { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, - { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, - { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, - { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, - { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, - { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, - { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", upload-time = "2025-10-13T15:30:47.625Z" }, ] [[package]] @@ -645,88 +966,249 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/43/25a8dcd3feedd735039a8f0b5b7e3b118232b5eae288c4fd9ab200d41094/rpds_py-2026.5.1.tar.gz", hash = "sha256:07b24fea40541e28570e5b795a4a38fbdcd12550c06bd0748005ecc8116ca256", upload-time = "2026-05-28T12:02:13.232Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e7/a78582dc57caa592dcc7d4fb69b61390561e908eb3d2f5df5928a8e354c0/rpds_py-2026.5.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3abe24a66e57adcfa645d718063a5fa5103ecc71ddbf26d78af8f9368018ff1d", upload-time = "2026-05-28T11:59:12.531Z" }, + { url = "https://files.pythonhosted.org/packages/a3/43/35e3f136343aef451e545ce8c38d36c2f93c0ed88703db8b64ba2b205c68/rpds_py-2026.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58b1d94308ddf0b1982f61f2eb54bf92997c9ece8a8093ef014250f4a517906c", upload-time = "2026-05-28T11:59:13.827Z" }, + { url = "https://files.pythonhosted.org/packages/20/e1/0f2160c5982d3157734d5cb3ed63d8b2d583a73c9864f77b666449f32cf8/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fa92420128dadce7f54bd73ba1825a273e9268fe9e35dbf7e6362890efa4e08", upload-time = "2026-05-28T11:59:15.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/11/ee0ba42aff83bf4effdbc576673c6be64c5e173978c3f6d537e94482f77d/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca653c6546386227cd9800d1bef6a348099acf8db4250341da6d90f663d6dfcb", upload-time = "2026-05-28T11:59:16.665Z" }, + { url = "https://files.pythonhosted.org/packages/11/df/d94aa6a499d4ac40afe2d7620f2c597fd3c0f182e854ad7cf3f596a81cb6/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66c93681c4729e4e3ecba31b8179fae083ff3118841672835140338b4b9867c1", upload-time = "2026-05-28T11:59:17.991Z" }, + { url = "https://files.pythonhosted.org/packages/1f/75/33d30f43bb2f458de11979486a591b1bf6e5651765ed1704c6197c2dc773/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ff257542e04796880e011e15cd4dc21c2599975df2aaa8f2c8495ca574e1a5", upload-time = "2026-05-28T11:59:19.434Z" }, + { url = "https://files.pythonhosted.org/packages/f4/1e/2c9096fc19d5fd084b0184ca2b651e659aa0a37e6fdbecf6ece47f147fe1/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6825cc329b290e93c5f6a9be2393118a763f6ccf6abd83704e0c102ca583644", upload-time = "2026-05-28T11:59:21Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e5/61ec9f8be8211ea7f48448195549e4aaf02004083475493b0e137702ecb2/rpds_py-2026.5.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:de42116e69cb53b911cc34aee5ab98f36c597b822545045d49e938818b99e5e4", upload-time = "2026-05-28T11:59:22.454Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ca/bcec1005c4f4a234f92a29078631fee49206c7265ccae966f18fd332e80e/rpds_py-2026.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0f920015df2a504bebaba6d4c31ccf3fcf942f92655c086da30b671aad19aa6", upload-time = "2026-05-28T11:59:23.845Z" }, + { url = "https://files.pythonhosted.org/packages/72/e6/4d5718c5cf26c522dc7c9999e238da1e77380b81d0c5d1df11e271ddfeb1/rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0408a24e44feb919423dc6d9da677cb5cddb894d2ca9e763967d156d9c60fab4", upload-time = "2026-05-28T11:59:25.184Z" }, + { url = "https://files.pythonhosted.org/packages/d4/25/2ee807bdb3e1f0b7eddf7782acd5665a8b5205a331a7d7244a52c4812fd9/rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cea68bcd53467561ae2f96a6bdad1544299ba97b5b0ddcd5ac3d376e5c781c24", upload-time = "2026-05-28T11:59:26.749Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c1/7d4c26f167f8c41501cc073d30ee22082b16ce358cf5b00ec97cbc7804ea/rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4be8b1d2a705cc37d08256004e1d07de143fa0075c8e85a3df020b776f62b732", upload-time = "2026-05-28T11:59:28.11Z" }, + { url = "https://files.pythonhosted.org/packages/04/1d/9d12b0a337bab46f4769f8857f4007e3b2d639e14f9a44a0efe157696e64/rpds_py-2026.5.1-cp312-cp312-win32.whl", hash = "sha256:6736718bd4fc49cbcb538ba30516fdbef161522acefb739657d48b97bd864fed", upload-time = "2026-05-28T11:59:29.689Z" }, + { url = "https://files.pythonhosted.org/packages/c5/93/e4116f2de7f56bc7406a76033dc501811ddeb22b7f056b92d632871ebb0c/rpds_py-2026.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:0a7d1eec967df0e9b22614a5e177622e0c89611d03727fa0cb48e45028907870", upload-time = "2026-05-28T11:59:31.033Z" }, + { url = "https://files.pythonhosted.org/packages/cb/53/6c3419d85eb2ec5938a37627c585b42d76a63bb731d6e42ed4b079ebf486/rpds_py-2026.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1841d067089e117142d79b98aa0df2f08b52f2ecc1819dd2700636c0db74a473", upload-time = "2026-05-28T11:59:32.318Z" }, + { url = "https://files.pythonhosted.org/packages/6c/32/14c961ad295f490eb0849ada8b79683e93a59b9de3afdd983eaf55fa6867/rpds_py-2026.5.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:efef4ac29c6ff495531eb17ee705b62841ecaa291b7c7077e848ea03e237164d", upload-time = "2026-05-28T11:59:33.655Z" }, + { url = "https://files.pythonhosted.org/packages/ca/bb/d1b85117967c11191441a7274ae616c65d93901d082c588f89a50a8da5ae/rpds_py-2026.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c39f5b67a8a2e67179ada2a954227d670fe65fa9098457f698f56ddf248709b3", upload-time = "2026-05-28T11:59:35Z" }, + { url = "https://files.pythonhosted.org/packages/7c/46/d84105f062e626a1b233f863907288a4708c2d833b8b4c6fb2764bc080c0/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5c30f3f04eef4fbd362226a6f31d7c8895ca4fbb6e0b790f6890a98d8da8559", upload-time = "2026-05-28T11:59:36.43Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/469d7959ce5b1201e1de135dc735b86db3b35dd0d1734f6a44246d5f061c/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:277f6c82f0580848796c7ecc8a7173aa3bfb928e4ff831261c2f60a81dc270db", upload-time = "2026-05-28T11:59:37.995Z" }, + { url = "https://files.pythonhosted.org/packages/dc/a2/57853d31a1116a561aa072794602ad3f6341e18d70a8523f1bd5b9fc1e5a/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63c2c4c213f1a4e3f3de28ecab029dbdee976324e729c0d7a55211be72576b02", upload-time = "2026-05-28T11:59:39.453Z" }, + { url = "https://files.pythonhosted.org/packages/99/63/3a8eabcad9314b7daf5c65f451d2c33d989235cd8a5762186cf2c3f5a4f8/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3350ec808fb538fe71a1f94dfaa0e29c598dfad805ce49f0caec5ae3183c652b", upload-time = "2026-05-28T11:59:40.896Z" }, + { url = "https://files.pythonhosted.org/packages/4b/25/05678d97fc25e2622df14dc530fb82023174ecfff6733991ed0d78f167bd/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1b964e3ab599e718dc46c018d104b1ebc007cbc6567d827c94a687fca56d77e", upload-time = "2026-05-28T11:59:42.626Z" }, + { url = "https://files.pythonhosted.org/packages/88/d1/8c90b6431e80a3b91b284a5c7c8c0c4f9c006444d90477a740d6e0f9c694/rpds_py-2026.5.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:19cb09fab7b7fc96b2a6e28f2e34b72a3705ff27b37edb77455316e5d3f3dc9b", upload-time = "2026-05-28T11:59:44.124Z" }, + { url = "https://files.pythonhosted.org/packages/ff/99/4638f672ab356682d633ee0da9255f5b67ce6efd0b85eb94ad3e255e65a5/rpds_py-2026.5.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abe76bcdba31e576cb83eeb8797aa0d882b738fef6dc65d0601fc753806a5b46", upload-time = "2026-05-28T11:59:47.177Z" }, + { url = "https://files.pythonhosted.org/packages/66/3f/3546524b6eb4cc2e1f363a3d638fa52f6c24faae3500c25fb488b02f1740/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8bff7073db3899158fff55ebf57b113a67030af26f80a18978f9f0aa60250ddf", upload-time = "2026-05-28T11:59:48.603Z" }, + { url = "https://files.pythonhosted.org/packages/c6/c3/7b3388c796fcf471bd17194242d4dc1a7608567c0fa422bcc1c5e79f9c1e/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8ba264fa49be666cd9cc56bf34ec7002fb3d27a4aee5bcb4d43d0d18feb1bb6f", upload-time = "2026-05-28T11:59:50.314Z" }, + { url = "https://files.pythonhosted.org/packages/61/1e/a3cb07f2795075d1d88efddae2f541359fde5f08c81ee114c29c2949c90a/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4860b603ddda0475a8885499b3729e90229d480105b42651962a5397d995fa89", upload-time = "2026-05-28T11:59:51.673Z" }, + { url = "https://files.pythonhosted.org/packages/a1/74/e758c03a5ef46f04c37f2651a2893db846d569ba8a7bca469d4b58939bcd/rpds_py-2026.5.1-cp313-cp313-win32.whl", hash = "sha256:7944270ae71383f6e2657dd7d5ce4eeb4ac2d0059a6738f0510583d462ab4842", upload-time = "2026-05-28T11:59:53.148Z" }, + { url = "https://files.pythonhosted.org/packages/70/ec/a2aca432db9c7359b40fa393eeeaa0d166c2f70175be956e75fa24197c44/rpds_py-2026.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:88647f43a73c4e01be19b04ceef0c8d3a1958153604d13c773becd8016f2a0cf", upload-time = "2026-05-28T11:59:54.505Z" }, + { url = "https://files.pythonhosted.org/packages/29/60/a73bfdd45b096574556acf303bbd9fa9eed36ca8a818b514e2a5d5fe2b9d/rpds_py-2026.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:453895624ecf7db7063b1004e44037522bbaef9ff6a945e59bc71662d7a03abd", upload-time = "2026-05-28T11:59:56.081Z" }, + { url = "https://files.pythonhosted.org/packages/18/e2/408105fd611823f00882aea810f3989a30d26b1bab8b6beb20f98c724e0e/rpds_py-2026.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:b4e4bc98639ec915f512fde3aa7a95e0041d95d9c3cc86eea841fa63cb1e8600", upload-time = "2026-05-28T11:59:57.448Z" }, + { url = "https://files.pythonhosted.org/packages/8d/58/5c4a43436843c90d0f6d19f82c200c80e3843ca9fa07b237623327f6d384/rpds_py-2026.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cacedb7a6e167680acba45ad5716e89067d225dc80da0d7040cae8c81d4572fa", upload-time = "2026-05-28T11:59:58.881Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c2/1a71acdacaf4e259b10278fb87b039ded3cf80041bcd89dd8a3ea702ded6/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68700371c5d7ae1412862ddfa719090925c93ecf351c566d66f09d04b136ea00", upload-time = "2026-05-28T12:00:00.516Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c8/535f3d9b65addd8e28aa87b83c6e526799c3717a88273db8ea795beeef7a/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:296c799becfa849c779c8725494fe9ed94959ed886787df4364b058465bad7f0", upload-time = "2026-05-28T12:00:02.394Z" }, + { url = "https://files.pythonhosted.org/packages/1c/91/dc033f313345c354ade914dbe73cdb90b615a4409ea02430d5356794f3d8/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3858b908218ee108d0bbfb2095ccc237648053c9bf98affad7cb079acaf1d97", upload-time = "2026-05-28T12:00:04.189Z" }, + { url = "https://files.pythonhosted.org/packages/27/fc/90fcbea459dbb8ddc18a2e0fd1de9412b48bc84ffff2db771cf714bacfd6/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4fb8d2e7cb2f850b169806d61d1b991738acec96500a75c30f49caf064ce7cef", upload-time = "2026-05-28T12:00:05.797Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1d/46cd11a228c9750684a798d98f878be6f614aa762438da7378f035e79e35/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27b74c10ed6a8f190f4287f53bcfea348b92a84a9c9f70d30183d1e6172d580d", upload-time = "2026-05-28T12:00:07.433Z" }, + { url = "https://files.pythonhosted.org/packages/24/4a/d9b0c6af3a1de03eb93741bbe8be2bdce84d8fda8224f3005451d86df389/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:b9a6528956191c48c52294a592dbd4a8386d7048bdb25c0efcb6b966466c6d83", upload-time = "2026-05-28T12:00:09.227Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b4/db7aaabdda6d020afc87d981bcc2f57a434c7dec60ecfc2ab3dd50b20351/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:af03e34e860047bc7a352b842856fcf78798fbb81132cc98bd2f907ab4eb9cd2", upload-time = "2026-05-28T12:00:10.779Z" }, + { url = "https://files.pythonhosted.org/packages/08/d6/070f6a41cbb343e2ac4171859bf3f3623e0ab002f72619d6d505313ec2de/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fea6e836d10abbe191d557d33bd58bd5987725fe63aa1eefe557d230209855bd", upload-time = "2026-05-28T12:00:12.443Z" }, + { url = "https://files.pythonhosted.org/packages/75/ab/1a71ea3589c4345dac0a0518f0e6a031cb42689277851b683c46d27463a5/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:fc0c0f878ea770a0a8a462456c5ad36fc9fe6358e6b76fdadc7f17575e0b8bf1", upload-time = "2026-05-28T12:00:14.09Z" }, + { url = "https://files.pythonhosted.org/packages/8a/22/9bf80a56069c0c443fcfefac639a86a744550a2898817a6dfd3e26654924/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e0b360f316d966b048b085857630b3cc51f3db2f07b06f440eac8f695374d1e3", upload-time = "2026-05-28T12:00:15.66Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/3b2c0a75c9e04125696f84ebdbbf304acf5a40b58ba4481cdb98a922c3ba/rpds_py-2026.5.1-cp313-cp313t-win32.whl", hash = "sha256:a2999883eedf72fdfb7520b92c7d4ec2572a71ff40239377aa604cc529eecafc", upload-time = "2026-05-28T12:00:17.291Z" }, + { url = "https://files.pythonhosted.org/packages/e7/8b/609157d5a25d37d4f29f92840ba531f416907c34ae5c5739dd21fc2bef98/rpds_py-2026.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e07be2a9d7122bd6e82dea89814ef8dc893feb1aae97fec1630f3263bbb30e55", upload-time = "2026-05-28T12:00:18.73Z" }, + { url = "https://files.pythonhosted.org/packages/d4/6f/19c1918a4b590d8de87e712e4abe4b3875771eff60216fb6153cf6665c68/rpds_py-2026.5.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1f2c391c3059798093b65df23aca2cac150460ae9c630d99dec83d703d9485b9", upload-time = "2026-05-28T12:00:20.217Z" }, + { url = "https://files.pythonhosted.org/packages/e5/60/a06fe7da34eca79dacbf958a2ba0c6eea85bc2b29de20080bf40f72f66fa/rpds_py-2026.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:413b424f7c4ee65ab5e5be91f5731be0f8b41a1ee2b12dfe810d716312e95a78", upload-time = "2026-05-28T12:00:21.711Z" }, + { url = "https://files.pythonhosted.org/packages/bf/ec/b2333b97b90e2a6ef6ca8ad386ee284968e74bcfe113b3f1a8d9036429a9/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c595a1d9255dce0599e13130d1440ab2506654f2b50294226ee06402f8fef63", upload-time = "2026-05-28T12:00:23.326Z" }, + { url = "https://files.pythonhosted.org/packages/14/7f/e00aae54067f2b488c4637961d5f58204d470795fc791085fa3f15060d2e/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1c27c5f6102eac8c03e7595a00827a53b271ba40a53b59ff8709170e0855ea4a", upload-time = "2026-05-28T12:00:24.89Z" }, + { url = "https://files.pythonhosted.org/packages/be/cc/423999bbb8ae8dc93c77fc1d5e984ade5eb89d237d3bb884ccfa72ae2890/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c7fcf61d44cacecaf3aea542b0e053db77972a4573e7ceda16fb2b399161195", upload-time = "2026-05-28T12:00:26.676Z" }, + { url = "https://files.pythonhosted.org/packages/0f/aa/c671bf660f12e68d3c52ff86c7066ed1372df5a0f4f2ff584e419b8207e7/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c817a189d4ee14290420e5ff051e4dd6baa13f3edf84685071dee07a6d538ee", upload-time = "2026-05-28T12:00:28.577Z" }, + { url = "https://files.pythonhosted.org/packages/19/c8/d63bb75b68afe77b229e3021c6031bcaf01da5db5b0e69d0d10f9ba679a7/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21846aac0ed2e0589f38c12dc44e77bb64e494b771eadbcf169cba00566ba7ba", upload-time = "2026-05-28T12:00:30.304Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/c51122014d8274ff37dc606d60049c3db7d83da02b5b282511e5a906a9a6/rpds_py-2026.5.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b317c87a13f769a4e787819bd508aaa5d69aa09b0880de9af6d3a8a54571cdec", upload-time = "2026-05-28T12:00:31.764Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f9/2790cb99c136a5363acdeacf5c27c56f3de0d4118a1f48fca83404c99c89/rpds_py-2026.5.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce87129d9f2c14fa6c4a8601fb80eb4488c80d38a20cd13758ef11123e14995d", upload-time = "2026-05-28T12:00:33.247Z" }, + { url = "https://files.pythonhosted.org/packages/e5/1b/e4fb584f8c75d35c38150ff6a332cda949e6f97acba1f4fd123b14ab56fe/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9cdddb6c1207d284d94fd1530adf57fbd797fe7c4b8704ba85f49414f2557e7d", upload-time = "2026-05-28T12:00:34.819Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f7/a6731b4216cb3793ea1af5391da240f5683dacc0d13e034fe5fc3503f240/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:4e237e139f94d3c036fd28eb9f564c99055476ff4ff05cd42be55ce349b5aa02", upload-time = "2026-05-28T12:00:36.268Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/2e051a81d95d8e63f4b35a1c463a87e8766bc3d083c067c5dfb6bf220747/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ed0954b524873214369184a9c82b0eaa45a3fbb9a798cd95b17e0d98499e7ea0", upload-time = "2026-05-28T12:00:37.82Z" }, + { url = "https://files.pythonhosted.org/packages/65/56/b5f6fdb2083e32bca8a8993d89e70db114b4756c9e2c38421328126689d2/rpds_py-2026.5.1-cp314-cp314-win32.whl", hash = "sha256:2d88621d6a7d4dfa633d21abe90f280bb205274e16b1d1e61c6ad4640b2453b7", upload-time = "2026-05-28T12:00:39.492Z" }, + { url = "https://files.pythonhosted.org/packages/fb/80/65a5aa96c155e611d1ed844e4e1f57f3e36b021f396d9f8585d756e6b90d/rpds_py-2026.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:cef8ac28d26f4dda3533060c20fbf80a325458fa9fd23ea72a73cdfa8e978838", upload-time = "2026-05-28T12:00:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/27/7c/ad185212e87b05f196daef92bc5f3caf07298eb47c295b5585c3dd3093ac/rpds_py-2026.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:eaaea962c68cdc68d4a533ba985ab8e9484277910bbfaa2ab3ef7732667bfed8", upload-time = "2026-05-28T12:00:43.15Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/e14ae18759020334646b031e708ab4158d653a938822bfb7b95ef2e93aa3/rpds_py-2026.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:21942f52dbbd5f8758bf021213d28bd45c39e873e65e2407faf5f1846f5761ad", upload-time = "2026-05-28T12:00:44.638Z" }, + { url = "https://files.pythonhosted.org/packages/31/9b/5f4a1e2f960bca3ac5d052b139dd31eed97b259f9d909173821760d542e8/rpds_py-2026.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f414556f6e3958300ff941e40c9f97e3dc9774ddd1b3434c475d73dd354bbed3", upload-time = "2026-05-28T12:00:46.14Z" }, + { url = "https://files.pythonhosted.org/packages/1a/71/1d9574d6a2fa20ab60eaa55c7467f5aa20cbc770f341a05f09c0876f59e2/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef1013a8625c74043210190b246f5b1551e09757c1f356c6e4160ef96c5bc081", upload-time = "2026-05-28T12:00:47.531Z" }, + { url = "https://files.pythonhosted.org/packages/0c/9a/37e99f4915a80aa71670263c1267f7ae0af95f53a3f61e6c3bdc016d4515/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cc68e231a77a5f0d774ae278a1f8e55c0456501820847c1e4efb3829f3441df6", upload-time = "2026-05-28T12:00:49.216Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ff/6e73f74b89d2e0715e0fc86b7dde893f9a61ae2f9b256ff3bdfe41ac4e94/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9baffb505aff33acc69b422a19f77806680f3c8632227d79f48de8a810d1c2c5", upload-time = "2026-05-28T12:00:51.111Z" }, + { url = "https://files.pythonhosted.org/packages/ea/e0/425faba25f59d74d4638b267f7c7a80e8649d2ef4db10a19b0c4a71e6e6f/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8d2f912928d426e8cfa396f7f3f8d29a59e6689c86dcca3c420730c1096322b", upload-time = "2026-05-28T12:00:52.77Z" }, + { url = "https://files.pythonhosted.org/packages/c6/76/7a41960e3fddae47fab43a28684d5da981401dffd88253de0944148654cb/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90f628283be835db980c941767d41c9a27b5239e54ba0a9c1335247e82406964", upload-time = "2026-05-28T12:00:54.215Z" }, + { url = "https://files.pythonhosted.org/packages/27/60/5f38dc70824fc6951b51d35377e577a3a3a4c81a6769cc5a2de25ebe0ad1/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:1ebb2f0ab7e16132995a72de805170e0203df0c3dd22e1ef1cd1fdd90bd7a131", upload-time = "2026-05-28T12:00:55.673Z" }, + { url = "https://files.pythonhosted.org/packages/60/1a/d60a38caa1505f4b9483c3fbbde12c94e1079154f4f401a6da96f7e77621/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f3df3d16ded76f1f8c9cdebd0e1ea55fdf4c23b812de189814da7cf229c22a81", upload-time = "2026-05-28T12:00:57.518Z" }, + { url = "https://files.pythonhosted.org/packages/87/ff/602fd3f174d6425f0bce05ad0dfbec0e96b38d0f7d08a79af5aa20083885/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9af8905b8f854990e40d5206aa5ac58d9b0fe0b7f351ff2bb086c20f6c8c6a47", upload-time = "2026-05-28T12:00:58.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c1/1be13327acdbead3eca1fde03b6a34dbb011f1e864e217f0d32cc1779a7f/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:036a36a87fb1cd3b214d11c4b3c4f7d2ddad933625dca1c900b56a057c07740a", upload-time = "2026-05-28T12:01:00.656Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d7/afb49b49d7f2be8b7ba1a9f0977fa5168003437b93086726f066544e8351/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:62ae3853454fe9ef283a03c96c2d835d39e84b14643a9d62c82ef0fb87d702ca", upload-time = "2026-05-28T12:01:02.22Z" }, + { url = "https://files.pythonhosted.org/packages/25/d1/dbef8c1f8a10f07beb62b5f054e20099fd9924b3ec001b8f0b6ac7813a85/rpds_py-2026.5.1-cp314-cp314t-win32.whl", hash = "sha256:6c3d771a46ec18b12af06ce36243a9a80b07a5d0515236332d90863ca8bb326a", upload-time = "2026-05-28T12:01:03.821Z" }, + { url = "https://files.pythonhosted.org/packages/2a/72/bfa4e61ab8e7dc1c8adf397e05e6cbdd4239357bd72b248d3de662f23915/rpds_py-2026.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c93c629be4636cf54337bd5f06c104d55e42ced54d681f6fe21ae510a65116f6", upload-time = "2026-05-28T12:01:05.194Z" }, + { url = "https://files.pythonhosted.org/packages/27/3a/7b5da92b640f67b6717ccafc83cdd06bfa7ff2395c3685c68922bb54d703/rpds_py-2026.5.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:3574b55c604b8f75dacb007136508bbc0db406e626301778096a133327e7f2fb", upload-time = "2026-05-28T12:01:06.722Z" }, + { url = "https://files.pythonhosted.org/packages/d7/8a/2aafd7ad355a1bd48ca76e2262b74b15e6432b5a1efe150efd4d779cd55d/rpds_py-2026.5.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:94068eb3ae6d43f5a786b7db96a406a34e6d5c24489feef32fd6e8946ea7b291", upload-time = "2026-05-28T12:01:08.441Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7d/6c9523c1abbe840a1b7fba3c516d48e1d3487cc80fea4366c4071cf56784/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a5b10e8ce894825f380a8f1b6444cf73c294dfea62afbb2d13e3a9e630cec1", upload-time = "2026-05-28T12:01:09.934Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5d/0b7b03fb1dc509321f01de3149784ab773e34c8573022029af8076afcb9c/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fc09f82e63d4bcd58149572f857a431bae851dc747e313c3b5bdf7abb907fda8", upload-time = "2026-05-28T12:01:11.48Z" }, + { url = "https://files.pythonhosted.org/packages/d7/e2/8ef6012999ebf1cb1c22f876d9ce5e63d960fd4631d2af3202d3f480aa25/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e10464d17df3b582745c25cec695cb9558bca2cb6ddb631aee1787fc72c767b2", upload-time = "2026-05-28T12:01:13.051Z" }, + { url = "https://files.pythonhosted.org/packages/80/af/1eeb029bec67582c226b7809172207cd005073af4ebd906e65ff494f4983/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ba05adbf15d994c38ec0b7ab32e858e5110c21e9009a00a86545fd220f84e038", upload-time = "2026-05-28T12:01:14.631Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/ffbe10711c4d766c1cab0557d6906c074f795814863c67b351355d29354a/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77c004fdc7b891967106f78ddfd7b076bfe6813c6139c6fff6aed3bcaa960b26", upload-time = "2026-05-28T12:01:16.153Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3a/30ba4a6ad457e5b070c18d742a33fb77d8d922b565cc881f8a5313d63bfe/rpds_py-2026.5.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:83bcf894486c9d78dd290d3c0124ff6dd8875d3025e2090a8ec49fcc37c55fdd", upload-time = "2026-05-28T12:01:17.809Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/62e242b53ce39c0814bd24e1a6e6eba6c92be716277745f317f9540a2e7b/rpds_py-2026.5.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3df104083952a0e0c6f10de33e440eabe98fb6317d23e1a58c68f6df08d01b9", upload-time = "2026-05-28T12:01:19.419Z" }, + { url = "https://files.pythonhosted.org/packages/38/c1/a770b9c186928a1ed0f7e6d7ae50e7f3950ed23e3f9e366dbc8e38cb55de/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:980450826cf22e133c57e0835070bdd0dd3f73b9b708c3ce223def2cb9469e14", upload-time = "2026-05-28T12:01:21.013Z" }, + { url = "https://files.pythonhosted.org/packages/21/7c/68e8579b95375b70d2a963103c42e705856cdb98569258bd807f4423891c/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:205dde846f24332ab0c1188699a043b8d165b79bb84529ce272c45048ff6be01", upload-time = "2026-05-28T12:01:22.548Z" }, + { url = "https://files.pythonhosted.org/packages/70/a1/a6135aed5730ff03ab957182259987ac11e55fb392a28dc6f0592048a280/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3966b82dd563176396df030f3dd52a6e54cb69b718e95e78bd555ed3d1e0185d", upload-time = "2026-05-28T12:01:24.118Z" }, + { url = "https://files.pythonhosted.org/packages/09/6e/f24201a76a84e6c49d0bdfdfcb735210e21701e9b21c5bfc0ba497dd62f6/rpds_py-2026.5.1-cp315-cp315-win32.whl", hash = "sha256:7818f8d0a415be74d2be3590b0a1c1f463a642f4d0217e7d10602dceef5b79aa", upload-time = "2026-05-28T12:01:25.522Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e4/966bc240bb0485fc265278f6de44d05834bf0b3618886e0b22e33d54c49a/rpds_py-2026.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:b3cc20c0d800af78fd0fac68086e28c1856cec51ea528bb81ea851aa40d39325", upload-time = "2026-05-28T12:01:27.062Z" }, + { url = "https://files.pythonhosted.org/packages/5c/5c/a15a59269cd5e74472734516c73795c15eccfc841b3d4b0228c3f53f19d0/rpds_py-2026.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:3609e9939a8a76cd904cf98a3f1f13b5dc7e150adeaee89e0ea09652ea213e16", upload-time = "2026-05-28T12:01:28.51Z" }, + { url = "https://files.pythonhosted.org/packages/e0/22/135ce03804e179a71ceb13be095deda4a279bc88f7a6b8fa161c5ad44e12/rpds_py-2026.5.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:5d333a7127d4b307601ac37792bee01bb95c867cbfacf21b6375b804d6bbd723", upload-time = "2026-05-28T12:01:30.214Z" }, + { url = "https://files.pythonhosted.org/packages/3b/5f/f1f6d2652eb9d848f6eb369d8db83a2da6249bb49ad2c2a48f45d54538d3/rpds_py-2026.5.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:b5f077b44a4f7808520f66dae234988d867deb9aed9be5da057ce9ba831b2a41", upload-time = "2026-05-28T12:01:31.656Z" }, + { url = "https://files.pythonhosted.org/packages/88/66/b74182775691ea2290c99e52ac8d5db844e56fbec90ce421f107658c8314/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d8f9b7b78c9538fc9e04e82ec0e888ff0c3cffcfad152c77e57cd09351a98a", upload-time = "2026-05-28T12:01:33.136Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8f/15e5a61d9f0a43902d36561d4f07cae6ae9f4716be825159fd72717f33af/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e3a8ae58895ac107ed934a6bf51e5846f95c53b9b940c2c6d310838fd5846358", upload-time = "2026-05-28T12:01:34.574Z" }, + { url = "https://files.pythonhosted.org/packages/02/c3/f859b12763a80540cdf2af0f15b19904cf756a71d7bdd3f82ff3e5b1bbf9/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0957cf3c2b8632ec7aaebffebea8005b353cc2a237b6e2ae3c2cac0820704cfb", upload-time = "2026-05-28T12:01:36.127Z" }, + { url = "https://files.pythonhosted.org/packages/1c/c7/ff27c2ac8411d30b03b1829fd88cae8dad1a4d0da48dd25e57c4038042e6/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c396c1304de421050b3681ea70f371874b54d41b0151e96109758144c231e30b", upload-time = "2026-05-28T12:01:37.635Z" }, + { url = "https://files.pythonhosted.org/packages/6e/67/fe92ee32a6cc05c77228a2f8b1762e7124f386ec20ff83d0757b762d58d0/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad1bff7f666b9598e573815affd666aac6a13a585dde336f843e33350c7fadc", upload-time = "2026-05-28T12:01:39.307Z" }, + { url = "https://files.pythonhosted.org/packages/f8/91/b4d6685c27aba55bd82f25b278be8237038117d05f9659a6213ad3408130/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:656a042550878f12d45752452d47094b7cfe5ad1e9d7b87b5a22ad3ae5ff8015", upload-time = "2026-05-28T12:01:41.043Z" }, + { url = "https://files.pythonhosted.org/packages/bd/79/2c1d832a53c8e0f8e98fc970ec257b950fecd4f62be2ab7182b500a0cbc8/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73c4bd4f70294737b5206a3e8e30ccadbf8a60301831c8ea23eec5dbeea1ecfa", upload-time = "2026-05-28T12:01:43.032Z" }, + { url = "https://files.pythonhosted.org/packages/78/c4/c98117b03c6a8581ab2c2dfccfe9a5ad82bd8128a3c28b46a6ad2d97c393/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:43bca78665423cabae77146f2fe7ce55272b6c8d55d82cca83effd42c7e13972", upload-time = "2026-05-28T12:01:44.648Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c1/bc479ca069200af730881b1bd525e3114b2b391a351509fcb1b772f28086/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:42d0f20e85e549c870749d0e247f0c10d318a45b7e9676d575d2dcb04a1b2e66", upload-time = "2026-05-28T12:01:46.337Z" }, + { url = "https://files.pythonhosted.org/packages/77/65/38ab2f90df44c2febfb63cc10ced40763d9b4bc94d173e734528663fe7f5/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b1be5c35683684d5331b93600c210e8367c254683d8a6df6bd21bd2da3a334fb", upload-time = "2026-05-28T12:01:48.109Z" }, + { url = "https://files.pythonhosted.org/packages/15/2d/ce1f605fe036aadd460e5822e578c6c7ec3a860936cca37d6e0f299daa77/rpds_py-2026.5.1-cp315-cp315t-win32.whl", hash = "sha256:75808f6c38ce7749bb68cc2770161aae5045e6c6f6781a9782e74b93304399df", upload-time = "2026-05-28T12:01:49.648Z" }, + { url = "https://files.pythonhosted.org/packages/79/cb/966040123eb102371559746908ef2c9471f4d43e17ec9a645a2258dab64b/rpds_py-2026.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:90bd6630002a1c7f09e7843dd79f0d24f3d2897cc25a753480917865d14f15b3", upload-time = "2026-05-28T12:01:51.408Z" }, ] [[package]] name = "ruff" version = "0.15.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/14/b0/73cf7550861e2b4824950b8b52eebdcc5adc792a00c514406556c5b80817/ruff-0.15.8.tar.gz", hash = "sha256:995f11f63597ee362130d1d5a327a87cb6f3f5eae3094c620bcc632329a4d26e", size = 4610921, upload-time = "2026-03-26T18:39:38.675Z" } +sdist = { url = "https://files.pythonhosted.org/packages/14/b0/73cf7550861e2b4824950b8b52eebdcc5adc792a00c514406556c5b80817/ruff-0.15.8.tar.gz", hash = "sha256:995f11f63597ee362130d1d5a327a87cb6f3f5eae3094c620bcc632329a4d26e", upload-time = "2026-03-26T18:39:38.675Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/92/c445b0cd6da6e7ae51e954939cb69f97e008dbe750cfca89b8cedc081be7/ruff-0.15.8-py3-none-linux_armv6l.whl", hash = "sha256:cbe05adeba76d58162762d6b239c9056f1a15a55bd4b346cfd21e26cd6ad7bc7", size = 10527394, upload-time = "2026-03-26T18:39:41.566Z" }, - { url = "https://files.pythonhosted.org/packages/eb/92/f1c662784d149ad1414cae450b082cf736430c12ca78367f20f5ed569d65/ruff-0.15.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d3e3d0b6ba8dca1b7ef9ab80a28e840a20070c4b62e56d675c24f366ef330570", size = 10905693, upload-time = "2026-03-26T18:39:30.364Z" }, - { url = "https://files.pythonhosted.org/packages/ca/f2/7a631a8af6d88bcef997eb1bf87cc3da158294c57044aafd3e17030613de/ruff-0.15.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6ee3ae5c65a42f273f126686353f2e08ff29927b7b7e203b711514370d500de3", size = 10323044, upload-time = "2026-03-26T18:39:33.37Z" }, - { url = "https://files.pythonhosted.org/packages/67/18/1bf38e20914a05e72ef3b9569b1d5c70a7ef26cd188d69e9ca8ef588d5bf/ruff-0.15.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdce027ada77baa448077ccc6ebb2fa9c3c62fd110d8659d601cf2f475858d94", size = 10629135, upload-time = "2026-03-26T18:39:44.142Z" }, - { url = "https://files.pythonhosted.org/packages/d2/e9/138c150ff9af60556121623d41aba18b7b57d95ac032e177b6a53789d279/ruff-0.15.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12e617fc01a95e5821648a6df341d80456bd627bfab8a829f7cfc26a14a4b4a3", size = 10348041, upload-time = "2026-03-26T18:39:52.178Z" }, - { url = "https://files.pythonhosted.org/packages/02/f1/5bfb9298d9c323f842c5ddeb85f1f10ef51516ac7a34ba446c9347d898df/ruff-0.15.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:432701303b26416d22ba696c39f2c6f12499b89093b61360abc34bcc9bf07762", size = 11121987, upload-time = "2026-03-26T18:39:55.195Z" }, - { url = "https://files.pythonhosted.org/packages/10/11/6da2e538704e753c04e8d86b1fc55712fdbdcc266af1a1ece7a51fff0d10/ruff-0.15.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d910ae974b7a06a33a057cb87d2a10792a3b2b3b35e33d2699fdf63ec8f6b17a", size = 11951057, upload-time = "2026-03-26T18:39:19.18Z" }, - { url = "https://files.pythonhosted.org/packages/83/f0/c9208c5fd5101bf87002fed774ff25a96eea313d305f1e5d5744698dc314/ruff-0.15.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2033f963c43949d51e6fdccd3946633c6b37c484f5f98c3035f49c27395a8ab8", size = 11464613, upload-time = "2026-03-26T18:40:06.301Z" }, - { url = "https://files.pythonhosted.org/packages/f8/22/d7f2fabdba4fae9f3b570e5605d5eb4500dcb7b770d3217dca4428484b17/ruff-0.15.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f29b989a55572fb885b77464cf24af05500806ab4edf9a0fd8977f9759d85b1", size = 11257557, upload-time = "2026-03-26T18:39:57.972Z" }, - { url = "https://files.pythonhosted.org/packages/71/8c/382a9620038cf6906446b23ce8632ab8c0811b8f9d3e764f58bedd0c9a6f/ruff-0.15.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ac51d486bf457cdc985a412fb1801b2dfd1bd8838372fc55de64b1510eff4bec", size = 11169440, upload-time = "2026-03-26T18:39:22.205Z" }, - { url = "https://files.pythonhosted.org/packages/4d/0d/0994c802a7eaaf99380085e4e40c845f8e32a562e20a38ec06174b52ef24/ruff-0.15.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c9861eb959edab053c10ad62c278835ee69ca527b6dcd72b47d5c1e5648964f6", size = 10605963, upload-time = "2026-03-26T18:39:46.682Z" }, - { url = "https://files.pythonhosted.org/packages/19/aa/d624b86f5b0aad7cef6bbf9cd47a6a02dfdc4f72c92a337d724e39c9d14b/ruff-0.15.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8d9a5b8ea13f26ae90838afc33f91b547e61b794865374f114f349e9036835fb", size = 10357484, upload-time = "2026-03-26T18:39:49.176Z" }, - { url = "https://files.pythonhosted.org/packages/35/c3/e0b7835d23001f7d999f3895c6b569927c4d39912286897f625736e1fd04/ruff-0.15.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c2a33a529fb3cbc23a7124b5c6ff121e4d6228029cba374777bd7649cc8598b8", size = 10830426, upload-time = "2026-03-26T18:40:03.702Z" }, - { url = "https://files.pythonhosted.org/packages/f0/51/ab20b322f637b369383adc341d761eaaa0f0203d6b9a7421cd6e783d81b9/ruff-0.15.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:75e5cd06b1cf3f47a3996cfc999226b19aa92e7cce682dcd62f80d7035f98f49", size = 11345125, upload-time = "2026-03-26T18:39:27.799Z" }, - { url = "https://files.pythonhosted.org/packages/37/e6/90b2b33419f59d0f2c4c8a48a4b74b460709a557e8e0064cf33ad894f983/ruff-0.15.8-py3-none-win32.whl", hash = "sha256:bc1f0a51254ba21767bfa9a8b5013ca8149dcf38092e6a9eb704d876de94dc34", size = 10571959, upload-time = "2026-03-26T18:39:36.117Z" }, - { url = "https://files.pythonhosted.org/packages/1f/a2/ef467cb77099062317154c63f234b8a7baf7cb690b99af760c5b68b9ee7f/ruff-0.15.8-py3-none-win_amd64.whl", hash = "sha256:04f79eff02a72db209d47d665ba7ebcad609d8918a134f86cb13dd132159fc89", size = 11743893, upload-time = "2026-03-26T18:39:25.01Z" }, - { url = "https://files.pythonhosted.org/packages/15/e2/77be4fff062fa78d9b2a4dea85d14785dac5f1d0c1fb58ed52331f0ebe28/ruff-0.15.8-py3-none-win_arm64.whl", hash = "sha256:cf891fa8e3bb430c0e7fac93851a5978fc99c8fa2c053b57b118972866f8e5f2", size = 11048175, upload-time = "2026-03-26T18:40:01.06Z" }, + { url = "https://files.pythonhosted.org/packages/4a/92/c445b0cd6da6e7ae51e954939cb69f97e008dbe750cfca89b8cedc081be7/ruff-0.15.8-py3-none-linux_armv6l.whl", hash = "sha256:cbe05adeba76d58162762d6b239c9056f1a15a55bd4b346cfd21e26cd6ad7bc7", upload-time = "2026-03-26T18:39:41.566Z" }, + { url = "https://files.pythonhosted.org/packages/eb/92/f1c662784d149ad1414cae450b082cf736430c12ca78367f20f5ed569d65/ruff-0.15.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d3e3d0b6ba8dca1b7ef9ab80a28e840a20070c4b62e56d675c24f366ef330570", upload-time = "2026-03-26T18:39:30.364Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f2/7a631a8af6d88bcef997eb1bf87cc3da158294c57044aafd3e17030613de/ruff-0.15.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6ee3ae5c65a42f273f126686353f2e08ff29927b7b7e203b711514370d500de3", upload-time = "2026-03-26T18:39:33.37Z" }, + { url = "https://files.pythonhosted.org/packages/67/18/1bf38e20914a05e72ef3b9569b1d5c70a7ef26cd188d69e9ca8ef588d5bf/ruff-0.15.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdce027ada77baa448077ccc6ebb2fa9c3c62fd110d8659d601cf2f475858d94", upload-time = "2026-03-26T18:39:44.142Z" }, + { url = "https://files.pythonhosted.org/packages/d2/e9/138c150ff9af60556121623d41aba18b7b57d95ac032e177b6a53789d279/ruff-0.15.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12e617fc01a95e5821648a6df341d80456bd627bfab8a829f7cfc26a14a4b4a3", upload-time = "2026-03-26T18:39:52.178Z" }, + { url = "https://files.pythonhosted.org/packages/02/f1/5bfb9298d9c323f842c5ddeb85f1f10ef51516ac7a34ba446c9347d898df/ruff-0.15.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:432701303b26416d22ba696c39f2c6f12499b89093b61360abc34bcc9bf07762", upload-time = "2026-03-26T18:39:55.195Z" }, + { url = "https://files.pythonhosted.org/packages/10/11/6da2e538704e753c04e8d86b1fc55712fdbdcc266af1a1ece7a51fff0d10/ruff-0.15.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d910ae974b7a06a33a057cb87d2a10792a3b2b3b35e33d2699fdf63ec8f6b17a", upload-time = "2026-03-26T18:39:19.18Z" }, + { url = "https://files.pythonhosted.org/packages/83/f0/c9208c5fd5101bf87002fed774ff25a96eea313d305f1e5d5744698dc314/ruff-0.15.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2033f963c43949d51e6fdccd3946633c6b37c484f5f98c3035f49c27395a8ab8", upload-time = "2026-03-26T18:40:06.301Z" }, + { url = "https://files.pythonhosted.org/packages/f8/22/d7f2fabdba4fae9f3b570e5605d5eb4500dcb7b770d3217dca4428484b17/ruff-0.15.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f29b989a55572fb885b77464cf24af05500806ab4edf9a0fd8977f9759d85b1", upload-time = "2026-03-26T18:39:57.972Z" }, + { url = "https://files.pythonhosted.org/packages/71/8c/382a9620038cf6906446b23ce8632ab8c0811b8f9d3e764f58bedd0c9a6f/ruff-0.15.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ac51d486bf457cdc985a412fb1801b2dfd1bd8838372fc55de64b1510eff4bec", upload-time = "2026-03-26T18:39:22.205Z" }, + { url = "https://files.pythonhosted.org/packages/4d/0d/0994c802a7eaaf99380085e4e40c845f8e32a562e20a38ec06174b52ef24/ruff-0.15.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c9861eb959edab053c10ad62c278835ee69ca527b6dcd72b47d5c1e5648964f6", upload-time = "2026-03-26T18:39:46.682Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/d624b86f5b0aad7cef6bbf9cd47a6a02dfdc4f72c92a337d724e39c9d14b/ruff-0.15.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8d9a5b8ea13f26ae90838afc33f91b547e61b794865374f114f349e9036835fb", upload-time = "2026-03-26T18:39:49.176Z" }, + { url = "https://files.pythonhosted.org/packages/35/c3/e0b7835d23001f7d999f3895c6b569927c4d39912286897f625736e1fd04/ruff-0.15.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c2a33a529fb3cbc23a7124b5c6ff121e4d6228029cba374777bd7649cc8598b8", upload-time = "2026-03-26T18:40:03.702Z" }, + { url = "https://files.pythonhosted.org/packages/f0/51/ab20b322f637b369383adc341d761eaaa0f0203d6b9a7421cd6e783d81b9/ruff-0.15.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:75e5cd06b1cf3f47a3996cfc999226b19aa92e7cce682dcd62f80d7035f98f49", upload-time = "2026-03-26T18:39:27.799Z" }, + { url = "https://files.pythonhosted.org/packages/37/e6/90b2b33419f59d0f2c4c8a48a4b74b460709a557e8e0064cf33ad894f983/ruff-0.15.8-py3-none-win32.whl", hash = "sha256:bc1f0a51254ba21767bfa9a8b5013ca8149dcf38092e6a9eb704d876de94dc34", upload-time = "2026-03-26T18:39:36.117Z" }, + { url = "https://files.pythonhosted.org/packages/1f/a2/ef467cb77099062317154c63f234b8a7baf7cb690b99af760c5b68b9ee7f/ruff-0.15.8-py3-none-win_amd64.whl", hash = "sha256:04f79eff02a72db209d47d665ba7ebcad609d8918a134f86cb13dd132159fc89", upload-time = "2026-03-26T18:39:25.01Z" }, + { url = "https://files.pythonhosted.org/packages/15/e2/77be4fff062fa78d9b2a4dea85d14785dac5f1d0c1fb58ed52331f0ebe28/ruff-0.15.8-py3-none-win_arm64.whl", hash = "sha256:cf891fa8e3bb430c0e7fac93851a5978fc99c8fa2c053b57b118972866f8e5f2", upload-time = "2026-03-26T18:40:01.06Z" }, ] [[package]] name = "sqlglot" version = "30.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/64/89299aefc6ebdf4fc899f5dc14c7fcb7eb9da9290a2b4d615ae7ab884b17/sqlglot-30.8.0.tar.gz", hash = "sha256:1c5f93fb742dd9aaa75eee6bb33a637794a858b9a86375fac23a2dc0f7bc127e", size = 5869750, upload-time = "2026-05-13T09:04:38.923Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/64/89299aefc6ebdf4fc899f5dc14c7fcb7eb9da9290a2b4d615ae7ab884b17/sqlglot-30.8.0.tar.gz", hash = "sha256:1c5f93fb742dd9aaa75eee6bb33a637794a858b9a86375fac23a2dc0f7bc127e", upload-time = "2026-05-13T09:04:38.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/4e/80705091aaf9c95e125d243f0aa871bc9f3670b4c9d963e6bad3b3dce8ff/sqlglot-30.8.0-py3-none-any.whl", hash = "sha256:af903378c331d5b72277a1b41118f07bc3e50cf4478e2d47eed12c96ee6a22a4", upload-time = "2026-05-13T09:04:36.336Z" }, +] + +[[package]] +name = "sse-starlette" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/2b/58abc2d1fd397e7dde08e947e05c884d8ef2f78d5e2588c17a12d42d6994/sse_starlette-3.4.4.tar.gz", hash = "sha256:07e0fa0460138baf25cdd5fb28683472c3995dc1642225191b3832d62526bcb0", upload-time = "2026-05-12T17:37:17.019Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/67/805710444ea8cc75fbf70b920ed431a560c4bf9c57f7d5a3117213189399/sse_starlette-3.4.4-py3-none-any.whl", hash = "sha256:3f4dd50d8aed2771a091f3a83000323fc3844541c16b4fe585ae2420cc6df973", upload-time = "2026-05-12T17:37:15.601Z" }, +] + +[[package]] +name = "starlette" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/44/ec35f1b6e83094b997da438a02c8c9b0ade2b1e84cfc48bd4656780760a6/starlette-1.2.1.tar.gz", hash = "sha256:9b9b5ebb992e67d6093741e63c2f59e4f6fff986f81163c087867bd7b924b3f6", upload-time = "2026-05-31T01:07:51.847Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/88/4e/80705091aaf9c95e125d243f0aa871bc9f3670b4c9d963e6bad3b3dce8ff/sqlglot-30.8.0-py3-none-any.whl", hash = "sha256:af903378c331d5b72277a1b41118f07bc3e50cf4478e2d47eed12c96ee6a22a4", size = 687831, upload-time = "2026-05-13T09:04:36.336Z" }, + { url = "https://files.pythonhosted.org/packages/1c/54/196d0c1db10af76baa4f64894448505d60d3cdf70ef92cbb35f46a4e4c71/starlette-1.2.1-py3-none-any.whl", hash = "sha256:4de0082d08c8f6764a85a54cf1120d6939507a19905c7768acad2a9f875d2b89", upload-time = "2026-05-31T01:07:50.09Z" }, ] [[package]] name = "tomlkit" version = "0.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", size = 161875, upload-time = "2026-05-10T07:38:22.245Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", upload-time = "2026-05-10T07:38:22.245Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", size = 41328, upload-time = "2026-05-10T07:38:23.517Z" }, + { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", upload-time = "2026-05-10T07:38:23.517Z" }, ] [[package]] name = "types-pyyaml" version = "6.0.12.20250915" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/69/3c51b36d04da19b92f9e815be12753125bd8bc247ba0470a982e6979e71c/types_pyyaml-6.0.12.20250915.tar.gz", hash = "sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3", size = 17522, upload-time = "2025-09-15T03:01:00.728Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/69/3c51b36d04da19b92f9e815be12753125bd8bc247ba0470a982e6979e71c/types_pyyaml-6.0.12.20250915.tar.gz", hash = "sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3", upload-time = "2025-09-15T03:01:00.728Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/e0/1eed384f02555dde685fff1a1ac805c1c7dcb6dd019c916fe659b1c1f9ec/types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6", size = 20338, upload-time = "2025-09-15T03:00:59.218Z" }, + { url = "https://files.pythonhosted.org/packages/bd/e0/1eed384f02555dde685fff1a1ac805c1c7dcb6dd019c916fe659b1c1f9ec/types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6", upload-time = "2025-09-15T03:00:59.218Z" }, ] [[package]] name = "typing-extensions" version = "4.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", upload-time = "2025-08-25T13:49:26.313Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", upload-time = "2025-10-01T02:14:40.154Z" }, ] [[package]] name = "urllib3" version = "2.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.49.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/1f/fa18009dea8469069cca78a4e877a008ab78f08b064bfc9ab891579077ff/uvicorn-0.49.0.tar.gz", hash = "sha256:ebf4271aa580d9de97f93192d4595176df6e91f9aae919ca73e4fc07df1e66a3", upload-time = "2026-06-03T22:01:30.448Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, + { url = "https://files.pythonhosted.org/packages/88/fa/e1388bbcf24ef3274f45c0c1c7b501fd14971037c1b6ee23610553307497/uvicorn-0.49.0-py3-none-any.whl", hash = "sha256:ba3d14c3ee7e41c6c654c46c9eb489d33213cdd30aa1696eab1374337c13f68f", upload-time = "2026-06-03T22:01:29.037Z" }, ] [[package]] name = "xmltodict" version = "1.0.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz", hash = "sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61", size = 26124, upload-time = "2026-02-22T02:21:22.074Z" } +sdist = { url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz", hash = "sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61", upload-time = "2026-02-22T02:21:22.074Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl", hash = "sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a", size = 13580, upload-time = "2026-02-22T02:21:21.039Z" }, + { url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl", hash = "sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a", upload-time = "2026-02-22T02:21:21.039Z" }, ] [[package]] @@ -739,7 +1221,7 @@ dependencies = [ { name = "tomlkit" }, { name = "xmltodict" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/6a/eb9721ed0929d0f55d167c2222d288b529723afbef0a07ed7aa6cca72380/yq-3.4.3.tar.gz", hash = "sha256:ba586a1a6f30cf705b2f92206712df2281cd320280210e7b7b80adcb8f256e3b", size = 33214, upload-time = "2024-04-27T15:39:43.29Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/6a/eb9721ed0929d0f55d167c2222d288b529723afbef0a07ed7aa6cca72380/yq-3.4.3.tar.gz", hash = "sha256:ba586a1a6f30cf705b2f92206712df2281cd320280210e7b7b80adcb8f256e3b", upload-time = "2024-04-27T15:39:43.29Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/ba/d1b21f3e57469030bd6536b91bb28fedd2511d4e68b5a575f2bdb3a3dbb6/yq-3.4.3-py3-none-any.whl", hash = "sha256:547e34bc3caacce83665fd3429bf7c85f8e8b6b9aaee3f953db1ad716ff3434d", size = 18812, upload-time = "2024-04-27T15:39:41.652Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ba/d1b21f3e57469030bd6536b91bb28fedd2511d4e68b5a575f2bdb3a3dbb6/yq-3.4.3-py3-none-any.whl", hash = "sha256:547e34bc3caacce83665fd3429bf7c85f8e8b6b9aaee3f953db1ad716ff3434d", upload-time = "2024-04-27T15:39:41.652Z" }, ] From fc56d6c07b2400af26ef7090f91654e91a41421e Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Thu, 25 Jun 2026 08:50:04 -0400 Subject: [PATCH 15/77] Improve preparer coverage for managed ingestion pipelines (#2) * Lakeflow connect ingestion pipeline handling * Add required connection parameters --- src/flowx/bundler/dab_writer.py | 81 +++++++++++++++++++++++-- src/flowx/bundler/setup_generator.py | 7 ++- src/flowx/preparer/workflow_preparer.py | 2 +- tests/unit/test_adapter.py | 34 ++++++++--- tests/unit/test_bundler.py | 16 +++++ uv.lock | 2 +- 6 files changed, 128 insertions(+), 14 deletions(-) diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index bb99399..362d48a 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -119,6 +119,9 @@ def write_bundle( _any_task_uses_classic_cluster(inner.tasks) for inner in workflow.inner_workflows ) + pipeline_resources = _collect_pipeline_resources(workflow) + pipeline_variable_declarations = _build_pipeline_variable_declarations(pipeline_resources, catalog, schema) + # 1. Write databricks.yml. When any task runs on classic compute, spark_version / node_type_id # defaults come from the ADF linked-service configs; when every task is serverless, they're omitted. databricks_yml_path = output_dir / "databricks.yml" @@ -130,6 +133,7 @@ def write_bundle( spark_version=inferred_spark_version, node_type_id=inferred_node_type_id, include_cluster_variables=bundle_uses_classic_cluster, + extra_variables=pipeline_variable_declarations, ) databricks_yml_path.write_text( yaml.dump( @@ -181,10 +185,8 @@ def write_bundle( # 2b. Write Lakeflow pipeline resources (Lakeflow Connect ingestion defs from the Copy preparer's LFC # branch). Each lives in its own YAML so the bundle parser merges them via the ``include`` glob. - pipelines_dir = resources_dir / "pipelines" - for resource in _collect_pipeline_resources(workflow): - pipelines_dir.mkdir(parents=True, exist_ok=True) - resource_yml_path = pipelines_dir / f"{resource['resource_key']}.yml" + for resource in pipeline_resources: + resource_yml_path = resources_dir / f"{resource['resource_key']}.yml" resource_yml_path.write_text( yaml.dump( _wrap_pipeline_resource(resource), @@ -568,6 +570,7 @@ def _build_databricks_yml( spark_version: str = _DEFAULT_SPARK_VERSION, node_type_id: str = _DEFAULT_NODE_TYPE_ID, include_cluster_variables: bool = True, + extra_variables: dict[str, Any] | None = None, ) -> dict[str, Any]: """Builds the root ``databricks.yml`` configuration as a dict. @@ -583,6 +586,9 @@ def _build_databricks_yml( False when no task in the bundle uses classic compute (every generated notebook runs on serverless), so the bundle stays free of unused tunables. + extra_variables: Additional variable declarations (name -> DAB + declaration dict) to merge into the ``variables`` block, e.g. + the source-side variables a Lakeflow Connect pipeline references. Returns: Dict ready for YAML serialization. @@ -609,6 +615,8 @@ def _build_databricks_yml( "description": "Databricks Runtime for the default job_cluster.", "default": spark_version, } + for name, declaration in (extra_variables or {}).items(): + variables.setdefault(name, declaration) # Declare a variable for each cross-bundle ExecutePipeline reference so `${var.X_job_id}` resolves and # `bundle validate` passes. Users fill in the numeric job ID per SETUP.md. for variable_name, target_pipeline in sorted(_cross_bundle_variables.items()): @@ -770,6 +778,71 @@ def _wrap_pipeline_resource(resource: dict[str, Any]) -> dict[str, Any]: return {"resources": {"pipelines": {resource["resource_key"]: resource["definition"]}}} +_VAR_REFERENCE_RE = re.compile(r"\$\{var\.([A-Za-z_][A-Za-z0-9_]*)\}") + +_BUILTIN_BUNDLE_VARIABLES = frozenset({"catalog", "schema", "node_type_id", "spark_version"}) + + +def _collect_variable_references(value: Any) -> set[str]: + """Returns every ``${var.NAME}`` variable name referenced anywhere within *value*.""" + refs: set[str] = set() + if isinstance(value, str): + refs.update(_VAR_REFERENCE_RE.findall(value)) + elif isinstance(value, dict): + for item in value.values(): + refs |= _collect_variable_references(item) + elif isinstance(value, list): + for item in value: + refs |= _collect_variable_references(item) + return refs + + +def _build_pipeline_variable_declarations( + pipeline_resources: list[dict[str, Any]], + catalog: str, + schema: str, +) -> dict[str, Any]: + """Returns ``variables:`` declarations for every ``${var.…}`` a pipeline resource references. + + Lakeflow Connect ingestion definitions fall back to ``${var.source_catalog}`` / + ``${var.source_schema}`` (and could reference further variables) when the translator + couldn't resolve a literal. Each such variable must appear in the root ``variables:`` + block or ``databricks bundle validate`` fails on an undefined reference. + + Args: + pipeline_resources: The pipeline-resource dicts collected for the bundle. + catalog: The migration target catalog (used as the source_catalog default). + schema: The migration target schema (used as the source_schema default). + + Returns: + Mapping of variable name to its DAB declaration dict. ``source_catalog`` and + ``source_schema`` get a sensible default so validation passes out of the box; + any other referenced variable is declared without a default (user fills it in). + """ + referenced: set[str] = set() + for resource in pipeline_resources: + referenced |= _collect_variable_references(resource.get("definition")) + referenced -= _BUILTIN_BUNDLE_VARIABLES + + source_defaults = {"source_catalog": catalog, "source_schema": schema} + declarations: dict[str, Any] = {} + for name in sorted(referenced): + if name in source_defaults: + kind = name.removeprefix("source_") + declarations[name] = { + "description": ( + f"Source-side {kind} the Lakeflow Connect ingestion reads from. " + f"Defaults to the migration {kind}; override with the real source {kind}." + ), + "default": source_defaults[name], + } + else: + declarations[name] = { + "description": f"Value for ${{var.{name}}} referenced by a generated pipeline resource.", + } + return declarations + + def _collect_required_cluster_keys(tasks: list[dict[str, Any]]) -> set[str]: """Walks every task and returns the set of job_cluster keys actually bound. diff --git a/src/flowx/bundler/setup_generator.py b/src/flowx/bundler/setup_generator.py index 6aa8e7e..838f5cc 100644 --- a/src/flowx/bundler/setup_generator.py +++ b/src/flowx/bundler/setup_generator.py @@ -300,11 +300,16 @@ def _generate_connection_setup_notebook( host = config.get("host", "PLACEHOLDER_HOST") port = config.get("port", "3306") + options = [f"host '{host}'", f"port '{port}'"] + if conn_type == "SQLSERVER": + options.append(f"user '{config.get('user', 'PLACEHOLDER_USER')}'") + options.append(f"password '{config.get('password', 'PLACEHOLDER_PASSWORD')}'") + lines: list[str] = [f"# Create connection: {conn_name}"] lines.append('spark.sql("""') lines.append(f" CREATE CONNECTION IF NOT EXISTS {conn_name}") lines.append(f" TYPE {conn_type}") - lines.append(f" OPTIONS (host '{host}', port '{port}')") + lines.append(f" OPTIONS ({', '.join(options)})") lines.append('""")') lines.append(f'print("Created connection: {conn_name}")') body_parts.append("\n".join(lines)) diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py index 9edc4b0..8b460b3 100644 --- a/src/flowx/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -44,7 +44,7 @@ class PreparedActivity: inner_workflows: list[PreparedWorkflow] = field(default_factory=list) # Switch renames its first case key; prepare_workflow reads this map to rewrite depends_on edges. task_key_remap: dict[str, str] = field(default_factory=dict) - # Lakeflow pipeline resources emitted under resources/pipelines/.yml ({resource_key, definition}). + # Lakeflow pipeline resources emitted under resources/.yml ({resource_key, definition}). pipeline_resources: list[dict[str, Any]] = field(default_factory=list) parameter_approximations: list[ParameterApproximation] = field(default_factory=list) diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 3807af1..f283c1a 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -957,7 +957,7 @@ def test_lakeflow_connect_emits_pipeline_resource_and_no_notebook(self, tmp_path workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) assert not (tmp_path / "src" / "notebooks" / "copy_a.py").exists() - pipeline_yml = tmp_path / "resources" / "pipelines" / "copy_a_lfc.yml" + pipeline_yml = tmp_path / "resources" / "copy_a_lfc.yml" assert pipeline_yml.exists() resource = yaml.safe_load(pipeline_yml.read_text()) lfc = resource["resources"]["pipelines"]["copy_a_lfc"] @@ -966,6 +966,26 @@ def test_lakeflow_connect_emits_pipeline_resource_and_no_notebook(self, tmp_path objects = lfc["ingestion_definition"]["objects"] assert objects[0]["table"]["destination_table"] == "raw.events" + def test_lakeflow_connect_declares_referenced_source_variables(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) + stamped = apply_configuration(pipeline, TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect")) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path, catalog="migration_cat", schema="migration_schema") + + resource = yaml.safe_load((tmp_path / "resources" / "copy_a_lfc.yml").read_text()) + table = resource["resources"]["pipelines"]["copy_a_lfc"]["ingestion_definition"]["objects"][0]["table"] + assert table["source_catalog"] == "${var.source_catalog}" + assert table["source_schema"] == "${var.source_schema}" + + variables = yaml.safe_load((tmp_path / "databricks.yml").read_text())["variables"] + assert variables["source_catalog"]["default"] == "migration_cat" + assert variables["source_schema"]["default"] == "migration_schema" + def test_lakeflow_connect_job_task_references_pipeline(self, tmp_path: Path): import yaml @@ -1076,7 +1096,7 @@ def test_query_copy_routes_to_query_based_connector_regardless_of_configuration( stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) - resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_q_lfc.yml").read_text()) + resource = yaml.safe_load((tmp_path / "resources" / "copy_q_lfc.yml").read_text()) objects = resource["resources"]["pipelines"]["copy_q_lfc"]["ingestion_definition"]["objects"] assert "table_configuration" in objects[0] table_config = objects[0]["table_configuration"] @@ -1097,7 +1117,7 @@ def test_table_copy_uses_cdc_connector_by_default(self, tmp_path: Path): stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) - resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_a_lfc.yml").read_text()) + resource = yaml.safe_load((tmp_path / "resources" / "copy_a_lfc.yml").read_text()) objects = resource["resources"]["pipelines"]["copy_a_lfc"]["ingestion_definition"]["objects"] assert "table" in objects[0] assert "table_configuration" not in objects[0] @@ -1124,7 +1144,7 @@ def test_table_copy_with_query_based_configuration_routes_to_cdc(self, tmp_path: stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) - resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_a_lfc.yml").read_text()) + resource = yaml.safe_load((tmp_path / "resources" / "copy_a_lfc.yml").read_text()) objects = resource["resources"]["pipelines"]["copy_a_lfc"]["ingestion_definition"]["objects"] assert "table" in objects[0] assert "table_configuration" not in objects[0] @@ -1155,7 +1175,7 @@ def test_consolidated_metadata_driven_motif_emits_single_pipeline(self, tmp_path stamped = dataclasses.replace(stamped, tasks=[consolidated_motif]) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) - resource_path = tmp_path / "resources" / "pipelines" / "motif_metadata_driven_bulk_copy_consolidated.yml" + resource_path = tmp_path / "resources" / "motif_metadata_driven_bulk_copy_consolidated.yml" assert resource_path.exists() resource = yaml.safe_load(resource_path.read_text()) pipeline_def = resource["resources"]["pipelines"]["motif_metadata_driven_bulk_copy_consolidated"] @@ -1195,7 +1215,7 @@ def test_table_based_copy_with_query_based_configuration_falls_back_to_cdc(self, ) stamped = apply_configuration(Pipeline(name="job", tasks=[copy]), prefs) write_bundle(prepare_workflow(stamped), tmp_path) - resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_customers_lfc.yml").read_text()) + resource = yaml.safe_load((tmp_path / "resources" / "copy_customers_lfc.yml").read_text()) obj = resource["resources"]["pipelines"]["copy_customers_lfc"]["ingestion_definition"]["objects"][0] assert "table" in obj assert obj["table"]["destination_table"] == "customers" @@ -1257,7 +1277,7 @@ def test_lakeflow_connect_dedupes_connection_across_copies(self, tmp_path: Path) assert body.count("CREATE CONNECTION IF NOT EXISTS") == 1 assert body.count("flowx_LS_AzureSqlDb_connection") >= 1 for pipeline_file in ("copy_a_lfc.yml", "copy_b_lfc.yml"): - resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / pipeline_file).read_text()) + resource = yaml.safe_load((tmp_path / "resources" / pipeline_file).read_text()) key = pipeline_file.replace(".yml", "") assert resource["resources"]["pipelines"][key]["ingestion_definition"]["connection_name"] == ( "flowx_LS_AzureSqlDb_connection" diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index 4cfa416..90ecb02 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -951,6 +951,22 @@ def test_connection_setup_notebook(self): nb = notebooks[0] assert nb.relative_path == "setup/create_connections.py" assert "sql_conn" in nb.content + assert "user 'PLACEHOLDER_USER'" in nb.content + assert "password 'PLACEHOLDER_PASSWORD'" in nb.content + + def test_connection_setup_notebook_non_sqlserver_omits_credentials(self): + from flowx.bundler.setup_generator import generate_setup_tasks + + setup_tasks = [ + SetupTask( + type="connection", + config={"connection_name": "my_conn", "connection_type": "MYSQL", "host": "mysql.example.com"}, + ), + ] + notebooks = generate_setup_tasks(secrets=[], setup_tasks=setup_tasks, catalog="main", schema="default") + content = notebooks[0].content + assert "user '" not in content + assert "password '" not in content def test_no_setup_when_empty(self): from flowx.bundler.setup_generator import generate_setup_tasks diff --git a/uv.lock b/uv.lock index ed81fa7..1d45f14 100644 --- a/uv.lock +++ b/uv.lock @@ -341,7 +341,7 @@ wheels = [ [[package]] name = "databricks-flowx" -version = "0.2.0" +version = "0.1.0" source = { editable = "." } dependencies = [ { name = "databricks-sdk" }, From 0ed89ef5a8524b5b9c1e09425b75c7618f324f11 Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:35:19 -0400 Subject: [PATCH 16/77] Add marketplace file (#3) --- .claude-plugin/marketplace.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .claude-plugin/marketplace.json diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..9377896 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,17 @@ +{ + "name": "flowx", + "owner": { + "name": "Greg Hansen", + "email": "gregory.hansen@databricks.com" + }, + "plugins": [ + { + "name": "flowx", + "source": "./", + "description": "Translate data pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. Deterministic translation for known activity types with agentic fallback.", + "version": "0.1.0", + "license": "MIT", + "keywords": ["adf", "databricks", "migration", "dabs", "lakeflow", "orchestration", "data-pipelines"] + } + ] +} From 83096a5d568da205375ef064ed333673d9fd6cbd Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Fri, 26 Jun 2026 10:47:34 -0400 Subject: [PATCH 17/77] Merge external (#3) * Revert "Update GitHub actions (#10)" This reverts commit 3c9cb71f274c6db55a3bfb4cd06a3b7768f2fc12. * Improve expression resolution, control flow parsing, and setup (#11) * Refactor expression parsing (#1) * Improve control flow conversion (#2) * Update repo structure * Format modules * Initial commit * Initial release * Improve preparer coverage for managed ingestion pipelines (#2) * Lakeflow connect ingestion pipeline handling * Add required connection parameters * Add marketplace file (#3) --------- Co-authored-by: Greg Hansen <163584195+ghanse@users.noreply.github.com> Co-authored-by: service-jira-pub-repo-auto --- .build-constraints.txt | 12 +- .claude-plugin/marketplace.json | 17 + .claude-plugin/plugin.json | 12 +- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/ISSUE_TEMPLATE/feature_request.yml | 2 +- .github/workflows/push.yml | 14 +- .gitignore | 6 +- AGENTS.md | 68 +- CHANGELOG.md | 8 +- .../models/__init__.py => CODEOWNERS.txt | 0 LICENSE.md | 24 + Makefile | 2 +- NOTICE.md | 3 + README.md | 39 +- SECURITY.md | 6 + app/README.md | 192 +++ app/app.py | 21 + app/app.yaml | 5 + app/deploy.sh | 96 ++ app/requirements.txt | 10 + docs/README.md | 2 +- docs/app/(home)/page.tsx | 2 +- docs/app/layout.tsx | 4 +- docs/content/docs/architecture.mdx | 90 ++ docs/content/docs/guide.mdx | 41 +- docs/content/docs/index.mdx | 17 +- docs/content/docs/installation.mdx | 135 +- docs/content/docs/meta.json | 3 +- docs/content/docs/options.mdx | 49 +- pyproject.toml | 19 +- requirements.txt | 6 +- scripts/bootstrap.sh | 106 +- skills/flowx-convert/SKILL.md | 434 +++++ .../references/activity-mapping.md | 2 +- .../references/expression-functions.md | 0 skills/flowx-discover/SKILL.md | 273 ++++ skills/flowx-migrate/SKILL.md | 422 +++++ .../references/workflow.md | 24 +- skills/flowx-package/SKILL.md | 344 ++++ skills/flowx-setup/SKILL.md | 172 ++ skills/ingest/SKILL.md | 212 --- skills/migrate/SKILL.md | 324 ---- skills/prepare/SKILL.md | 254 --- skills/translate/SKILL.md | 329 ---- src/AGENTS.md | 16 +- src/flowx/__init__.py | 3 + src/flowx/adapter/__init__.py | 75 + src/flowx/adapter/__main__.py | 792 +++++++++ src/{orchestra => flowx}/adapter/constants.py | 48 +- src/{orchestra => flowx}/adapter/models.py | 115 +- .../adapter/operations.py | 872 +++++++--- .../adapter/predicates.py | 2 +- src/flowx/adapter/session.py | 486 ++++++ src/{orchestra => flowx}/bundler/__init__.py | 0 src/{orchestra => flowx}/bundler/constants.py | 0 .../bundler/dab_writer.py | 424 +++-- .../bundler/inner_job_params.py | 32 +- .../bundler/notebook_writer.py | 0 .../bundler/prereqs_writer.py | 58 +- .../bundler/setup_generator.py | 44 +- src/flowx/mcp/__init__.py | 18 + src/flowx/mcp/__main__.py | 6 + src/flowx/mcp/runner.py | 365 +++++ src/flowx/mcp/server.py | 554 +++++++ .../parser => flowx/models}/__init__.py | 0 src/{orchestra => flowx}/models/adf_ast.py | 6 + src/{orchestra => flowx}/models/dab.py | 0 src/{orchestra => flowx}/models/ir.py | 57 +- src/{orchestra => flowx}/models/motifs.py | 20 +- .../models/source_types.py | 15 +- src/{orchestra => flowx}/motifs/__init__.py | 0 src/{orchestra => flowx}/motifs/collapser.py | 12 +- src/{orchestra => flowx}/motifs/detector.py | 41 +- .../translator => flowx/parser}/__init__.py | 0 src/{orchestra => flowx}/parser/adf_loader.py | 310 +++- .../parser/expression_parser.py | 121 +- .../parser/ir_rewriter.py | 12 +- src/{orchestra => flowx}/preparer/__init__.py | 0 .../preparer/activity_preparers/__init__.py | 0 .../activity_preparers/append_variable.py | 0 .../preparer/activity_preparers/copy.py | 12 +- .../activity_preparers/databricks_job.py | 0 .../preparer/activity_preparers/delete.py | 0 .../activity_preparers/execute_pipeline.py | 0 .../preparer/activity_preparers/filter.py | 0 .../preparer/activity_preparers/for_each.py | 24 +- .../preparer/activity_preparers/helpers.py | 0 .../activity_preparers/if_condition.py | 0 .../preparer/activity_preparers/lookup.py | 0 .../preparer/activity_preparers/motif.py | 96 +- .../preparer/activity_preparers/naming.py | 0 .../preparer/activity_preparers/notebook.py | 2 +- .../activity_preparers/set_variable.py | 0 .../preparer/activity_preparers/spark_jar.py | 0 .../activity_preparers/spark_python.py | 0 .../preparer/activity_preparers/switch.py | 9 +- .../preparer/activity_preparers/wait.py | 0 .../activity_preparers/web_activity.py | 14 +- .../preparer/code_generator.py | 177 +- src/flowx/preparer/notifications.py | 158 ++ .../preparer/workflow_preparer.py | 93 +- .../preparer/workspace_downloader.py | 187 ++- src/flowx/reporting/__init__.py | 1 + src/flowx/reporting/coverage.py | 117 ++ src/flowx/reporting/dashboard.py | 97 ++ src/flowx/reporting/dashboard_template.json | 538 +++++++ src/flowx/reporting/results.py | 171 ++ .../translator}/__init__.py | 0 .../activity_translators/__init__.py | 0 .../activity_translators/append_variable.py | 0 .../translator/activity_translators/copy.py | 25 +- .../activity_translators/databricks_job.py | 0 .../translator/activity_translators/delete.py | 0 .../activity_translators/execute_pipeline.py | 0 .../translator/activity_translators/filter.py | 12 +- .../activity_translators/for_each.py | 8 +- .../activity_translators/if_condition.py | 69 +- .../translator/activity_translators/lookup.py | 38 +- .../activity_translators/notebook.py | 44 +- .../activity_translators/resolve.py | 12 +- .../activity_translators/set_variable.py | 32 +- .../activity_translators/spark_jar.py | 0 .../activity_translators/spark_python.py | 2 +- .../translator/activity_translators/switch.py | 4 +- .../translator/activity_translators/wait.py | 0 .../activity_translators/web_activity.py | 162 ++ src/{orchestra => flowx}/translator/engine.py | 440 +++-- .../translator/query_analysis.py | 0 src/{orchestra => flowx}/utils.py | 4 +- src/flowx/validate/__init__.py | 27 + src/flowx/validate/bundle_invariants.py | 181 +++ src/flowx/validate/dag_equivalence.py | 408 +++++ src/orchestra/__init__.py | 3 - src/orchestra/adapter/__init__.py | 96 -- src/orchestra/adapter/__main__.py | 597 ------- src/orchestra/adapter/session.py | 450 ------ .../activity_translators/web_activity.py | 104 -- tests/integration/test_end_to_end.py | 2 +- tests/integration/test_path_equivalence.py | 59 + tests/unit/test_adapter.py | 659 ++++---- tests/unit/test_adf_loader.py | 50 + tests/unit/test_bundle_invariants.py | 61 + tests/unit/test_bundler.py | 16 + tests/unit/test_code_generator.py | 6 +- tests/unit/test_dag_equivalence.py | 220 +++ tests/unit/test_expression_parser.py | 12 +- tests/unit/test_for_each_inner_job_params.py | 8 +- tests/unit/test_mcp_migrate.py | 117 ++ tests/unit/test_merge_agentic.py | 100 ++ tests/unit/test_motifs.py | 2 +- tests/unit/test_notify.py | 397 +++++ tests/unit/test_param_dedup.py | 36 + tests/unit/test_preparers.py | 79 +- tests/unit/test_prereqs_writer.py | 7 +- tests/unit/test_profile_report.py | 107 ++ tests/unit/test_reporting_coverage.py | 90 ++ tests/unit/test_reporting_dashboard.py | 101 ++ tests/unit/test_reporting_results.py | 180 +++ tests/unit/test_translators.py | 48 +- tests/unit/test_until_agentic_handler.py | 71 + .../unit/test_web_body_and_param_defaults.py | 71 + tests/unit/test_workspace_downloader.py | 28 + uv.lock | 1432 +++++++++++------ 163 files changed, 12160 insertions(+), 4520 deletions(-) create mode 100644 .claude-plugin/marketplace.json rename src/orchestra/models/__init__.py => CODEOWNERS.txt (100%) create mode 100644 LICENSE.md create mode 100644 NOTICE.md create mode 100644 SECURITY.md create mode 100644 app/README.md create mode 100644 app/app.py create mode 100644 app/app.yaml create mode 100755 app/deploy.sh create mode 100644 app/requirements.txt create mode 100644 docs/content/docs/architecture.mdx mode change 100755 => 100644 scripts/bootstrap.sh create mode 100644 skills/flowx-convert/SKILL.md rename skills/{translate => flowx-convert}/references/activity-mapping.md (99%) rename skills/{translate => flowx-convert}/references/expression-functions.md (100%) create mode 100644 skills/flowx-discover/SKILL.md create mode 100644 skills/flowx-migrate/SKILL.md rename skills/{migrate => flowx-migrate}/references/workflow.md (93%) create mode 100644 skills/flowx-package/SKILL.md create mode 100644 skills/flowx-setup/SKILL.md delete mode 100644 skills/ingest/SKILL.md delete mode 100644 skills/migrate/SKILL.md delete mode 100644 skills/prepare/SKILL.md delete mode 100644 skills/translate/SKILL.md create mode 100644 src/flowx/__init__.py create mode 100644 src/flowx/adapter/__init__.py create mode 100644 src/flowx/adapter/__main__.py rename src/{orchestra => flowx}/adapter/constants.py (51%) rename src/{orchestra => flowx}/adapter/models.py (72%) rename src/{orchestra => flowx}/adapter/operations.py (51%) rename src/{orchestra => flowx}/adapter/predicates.py (99%) create mode 100644 src/flowx/adapter/session.py rename src/{orchestra => flowx}/bundler/__init__.py (100%) rename src/{orchestra => flowx}/bundler/constants.py (100%) rename src/{orchestra => flowx}/bundler/dab_writer.py (79%) rename src/{orchestra => flowx}/bundler/inner_job_params.py (94%) rename src/{orchestra => flowx}/bundler/notebook_writer.py (100%) rename src/{orchestra => flowx}/bundler/prereqs_writer.py (94%) rename src/{orchestra => flowx}/bundler/setup_generator.py (88%) create mode 100644 src/flowx/mcp/__init__.py create mode 100644 src/flowx/mcp/__main__.py create mode 100644 src/flowx/mcp/runner.py create mode 100644 src/flowx/mcp/server.py rename src/{orchestra/parser => flowx/models}/__init__.py (100%) rename src/{orchestra => flowx}/models/adf_ast.py (96%) rename src/{orchestra => flowx}/models/dab.py (100%) rename src/{orchestra => flowx}/models/ir.py (93%) rename src/{orchestra => flowx}/models/motifs.py (93%) rename src/{orchestra => flowx}/models/source_types.py (56%) rename src/{orchestra => flowx}/motifs/__init__.py (100%) rename src/{orchestra => flowx}/motifs/collapser.py (91%) rename src/{orchestra => flowx}/motifs/detector.py (95%) rename src/{orchestra/translator => flowx/parser}/__init__.py (100%) rename src/{orchestra => flowx}/parser/adf_loader.py (66%) rename src/{orchestra => flowx}/parser/expression_parser.py (92%) rename src/{orchestra => flowx}/parser/ir_rewriter.py (96%) rename src/{orchestra => flowx}/preparer/__init__.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/__init__.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/append_variable.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/copy.py (97%) rename src/{orchestra => flowx}/preparer/activity_preparers/databricks_job.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/delete.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/execute_pipeline.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/filter.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/for_each.py (93%) rename src/{orchestra => flowx}/preparer/activity_preparers/helpers.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/if_condition.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/lookup.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/motif.py (56%) rename src/{orchestra => flowx}/preparer/activity_preparers/naming.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/notebook.py (99%) rename src/{orchestra => flowx}/preparer/activity_preparers/set_variable.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/spark_jar.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/spark_python.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/switch.py (96%) rename src/{orchestra => flowx}/preparer/activity_preparers/wait.py (100%) rename src/{orchestra => flowx}/preparer/activity_preparers/web_activity.py (91%) rename src/{orchestra => flowx}/preparer/code_generator.py (90%) create mode 100644 src/flowx/preparer/notifications.py rename src/{orchestra => flowx}/preparer/workflow_preparer.py (86%) rename src/{orchestra => flowx}/preparer/workspace_downloader.py (55%) create mode 100644 src/flowx/reporting/__init__.py create mode 100644 src/flowx/reporting/coverage.py create mode 100644 src/flowx/reporting/dashboard.py create mode 100644 src/flowx/reporting/dashboard_template.json create mode 100644 src/flowx/reporting/results.py rename src/{orchestra/translator/activity_translators => flowx/translator}/__init__.py (100%) create mode 100644 src/flowx/translator/activity_translators/__init__.py rename src/{orchestra => flowx}/translator/activity_translators/append_variable.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/copy.py (96%) rename src/{orchestra => flowx}/translator/activity_translators/databricks_job.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/delete.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/execute_pipeline.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/filter.py (82%) rename src/{orchestra => flowx}/translator/activity_translators/for_each.py (90%) rename src/{orchestra => flowx}/translator/activity_translators/if_condition.py (82%) rename src/{orchestra => flowx}/translator/activity_translators/lookup.py (84%) rename src/{orchestra => flowx}/translator/activity_translators/notebook.py (83%) rename src/{orchestra => flowx}/translator/activity_translators/resolve.py (92%) rename src/{orchestra => flowx}/translator/activity_translators/set_variable.py (79%) rename src/{orchestra => flowx}/translator/activity_translators/spark_jar.py (100%) rename src/{orchestra => flowx}/translator/activity_translators/spark_python.py (95%) rename src/{orchestra => flowx}/translator/activity_translators/switch.py (97%) rename src/{orchestra => flowx}/translator/activity_translators/wait.py (100%) create mode 100644 src/flowx/translator/activity_translators/web_activity.py rename src/{orchestra => flowx}/translator/engine.py (77%) rename src/{orchestra => flowx}/translator/query_analysis.py (100%) rename src/{orchestra => flowx}/utils.py (94%) create mode 100644 src/flowx/validate/__init__.py create mode 100644 src/flowx/validate/bundle_invariants.py create mode 100644 src/flowx/validate/dag_equivalence.py delete mode 100644 src/orchestra/__init__.py delete mode 100644 src/orchestra/adapter/__init__.py delete mode 100644 src/orchestra/adapter/__main__.py delete mode 100644 src/orchestra/adapter/session.py delete mode 100644 src/orchestra/translator/activity_translators/web_activity.py create mode 100644 tests/integration/test_path_equivalence.py create mode 100644 tests/unit/test_bundle_invariants.py create mode 100644 tests/unit/test_dag_equivalence.py create mode 100644 tests/unit/test_mcp_migrate.py create mode 100644 tests/unit/test_merge_agentic.py create mode 100644 tests/unit/test_notify.py create mode 100644 tests/unit/test_param_dedup.py create mode 100644 tests/unit/test_profile_report.py create mode 100644 tests/unit/test_reporting_coverage.py create mode 100644 tests/unit/test_reporting_dashboard.py create mode 100644 tests/unit/test_reporting_results.py create mode 100644 tests/unit/test_until_agentic_handler.py create mode 100644 tests/unit/test_web_body_and_param_defaults.py diff --git a/.build-constraints.txt b/.build-constraints.txt index c6be8d7..ee7a59e 100644 --- a/.build-constraints.txt +++ b/.build-constraints.txt @@ -1,6 +1,6 @@ -hatchling==1.29.0 \ - --hash=sha256:50af9343281f34785fab12da82e445ed987a6efb34fd8c2fc0f6e6630dbcc1b0 \ - --hash=sha256:793c31816d952cee405b83488ce001c719f325d9cda69f1fc4cd750527640ea6 +hatchling==1.30.1 \ + --hash=sha256:161eacafb3c6f91526e92116d21426369f2c36e98c36a864f11a96345ad4ee31 \ + --hash=sha256:eee4fd45357f72ebb3d7a42e5d72cfb5e29ed426d79e8836288926c4258d5f2e packaging==26.2 \ --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 @@ -13,7 +13,7 @@ pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 # via hatchling -trove-classifiers==2026.5.22.10 \ - --hash=sha256:01fe864225726e03efb843827ecabfe319fc4dee8dd66d65b8996cb09be46e2c \ - --hash=sha256:5477e9974e91904fb2cfa4a7581ab6e2f30c2c38d847fd00ed866080748101d5 +trove-classifiers==2026.6.1.19 \ + --hash=sha256:ab4c4ec93cc4a4e7815fa759906e05e6bb3f2fbd92ea0f897288c6a43efd15b3 \ + --hash=sha256:c5132b4b61a829d11cfbd2d72e97f20a45ed6edb95e45c5efdeb5e00836b2745 # via hatchling diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..9377896 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,17 @@ +{ + "name": "flowx", + "owner": { + "name": "Greg Hansen", + "email": "gregory.hansen@databricks.com" + }, + "plugins": [ + { + "name": "flowx", + "source": "./", + "description": "Translate data pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. Deterministic translation for known activity types with agentic fallback.", + "version": "0.1.0", + "license": "MIT", + "keywords": ["adf", "databricks", "migration", "dabs", "lakeflow", "orchestration", "data-pipelines"] + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index cf9403a..e41c463 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "flowx", - "version": "0.2.0", + "version": "0.1.0", "description": "Translate Azure Data Factory pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. Deterministic translation for known activity types with agentic fallback.", "author": { "name": "Greg Hansen", @@ -9,10 +9,10 @@ "license": "MIT", "keywords": ["adf", "databricks", "migration", "dabs", "lakeflow", "orchestration", "azure-data-factory"], "skills": [ - "./skills/setup", - "./skills/ingest", - "./skills/translate", - "./skills/prepare", - "./skills/migrate" + "./skills/flowx-setup", + "./skills/flowx-discover", + "./skills/flowx-convert", + "./skills/flowx-package", + "./skills/flowx-migrate" ] } diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 78d82a7..67fd130 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,7 +1,7 @@ # See https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms # and https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-githubs-form-schema name: Bug Report -description: Something is not working with Flowx +description: Something is not working with flowx title: "[BUG]: " labels: ["bug"] body: diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index b8f2257..056de77 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,7 +1,7 @@ # See https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms # and https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-githubs-form-schema name: Feature Request -description: Something new needs to happen with Flowx +description: Something new needs to happen with flowx title: "[FEATURE]: " labels: ["enhancement"] body: diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 71081c5..5063847 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -9,10 +9,11 @@ jobs: ci: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 with: + version: "0.11.2" + checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" python-version: "3.12" - name: Scrub internal proxy URLs from uv.lock run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock @@ -27,10 +28,11 @@ jobs: fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 with: + version: "0.11.2" + checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" python-version: "3.12" - name: Scrub internal proxy URLs from uv.lock run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock diff --git a/.gitignore b/.gitignore index 20db1d0..01bef33 100644 --- a/.gitignore +++ b/.gitignore @@ -27,13 +27,14 @@ ENV/ htmlcov/ .mypy_cache/ .ruff_cache/ +fixlog/ # OS .DS_Store Thumbs.db -# Flowx output directories -orchestra_output/ +# flowx output directories +flowx_output/ dab_output/ # Temporary ingest downloads @@ -45,3 +46,4 @@ tmp_adf_ingest_*/ *.pem *.key credentials.json +/fixlog/ diff --git a/AGENTS.md b/AGENTS.md index 6d5d79e..3bfe895 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ -# AI Agent Guidelines for Flowx +# AI Agent Guidelines for flowx ## Quick Command Reference @@ -10,31 +10,55 @@ make fmt # Format + lint (ruff + mypy) make clean # Remove build artifacts ``` -To run the **plugin skills** (ingest/translate/prepare/migrate) without a uv-based dev setup, +To run the **plugin skills** (discover/convert/package/migrate) without a uv-based dev setup, bootstrap a self-contained virtual environment with pip via the `setup` skill or directly: ```bash -bash scripts/bootstrap.sh # creates .venv and pip-installs requirements.txt -# then run plugin code with src/ on PYTHONPATH: -PYTHONPATH=src .venv/bin/python -m flowx.adapter inputs ingest +bash scripts/bootstrap.sh # creates the venv, pip-installs requirements.txt, writes .migration-venv +# then run plugin code with src/ on PYTHONPATH, using the interpreter from the marker file: +PY="$(cat .migration-venv)" +PYTHONPATH=src "$PY" -m flowx.adapter inputs discover ``` +`bootstrap.sh` creates the venv at `/Workspace/Users//.migration-skills` when running +under Databricks (Genie Code / notebooks; detected via `DATABRICKS_RUNTIME_VERSION`) and at +`/.venv` everywhere else. It writes the resolved interpreter path to +`/.migration-venv`; read that marker file rather than hardcoding an interpreter path. + +### Databricks Serverless / Genie Code Compatibility + +All skills and the bootstrap script are designed to run on **Databricks serverless compute** +(Genie Code, notebook serverless) as well as local machines. Key adaptations: + +- **venv:** `bootstrap.sh` creates the venv at `/Workspace/Users//.migration-skills` + on Databricks and `/.venv` locally, writing the interpreter path to + `/.migration-venv`. It falls back to `--without-pip` + `get-pip.py` when `ensurepip` + is unavailable (standard on serverless images). +- **Auth:** `workspace_downloader.py` auto-detects `DATABRICKS_RUNTIME_VERSION` and writes + `~/.databrickscfg` from `dbruntime.databricks_repl_context` so the SDK can authenticate. +- **CLI:** `databricks bundle validate/deploy` is NOT available on serverless — use the web + terminal or a local CLI session for those steps. + ## Project Overview -Flowx is an agent plugin that translates Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs via Declarative Automation Bundles (DABs). +flowx is an agent plugin that translates Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs via Declarative Automation Bundles (DABs). ## Data Flow ``` -ADF JSON -> Parse (AST) -> Classify (Inventory) -> Translate (IR) -> Prepare (Tasks + Notebooks) -> Bundle (DABs) +ADF JSON -> Parse (AST) -> Classify (Inventory) -> Convert (IR) -> Package (Tasks + Notebooks) -> Bundle (DABs) ``` ## Architecture ### Three-Phase Pipeline -1. **Ingest** -- Parse ADF JSON from UC volumes -> typed AST -> inventory.json -2. **Translate** -- Registry dispatch + topological sort -> Pipeline IR (deterministic + agentic gaps) -3. **Prepare** -- IR -> DAB YAML + generated notebooks + setup scripts +All three phases write into one shared `` (default `./flowx_output`): +the DAB bundle at the top level, kept artifacts under `metadata/`, and transient +intermediates under `.work/` (pruned by `package`). + +1. **Discover** -- Parse ADF JSON from UC volumes -> typed AST -> `metadata/inventory.json` + `metadata/profile_report.csv` + verbatim `metadata/.arm.json` +2. **Convert** -- Registry dispatch + topological sort -> Pipeline IR (deterministic + agentic gaps); transient report at `.work/translation_report.json` +3. **Package** -- IR -> DAB YAML + generated notebooks + setup scripts; prunes `.work/` ### Key Patterns - `@dataclass(slots=True, kw_only=True)` for all models @@ -49,7 +73,7 @@ ADF JSON -> Parse (AST) -> Classify (Inventory) -> Translate (IR) -> Prepare (Ta | `models/adf_ast.py` | Typed AST nodes for ADF definitions | | `models/ir.py` | Databricks intermediate representation | | `models/dab.py` | DAB output schema types | -| `parser/adf_loader.py` | Parses ADF exports, produces inventory.json | +| `parser/adf_loader.py` | Parses ADF exports, produces `metadata/inventory.json` + `metadata/profile_report.csv` | | `parser/expression_parser.py` | Translates ADF expressions (@activity, @pipeline, @variables) | | `translator/engine.py` | Registry dispatch, topological sort, context threading | | `translator/activity_translators/` | One module per deterministic activity type (16 total) | @@ -59,6 +83,9 @@ ADF JSON -> Parse (AST) -> Classify (Inventory) -> Translate (IR) -> Prepare (Ta | `bundler/dab_writer.py` | Generates databricks.yml, job YAML, resources | | `bundler/notebook_writer.py` | Writes generated notebooks to bundle | | `bundler/setup_generator.py` | Setup scripts for UC volumes, secrets, connections | +| `reporting/coverage.py` | Builds per-pipeline coverage rows from `metadata/` | +| `reporting/results.py` | Writes per-run coverage to a UC table (run_id/run_date/run_by) via the SDK | +| `reporting/dashboard.py` | Installs + publishes an AI/BI coverage dashboard over the results table | ## Activity Types @@ -96,3 +123,22 @@ ExecuteDataFlow, SqlServerStoredProcedure, AzureFunction, WebHook, Custom, Execu 6. Move from AGENTIC_TYPES to DETERMINISTIC_TYPES in adf_loader.py 7. Update activity-mapping.md reference 8. Add test fixtures and unit tests + +## MCP server design notes + +Rationale behind non-obvious choices in `src/flowx/mcp/server.py` (kept here so the code carries +only one-line pointers): + +- **`@mcp.tool(structured_output=False)`** — FastMCP derives an `outputSchema` from a tool's + `-> dict` return annotation, but Databricks Genie Code's MCP client rejects tools that declare an + `outputSchema` (a 2025-06-18 spec feature): `tools/list` fails and Genie reports "can't fetch + tools" even though `initialize` succeeded. Suppressing it returns the dict as JSON text instead, + which every client accepts. +- **`build_http_app` returns FastMCP's own app** — it is not mounted inside another Starlette app. + Starlette does not run the lifespan of a *mounted* sub-app, so mounting leaves FastMCP's + StreamableHTTP session manager uninitialized and every `/mcp` request 500s with "Task group is not + initialized". +- **DNS-rebinding protection disabled** (`_transport_security`) — behind the Databricks Apps OAuth + proxy the SDK sees the workspace `Origin` and a proxied `Host: localhost:`, so its Host/Origin + allowlist misfires (403/421) while adding nothing on top of the proxy's authentication. Browser + CORS is a separate concern configured via `FLOWX_ALLOWED_ORIGINS`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d612ff..79d76a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,10 @@ -# Flowx Changelog +# flowx Changelog -All notable changes to Flowx will be documented in this file. +All notable changes to flowx will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). -## [0.0.1] +## [0.1.0] ### Added -- Initial release of the Flowx library +- Initial release of the flowx library diff --git a/src/orchestra/models/__init__.py b/CODEOWNERS.txt similarity index 100% rename from src/orchestra/models/__init__.py rename to CODEOWNERS.txt diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..7e2ee1e --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,24 @@ +## DB license + +**Definitions**. + +Agreement: The agreement between Databricks, Inc., and you governing the use of the Databricks Services, as that term is defined in the Master Cloud Services Agreement (MCSA) located at www.databricks.com/legal/mcsa. + +Licensed Materials: The source code, object code, data, and/or other works to which this license applies. + +**Scope of Use**. You may not use the Licensed Materials except in connection with your use of the Databricks Services pursuant to the Agreement. Your use of the Licensed Materials must comply at all times with any restrictions applicable to the Databricks Services, generally, and must be used in accordance with any applicable documentation. You may view, use, copy, modify, publish, and/or distribute the Licensed Materials solely for the purposes of using the Licensed Materials within or connecting to the Databricks Services. If you do not agree to these terms, you may not view, use, copy, modify, publish, and/or distribute the Licensed Materials. + +**Redistribution**. You may redistribute and sublicense the Licensed Materials so long as all use is in compliance with these terms. In addition: + +- You must give any other recipients a copy of this License; +- You must cause any modified files to carry prominent notices stating that you changed the files; +- You must retain, in any derivative works that you distribute, all copyright, patent, trademark, and attribution notices, excluding those notices that do not pertain to any part of the derivative works; and +- If a "NOTICE" text file is provided as part of its distribution, then any derivative works that you distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the derivative works. + +You may add your own copyright statement to your modifications and may provide additional license terms and conditions for use, reproduction, or distribution of your modifications, or for any such derivative works as a whole, provided your use, reproduction, and distribution of the Licensed Materials otherwise complies with the conditions stated in this License. + +**Termination**. This license terminates automatically upon your breach of these terms or upon the termination of your Agreement. Additionally, Databricks may terminate this license at any time on notice. Upon termination, you must permanently delete the Licensed Materials and all copies thereof. + +**DISCLAIMER; LIMITATION OF LIABILITY.** + +THE LICENSED MATERIALS ARE PROVIDED “AS-IS” AND WITH ALL FAULTS. DATABRICKS, ON BEHALF OF ITSELF AND ITS LICENSORS, SPECIFICALLY DISCLAIMS ALL WARRANTIES RELATING TO THE LICENSED MATERIALS, EXPRESS AND IMPLIED, INCLUDING, WITHOUT LIMITATION, IMPLIED WARRANTIES, CONDITIONS AND OTHER TERMS OF MERCHANTABILITY, SATISFACTORY QUALITY OR FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. DATABRICKS AND ITS LICENSORS TOTAL AGGREGATE LIABILITY RELATING TO OR ARISING OUT OF YOUR USE OF OR DATABRICKS’ PROVISIONING OF THE LICENSED MATERIALS SHALL BE LIMITED TO ONE THOUSAND ($1,000) DOLLARS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE LICENSED MATERIALS OR THE USE OR OTHER DEALINGS IN THE LICENSED MATERIALS. diff --git a/Makefile b/Makefile index 419faa1..a99a435 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ lock-dependencies: requirements: uv export --frozen --no-dev --no-emit-project --no-hashes --format requirements-txt -o requirements.txt -precommit: fmt requirements +precommit: fmt lock-dependencies requirements help: @echo "Available targets:" diff --git a/NOTICE.md b/NOTICE.md new file mode 100644 index 0000000..3ab6a0d --- /dev/null +++ b/NOTICE.md @@ -0,0 +1,3 @@ +## Support +Databricks does not offer official support for Databricks Solutions and its repository. +For any issue with this assets or the demos installed, please open an issue using github and the team will have a look on a best effort basis. diff --git a/README.md b/README.md index f265657..0ba13d5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ -# Flowx +# flowx ADF to Databricks Lakeflow Jobs translator via Declarative Automation Bundles. -Flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). It deterministically translates known activity types and falls back to agentic LLM-assisted translation for complex or rare types. +flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). It deterministically translates known activity types and falls back to agentic LLM-assisted translation for complex or rare types. ## Architecture ``` - Flowx Pipeline + flowx Pipeline ================== ADF JSON (UC Volumes) | v +------------------+ - | 1. INGEST | Parse ADF ARM/JSON exports - | adf_loader.py | -> Typed AST -> inventory.json + | 1. PROFILE | Parse ADF ARM/JSON exports + | adf_loader.py | -> Typed AST -> metadata/inventory.json +------------------+ | v @@ -43,14 +43,14 @@ Flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline de 2. Run the end-to-end migration: ``` - /flowx:migrate + /flowx:flowx-migrate ``` Or run individual phases: ``` - /flowx:ingest # Parse ADF JSON, produce inventory - /flowx:translate # Deterministic + agentic translation - /flowx:prepare # Generate DABs project + /flowx:flowx-discover # Parse ADF JSON, produce inventory + complexity report + /flowx:flowx-convert # Deterministic + agentic translation + /flowx:flowx-package # Generate DABs project ``` ## Supported ADF Activity Types @@ -95,20 +95,22 @@ Flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline de ## How It Works -### Phase 1: Ingest -Reads ADF JSON definitions from Unity Catalog volumes, normalizes ARM template format, parses into typed AST nodes, and classifies each activity as deterministic, agentic, or unsupported. Produces `inventory.json`. +### Phase 1: Discover +Reads ADF JSON definitions from Unity Catalog volumes, normalizes ARM template format, parses into typed AST nodes, and classifies each activity as deterministic, agentic, or unsupported. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`. -### Phase 2: Translate +### Phase 2: Convert Applies deterministic translators via registry dispatch, resolves dependencies through topological sort, and threads immutable `TranslationContext` through control-flow visitors. Agentic gaps are flagged for LLM-assisted translation. Produces Pipeline IR. -### Phase 3: Prepare +### Phase 3: Package Converts Pipeline IR into a deployable DABs project: `databricks.yml`, per-job YAML resource files, generated Python notebooks, and setup scripts for UC volumes, secrets, and connections. ## Output Format +All three phases write into one shared output directory (default `./flowx_output`): + ``` -dab_output/ - databricks.yml # Bundle configuration +flowx_output/ + databricks.yml # Bundle configuration (package) resources/ jobs/ .yml # One job per ADF pipeline @@ -120,6 +122,13 @@ dab_output/ create_volumes.py # UC volume setup create_secrets.py # Secret scope setup create_connections.py # Connection setup + SETUP.md # Setup instructions (package) + metadata/ + inventory.json # discover: activity inventory + profile_report.csv # profile: per-pipeline complexity report + .arm.json # discover: verbatim original ADF/ARM source + configuration.json # modify: collected configuration answers + .work/ # transient intermediates (translation report, IR, gaps.json); pruned by prepare ``` ## Development diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..75e9821 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,6 @@ +# Security Policy + +## Reporting a Vulnerability + +Please email bugbounty@databricks.com to report any security vulnerabilities. We will acknowledge receipt of your vulnerability and strive to send you regular updates about our progress. If you're curious about the status of your disclosure please feel free to email us again. + diff --git a/app/README.md b/app/README.md new file mode 100644 index 0000000..6f52aa3 --- /dev/null +++ b/app/README.md @@ -0,0 +1,192 @@ +# flowx MCP server (Databricks App) + +Hosts flowx's migration phases and helper operations as [Model Context +Protocol](https://modelcontextprotocol.io) tools so agentic clients (Claude +Code, Claude Desktop, Databricks Genie Code) can drive an ADF → Databricks +Lakeflow migration by calling tools instead of shelling out to a CLI. + +The MCP server itself lives in the flowx package at +[`src/flowx/mcp/`](../src/flowx/mcp); this directory is just the +Databricks App wrapper and deployment tooling. + +## Tool + +The server exposes a **single** MCP tool, `flowx(command, parameters)`, to stay well under +host tool-count limits (e.g. Genie Code's 20-tools-across-all-servers cap). `command` selects the +operation; `parameters` is its keyword-argument dict. + +| `command` | Wraps | Purpose | +|-----------|-------|---------| +| `inputs` | `adapter inputs` | List a phase's input prompts/defaults | +| `discover` | `adapter discover` | Parse ADF JSON, classify activities | +| `convert` | `adapter convert` | ADF activities → Databricks IR | +| `merge_agentic` | `adapter convert --merge-agentic` | Merge agent-produced results into the report | +| `inspect` | `adapter inspect` | Surface pending translation options | +| `apply_answers` | `adapter modify` | Apply answers → stamped IR | +| `materialize_lookup` | `adapter materialize-lookup` | CSV → lookup-values JSON | +| `workspace_paths` | `adapter workspace-paths` | Detect workspace paths / hosts | +| `package` | `adapter package` | Emit the deployable DAB bundle | +| `migrate` | discover→convert→package | Full non-interactive migration | +| `record_results` | `adapter record-results` | Write coverage to a UC table | +| `install_dashboard` | `adapter install-dashboard` | Publish the coverage dashboard | + +Example: `flowx(command="discover", parameters={"adf_source_path": "/Volumes/main/default/adf_export", "output_dir": "./out"})`. + +Each command is a thin bridge over `python -m flowx.adapter` (the same entry point the agent +skills use), then reads back the JSON/CSV artifacts each phase writes — so the MCP surface stays in +lockstep with the tested CLI contract. + +## Run locally + +```bash +pip install -e ".[mcp]" # from the repo root + +# stdio transport (Claude Code / Claude Desktop): +python -m flowx.mcp + +# streamable-HTTP transport (same server Databricks Apps runs): +python -m flowx.mcp --http --port 8000 +# MCP endpoint: http://localhost:8000/mcp health: http://localhost:8000/ +``` + +Register the stdio server with a local MCP client, e.g.: + +```json +{ "mcpServers": { "flowx": { "command": "python", "args": ["-m", "flowx.mcp"] } } } +``` + +## Deploy as a Databricks App (for Genie Code) + +```bash +# Authenticated Databricks CLI (v0.230+) required. +./app/deploy.sh +``` + +`deploy.sh` stages a self-contained bundle in a temporary directory outside the repo (the app entrypoint +plus a vendored copy of the pure-Python `flowx` package), syncs it to your +workspace, and creates/deploys the app (default name **`mcp-flowx`**). + +> **Clone into `/Workspace/Shared`.** `deploy.sh` deploys the app source from +> `/Workspace/Shared/` because the app's service principal **cannot read +> private `/Workspace/Users/` folders** by default. Clone flowx into a Git +> folder under `/Workspace/Shared` (e.g. `/Workspace/Shared/flowx`); if that +> folder is restricted in your workspace, use another all-users location and pass it +> via `APP_SOURCE_PATH`. + +End-to-end, to use it from Genie Code: + +1. **Deploy** with `./app/deploy.sh`. The app is named `mcp-flowx` and deploys the + source from `/Workspace/Shared/mcp-flowx` (override with `APP_SOURCE_PATH`). The + script prints the app URL; the MCP endpoint is `/mcp`. +2. **Grant app access:** give **Can use** on `mcp-flowx` to the users / service + principals that will call it (Apps UI → *Permissions*, or + `databricks apps set-permissions mcp-flowx ...`). +3. **Grant data access:** the app authenticates as its own service principal, so + grant that principal access to the catalogs / schemas / Unity Catalog volumes + the migration reads/writes (and any SQL warehouse used by `record-results` / + `install-dashboard`). +4. **Add it in Genie Code (Agent mode):** open Genie Code **Settings → MCP Servers → + Add Server**, choose **Custom MCP server**, select the `mcp-flowx` app, and + **Save**. The `flowx` tool becomes available immediately. Verify via the + health endpoint `/`. + +> The `mcp-` name prefix also makes the app **auto-listed in the AI Playground**. Genie Code's +> **Custom MCP server** picker selects any Databricks App by name regardless of prefix. + +Genie Code requires a custom MCP app to be (1) in the **same workspace**, (2) reachable +at `https:///mcp`, and (3) **stateless** — this server sets +`stateless_http=True` and adds CORS, so it qualifies. If a browser CORS error appears, +set the app env var `FLOWX_ALLOWED_ORIGINS` to your workspace URL and redeploy. +MCP access is capped at **20 tools** across all servers; flowx exposes just **one** tool +(`flowx`, with 12 commands), so it uses a single slot. + +> Run `./app/deploy.sh` from a Databricks CLI session (workspace web terminal or a +> local machine) — `databricks apps` deploy is not available from serverless +> notebook Python. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp) +> and [host a custom MCP server](https://docs.databricks.com/aws/en/generative-ai/mcp/custom-mcp). + +## Troubleshooting + +**Genie Code can't connect / add the server (is it CORS or the server?)** — Tell them apart +from the app's logs (`databricks apps logs ` or the app UI): + +- **Server-side error (not CORS):** the logs show a `500` / `RuntimeError: Task group is not + initialized` on `POST /mcp`. That means the StreamableHTTP session manager never started — + it happens if the MCP app is *mounted inside another Starlette app* (whose lifespan doesn't + run the sub-app). The server now avoids this by serving FastMCP's own app directly; make sure + you redeployed the current `app/`. Sanity-check the app is up with `curl /` + (expect `{"status":"ok"}`). +- **CORS:** the request reaches the server fine but the **browser console** shows a CORS error + (blocked by `Access-Control-Allow-Origin`), with no corresponding 500 in the app logs. Set the + app env var `FLOWX_ALLOWED_ORIGINS` to your workspace URL and redeploy. + +**Server connects but Genie Code "fails to fetch tools"** — the connection (`initialize`) succeeds, +but `tools/list` comes back empty / errors. This is a **schema-compatibility** problem, not a +transport one: Genie Code's MCP client rejects tools that declare an `outputSchema` (the structured- +output feature from the 2025-06-18 spec). FastMCP derives one automatically from a tool's return-type +annotation, so the `flowx` tool registers with `@mcp.tool(structured_output=False)` to suppress +it (the result is still returned as JSON text). If you see this after customizing the server, make +sure no tool emits an `outputSchema` — check with `curl`-ing a `tools/list` request or inspecting +`mcp.list_tools()`. + +**`403 Forbidden` with `Invalid Origin header`, or `421 Misdirected Request` with `Invalid Host +header: localhost:8000`** (from `transport_security.py` in the logs) — this is the MCP SDK's +DNS-rebinding protection, **not** CORS. Behind the Databricks Apps OAuth proxy the app sees the +workspace `Origin` and a proxied `Host: localhost:8000`, so that allowlist check misfires. The +server now **disables** DNS-rebinding protection (the OAuth proxy already authenticates every +request); just redeploy the current `app/`. Note: `FLOWX_ALLOWED_ORIGINS` controls **only** +browser CORS now — it does not enable or affect this Host/Origin check. + + + +**`mkdir: cannot create directory ...: Permission denied`** — On Databricks (Genie web +terminal / serverless), `/tmp` and `$TMPDIR` are often not writable, while the `/Workspace` +filesystem where the repo lives is. `deploy.sh` stages the bundle in the **repo's parent +directory** first (e.g. `/Workspace/Shared`, which is writable and outside the git repo), +falling back to `$TMPDIR` / `/tmp` / `$HOME` for local runs. If it still can't find a writable +base, set `TMPDIR` to a writable path and re-run. (Staging is never placed inside the repo, +so `databricks sync` won't drop files via the repo's `.gitignore`.) + +**`Error: please specify target`** — The Databricks CLI (v0.298+) makes `sync` and +`apps deploy` bundle-aware: if a `databricks.yml` is discoverable in the working +directory or any parent (for example a generated `flowx_output/databricks.yml`, +or one in your workspace home), the CLI loads that bundle and—when it has multiple +targets with no default—aborts with this error before deploying. `deploy.sh` already +runs every CLI call from a throwaway directory to avoid this; if you invoke the CLI +manually, do the same (or pass `--target `), and don't run it from inside a +generated bundle directory. + +## Inputs and outputs on a hosted app + +A Databricks App can't read the user's workspace / UC Volume files (only the container's +ephemeral disk is local; `/Volumes/...` is **not** auto-mounted). The MCP surface handles this +by passing data **inline** through the tool, since the calling agent *can* read those files: + +- **Input — small jobs:** pass the ADF JSON via `adf_definitions` — a mapping of relative path → JSON + content mirroring the ADF Git-export layout (`pipeline/…`, `dataset/…`, `linkedService/…`, + `trigger/…`); a single ARM-template object is also accepted. The server materializes it to a temp + dir. Capped at ~5 MB (`FLOWX_MAX_INLINE_BYTES`) since it flows through the agent's context. +- **Input — large factories (recommended):** point the server at the source by reference, so the + bytes bypass the agent and it scales to thousands of pipelines: + - `adf_volume_path` — a UC Volume directory, downloaded via the SDK **Files API**. + - `adf_workspace_path` — a `/Workspace` directory (e.g. an ADF Git folder), listed and downloaded + via the SDK **Workspace API** (workspace files use the Workspace API, *not* the Files API). + + Grant the app's service principal read on whichever path you use. (`adf_source_path` / `source_dir` + remain for paths the server itself can read — local hosting or a mounted volume.) +- **Output — large bundles (recommended):** `package`/`migrate` write the DAB to the target via the + SDK so the contents bypass the agent. Pass `output_volume_path` (uploaded to a UC Volume via the SDK + Files API) **or** `output_workspace_path` (uploaded to a `/Workspace` directory via the SDK Workspace + API, with `ImportFormat.RAW` so files land verbatim rather than as notebooks). Either returns + `bundle_uploaded` (location + file list). Grant the service principal write on the target. +- **Output — small bundles:** without an output path, `package`/`migrate` return the DAB inline as + `bundle = {"files": {relpath: text, …}, "truncated": [...]}` (capped ~2 MB) for the caller to persist. + +## Known constraints / follow-ups + +- **`databricks bundle validate/deploy`** of the *generated* DAB is a separate, + user-driven step (run from a CLI session); it is intentionally not invoked by + these tools. +- **Long-running phases.** Large factories can exceed default client timeouts; + the server-side subprocess timeout is configurable via `FLOWX_MCP_TIMEOUT` + (seconds). diff --git a/app/app.py b/app/app.py new file mode 100644 index 0000000..b12226d --- /dev/null +++ b/app/app.py @@ -0,0 +1,21 @@ +"""Databricks App entry point for the flowx MCP server. + +Databricks Apps run this module's ``command`` from ``app.yaml``. The flowx +package is vendored alongside this file by ``deploy.sh`` (into ``flowx/``), +so it imports directly without a separate install step. + +The module also exposes ``app`` so it can be served with ``uvicorn app:app``. +""" + +import os + +from flowx.mcp.server import build_http_app + +app = build_http_app() + +if __name__ == "__main__": + import uvicorn + + # Databricks Apps inject the port to bind via DATABRICKS_APP_PORT. + port = int(os.environ.get("DATABRICKS_APP_PORT", "8000")) + uvicorn.run(app, host="0.0.0.0", port=port) diff --git a/app/app.yaml b/app/app.yaml new file mode 100644 index 0000000..748afd0 --- /dev/null +++ b/app/app.yaml @@ -0,0 +1,5 @@ +# Databricks App manifest for the flowx MCP server. +# The app serves the MCP streamable-HTTP transport at /mcp and a health check at /. +command: + - python + - app.py diff --git a/app/deploy.sh b/app/deploy.sh new file mode 100755 index 0000000..68a1ca2 --- /dev/null +++ b/app/deploy.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Deploy the flowx MCP server as a Databricks App. +# +# Builds a self-contained source bundle (app entrypoint + vendored flowx +# package), syncs it to the workspace, and creates/deploys the app. +# +# Requirements: Databricks CLI v0.230+ authenticated to the target workspace. +# +# Env overrides: +# APP_NAME App name (default: mcp-flowx). The `mcp-` prefix makes the app +# auto-listed in the AI Playground; Genie Code's "Add Server > Custom MCP +# server" picker can also select any Databricks App by name. +# APP_SOURCE_PATH Workspace source path the app deploys from (default: +# /Workspace/Shared/). It MUST be readable by the app's +# service principal, so it defaults to /Workspace/Shared — NOT a user's +# private /Workspace/Users/ home, which the app SP cannot read. +# DATABRICKS_PROFILE CLI profile to use (default: env/DEFAULT auth) +set -euo pipefail + +APP_NAME="${APP_NAME:-mcp-flowx}" +APP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$APP_DIR/.." && pwd)" + +PROFILE_FLAG=() +if [ -n "${DATABRICKS_PROFILE:-}" ]; then + PROFILE_FLAG=(--profile "$DATABRICKS_PROFILE") +fi + +# Stage the bundle OUTSIDE the repo. `databricks sync` is git-aware and applies the +# enclosing repo's .gitignore: a staging dir inside the repo (e.g. app/.build, which +# .gitignore lists) gets its files excluded, so the deployed source is missing app.py +# and flowx/. A temp dir has no enclosing git repo / .gitignore, so every file syncs. +# +# The CLI is also bundle-aware: if a databricks.yml is discoverable in the working +# directory or any parent (e.g. a generated flowx_output/databricks.yml), `sync` / +# `apps deploy` load it and fail with "Error: please specify target". Running from the +# (databricks.yml-free) staging dir avoids that too. All CLI paths are absolute. +# +# Create the staging dir in a writable location, trying the repo's PARENT first. On +# Databricks (Genie web terminal / serverless) the repo lives on the writable /Workspace +# filesystem while /tmp and $TMPDIR are often unwritable — a bare `mktemp -d` there fails +# with "mkdir: cannot create directory ...: Permission denied". The repo's parent +# (e.g. /Workspace/Shared) is writable AND outside the git repo, so staging there both +# succeeds and avoids the repo's .gitignore (which would otherwise make `databricks sync` +# drop staged files). $TMPDIR/tmp/$HOME are fallbacks for local (non-Databricks) runs. +STAGE_DIR="" +for _base in "$(dirname "$REPO_ROOT")" "${TMPDIR:-}" /tmp /local_disk0/tmp "$HOME"; do + [ -n "$_base" ] && [ -d "$_base" ] && [ -w "$_base" ] || continue + STAGE_DIR="$(mktemp -d "${_base%/}/mcp-flowx-build.XXXXXX" 2>/dev/null)" && break +done +if [ -z "$STAGE_DIR" ]; then + echo "ERROR: could not create a writable staging directory (tried the repo parent, \$TMPDIR, /tmp, \$HOME)." >&2 + echo " Set TMPDIR to a writable local path and re-run." >&2 + exit 1 +fi +trap 'rm -rf "$STAGE_DIR"' EXIT +dbx() { (cd "$STAGE_DIR" && databricks "${PROFILE_FLAG[@]}" "$@"); } + +echo "==> Staging self-contained app bundle in $STAGE_DIR (outside the repo so sync includes every file)" +cp "$APP_DIR/app.py" "$APP_DIR/app.yaml" "$APP_DIR/requirements.txt" "$STAGE_DIR/" +cp -R "$REPO_ROOT/src/flowx" "$STAGE_DIR/flowx" +find "$STAGE_DIR/flowx" -type d -name '__pycache__' -prune -exec rm -rf {} + 2>/dev/null || true + +# Deploy the source from a location the app's service principal can read. A user's +# private /Workspace/Users/ home is NOT readable by the app SP, so default to +# the shared workspace folder. Override with APP_SOURCE_PATH if you use a different +# all-users location. +SOURCE_PATH="${APP_SOURCE_PATH:-/Workspace/Shared/$APP_NAME}" + +echo "==> Ensuring app '$APP_NAME' exists" +if ! dbx apps get "$APP_NAME" >/dev/null 2>&1; then + dbx apps create "$APP_NAME" +fi + +echo "==> Syncing bundle to $SOURCE_PATH" +dbx sync --full "$STAGE_DIR" "$SOURCE_PATH" + +echo "==> Deploying app" +dbx apps deploy "$APP_NAME" --source-code-path "$SOURCE_PATH" + +echo "==> Deployed. App details:" +APP_URL="$(dbx apps get "$APP_NAME" --output json \ + | python3 -c 'import sys, json; print(json.load(sys.stdin).get("url", ""))')" +echo " name: $APP_NAME" +echo " url: ${APP_URL:-(pending — re-run 'databricks apps get $APP_NAME')}" +echo +echo "==> Next steps to use it in Genie Code:" +echo " 1. MCP endpoint: ${APP_URL:-}/mcp" +echo " 2. Grant 'Can use' on the app to the users / service principals that will call it" +echo " (Apps UI > Permissions, or: databricks apps set-permissions $APP_NAME ...)." +echo " 3. Grant that app's service principal access to the catalogs/schemas/volumes the" +echo " migration touches (and any SQL warehouse used by the reporting tools)." +echo " 4. Add it in Genie Code (Agent mode): Settings > MCP Servers > Add Server >" +echo " Custom MCP server > select '$APP_NAME' > Save. Tools appear immediately." +echo " 5. If a browser CORS error appears, set the app env var FLOWX_ALLOWED_ORIGINS" +echo " to your workspace URL and redeploy." diff --git a/app/requirements.txt b/app/requirements.txt new file mode 100644 index 0000000..fd66841 --- /dev/null +++ b/app/requirements.txt @@ -0,0 +1,10 @@ +# Databricks App dependencies for the flowx MCP server. +# The flowx package itself is vendored next to app.py by deploy.sh, so only +# third-party runtime dependencies are installed here (flowx's own deps plus +# the MCP/HTTP server stack). +mcp>=1.12 +uvicorn>=0.30 +starlette>=0.40 +databricks-sdk>=0.40 +pyyaml>=6.0 +sqlglot>=25.0 diff --git a/docs/README.md b/docs/README.md index 6e9f65b..f151cad 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,4 +1,4 @@ -# Flowx docs +# flowx docs Documentation site for [flowx](https://github.com/ghanse/flowx), built with [fumadocs](https://fumadocs.dev) and deployed to GitHub Pages. diff --git a/docs/app/(home)/page.tsx b/docs/app/(home)/page.tsx index 266cfcf..021cb6b 100644 --- a/docs/app/(home)/page.tsx +++ b/docs/app/(home)/page.tsx @@ -3,7 +3,7 @@ import Link from 'next/link'; export default function HomePage() { return (
-

Flowx

+

flowx

Programmatically translate your data pipelines to Databricks Lakeflow jobs.

diff --git a/docs/app/layout.tsx b/docs/app/layout.tsx index 975d91b..da935fb 100644 --- a/docs/app/layout.tsx +++ b/docs/app/layout.tsx @@ -9,8 +9,8 @@ const inter = Inter({ export const metadata = { title: { - default: 'Flowx', - template: '%s | Flowx', + default: 'flowx', + template: '%s | flowx', }, description: 'Translate Azure Data Factory pipelines to Databricks Lakeflow Jobs.', diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx new file mode 100644 index 0000000..bed4d9e --- /dev/null +++ b/docs/content/docs/architecture.mdx @@ -0,0 +1,90 @@ +--- +title: Architecture +description: How flowx is structured as a set of agent skills and MCP tools. +--- + +import { Callout } from 'fumadocs-ui/components/callout'; + +flowx translates data pipelines into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). A set of Python functions is exposed through two surfaces: **agent skills** invoked through a CLI and **MCP tools** called by an agent session. + +## Three-phase pipeline + +```text +ADF JSON ──▶ discover ──▶ convert ──▶ package ──▶ databricks.yml (DAB) + (parse + (activities (IR → jobs, + classify) → IR) notebooks, setup) +``` + +All three phases share one `output_dir` (default `./flowx_output`): + +| Phase | Module | Reads | Writes | +|---------------|-------------------------|------------------------|------------------------------------------------------------------------------------------| +| **profile** | `parser/adf_loader.py` | ADF JSON exports | `metadata/inventory.json`, `metadata/profile_report.csv`, `metadata/.arm.json` | +| **translate** | `translator/engine.py` | inventory + ADF source | `.work/translation_report.json` (transient IR) | +| **prepare** | `bundler/dab_writer.py` | translation report | `databricks.yml`, `resources/`, `src/`, `setup/` (and prunes `.work/`) | + +Each activity is classified with a `TranslationStrategy`: +* `DETERMINISTIC` (in-process translators) +* `AGENTIC` (LLM-assisted gaps) +* `UNSUPPORTED` + +The reporting layer (`reporting/`) can write per-run coverage to a Unity Catalog table and publish an AI/BI dashboard. + +## Two surfaces over one core + +```text + ┌─────────────────────────────┐ + Agent skills ───▶│ python -m flowx.adapter │───▶ phase modules (discover/ + (discover/..., │ (unified CLI entry point) │ convert/package) + + setup, migrate) └─────────────────────────────┘ adapter operations + ▲ + │ subprocess (same contract) + ┌─────────────────────────────┐ + MCP tool ──────▶│ flowx.mcp (FastMCP) │ + (Claude / Genie) │ 1 tool, 12 commands → │ + │ adapter bridge │ + └─────────────────────────────┘ +``` + +The unified `flowx.adapter` CLI is the single contract. Both surfaces go through it: + +- **Agent skills** (`skills/`) shell out to `python -m flowx.adapter …` directly. +- **MCP tool** (`src/flowx/mcp/`) is a thin bridge: the single `flowx(command, parameters)` tool builds the same adapter arguments for the chosen `command`, runs them as a subprocess via `flowx.mcp.runner`, then reads back the JSON/CSV artifacts and returns a structured result. No phase logic is duplicated, so the tool surface can't drift from the tested CLI. A single tool also keeps flowx to **one** of the host's tool slots (e.g. Genie Code's 20-tool cap). + +### MCP tool layer + +| Module | Purpose | +|-------------------|---------------------------------------------------------------------------------------------------------------------------| +| `mcp/server.py` | `FastMCP` server; registers tools and builds the stdio / streamable-HTTP apps | +| `mcp/runner.py` | Subprocess bridge to `flowx.adapter` with artifact summarizers (for running translation without the `mcp` dependency) | +| `mcp/__main__.py` | `python -m flowx.mcp` entry point (stdio default, `--http` for hosting) | + +The `flowx` tool's `command` selects the adapter operation: `inputs`, `discover`, `convert`, `merge_agentic`, `inspect`, `apply_answers`, `materialize_lookup`, `workspace_paths`, `package`, `migrate`, `record_results`, and `install_dashboard` (with `parameters` carrying that command's arguments). + +## Deployment topology + +The MCP server runs in whichever transport fits the calling tool. This is chosen when the `setup` skill is invoked: + +- **Local / Claude Code:** `setup` installs `mcp` + `uvicorn` + `starlette` into the venv; the server runs over **stdio** and is registered with the MCP client. +- **Databricks Genie Code:** `setup` runs `app/deploy.sh`, which stages a self-contained bundle (app entrypoint + a vendored copy of the pure-Python flowx source), syncs it to the workspace, and creates/deploys the **`mcp-flowx` Databricks App**. The app serves the MCP streamable-HTTP transport at `/mcp` (health at `/`) via `uvicorn` → `Starlette` → `FastMCP`. To meet Genie Code's requirements the server runs **stateless** (`stateless_http=True`) and enables CORS (origins via `FLOWX_ALLOWED_ORIGINS`). You add it in Genie Code under **Settings → MCP Servers → Add Server → Custom MCP server**; Genie connects over OAuth, access is governed by the app's Databricks Apps permissions, and the app authenticates to the workspace as its own service principal. + +```text + Local (Claude Code / other agents) Databricks Genie Code + ─────────────────────────────────── ───────────────────────────────── + agent ◀── stdio ──▶ python -m Genie ◀── HTTPS /mcp ──▶ Databricks App + flowx.mcp (uvicorn → Starlette + (in the venv) → FastMCP streamable-HTTP) + app authenticates as its + own service principal +``` + +See [Installation](/docs/installation#running-flowx-as-an-mcp-server) for the exact commands and the [app README](https://github.com/ghanse/flowx/tree/main/app) for deployment details. + + +A Databricks App can't read the user's workspace / UC Volume files (`/Volumes/...` is **not** auto-mounted). Two ways to get data in/out of the `flowx` tool: + +- **Small jobs — inline.** `command="discover"`/`"migrate"` accept `adf_definitions` (a mapping of relative path → ARM JSON, Git-export layout), supplied by the agent and materialized to a temp dir; `package`/`migrate` return the DAB inline as `bundle = {"files": {relpath: text, …}}`. Inline data flows through the agent's context, so it's capped (~5 MB in). +- **Large factories — by reference (recommended).** Point the server at the source: `adf_volume_path` (a UC Volume read via the SDK Files API) or `adf_workspace_path` (a `/Workspace` directory — e.g. an ADF Git folder — read via the SDK Workspace API). Write the DAB to `output_volume_path` (UC Volume, SDK Files API) or `output_workspace_path` (`/Workspace` directory, SDK Workspace API with `ImportFormat.RAW`), returned as `bundle_uploaded`. The bytes bypass the agent entirely, so it scales to thousands of pipelines; grant the app's service principal read on the source and write on the output target. + +Locally hosted, ordinary `adf_source_path` / `output_dir` paths and mounted volumes work as-is. + diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx index e6b482d..d758899 100644 --- a/docs/content/docs/guide.mdx +++ b/docs/content/docs/guide.mdx @@ -35,15 +35,15 @@ databricks fs cp -r ./adf-export dbfs:/Volumes/main/default/adf_export ## Run the end-to-end migration -Open a fresh conversation and prompt your agent with the path to your JSON templates and a target directory for the output bundle: +Open a fresh conversation and prompt your agent with the path to your JSON templates and a target output directory: -> Use flowx to migrate the ADF pipelines at `/Volumes/main/default/adf_export` into a Databricks Asset Bundle at `./bundle/`. +> Use flowx to migrate the ADF pipelines at `/Volumes/main/default/adf_export` into a Databricks Asset Bundle at `./flowx_output/`. -Flowx will use the `migrate` skill to chain 3 other skills: +flowx will use the `migrate` skill to chain 3 other skills. All three phases write into one shared output directory (default `./flowx_output`): -1. `ingest` parses every JSON file, builds an inventory, and assigns a translation strategy for each resource. This can be deterministic, agentic, or unsupported. -2. `translate` converts each activity to an intermediate representation. The agent will ask for confirmation before running any LLM-based translation. -3. `prepare` creates a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) with job configuration files, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). +1. `discover` parses every JSON file, builds an inventory, assigns a translation strategy for each resource (deterministic, agentic, or unsupported), and emits a `metadata/profile_report.csv` complexity report (one row per pipeline). +2. `convert` converts each activity to an intermediate representation. The agent will ask for confirmation before running any LLM-based translation. +3. `package` creates a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) with job configuration files, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). @@ -51,18 +51,37 @@ Flowx will use the `migrate` skill to chain 3 other skills: ## Review the output -Flowx creates Declarative Automation Bundles in the local file system. The generated bundle can be reviewed and modified before deployment. +flowx writes everything into a single shared output directory (default `./flowx_output`). The generated bundle can be reviewed and modified before deployment. The layout is: -Each bundle contains a top-level `databricks.yml` file with deployment targets and other variables, a `resources/` folder with job configuration, -a `src/` folder with code required to run the pipeline, and a `setup/` folder with scripts for creating supporting resources. +```text +flowx_output/ +├── databricks.yml # Bundle configuration (from package) +├── resources/ # Job configuration (from package) +├── src/ # Code required to run the pipeline (from package) +├── SETUP.md # Setup instructions (from package) +├── metadata/ +│ ├── inventory.json # discover: activity inventory +│ ├── profile_report.csv # profile: per-pipeline complexity report +│ ├── .arm.json # discover: verbatim original ADF/ARM pipeline source +│ └── configuration.json # modify: the collected configuration answers +└── .work/ # transient intermediates (translation report, IR, gaps.json); pruned by prepare +``` -The `translation_report.json` file lists every activity, its translation strategy, warnings raised during translation, and the location of any -generated artifacts. Review the translation report for any warnings, unsupported resources, or to-do items before deploying to your Databricks workspace. +The bundle itself contains a top-level `databricks.yml` file with deployment targets and other variables, a `resources/` folder with job configuration, +a `src/` folder with code required to run the pipeline, and a `SETUP.md` file describing supporting resources to create. + +During translation, a transient `translation_report.json` is written under `flowx_output/.work/`. It lists every activity, its translation strategy, warnings raised during translation, and the location of any +generated artifacts. Review the translation report for any warnings, unsupported resources, or to-do items before deploying to your Databricks workspace. The `package` phase prunes `.work/` after building the bundle (pass `--keep-intermediates` to retain it). Connection strings, credentials, and other protected configuration parameters are emitted as `SecretInstruction` setup steps that require [Databricks Secrets](https://docs.databricks.com/aws/en/security/secrets/). Run the setup scripts and populate secret values before deploying and running pipelines in your Databricks workspace. + +When running with workspace auth (e.g. Genie Code), `package` can optionally persist this run's +coverage to a Unity Catalog table — one row per pipeline stamped with a UUID `run_id`, `run_date`, +and `run_by` (`record-results`) — and install a published AI/BI coverage dashboard over that table +(`install-dashboard`). See [Configuration options](/docs/options) for details. diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index c8ed057..bc36385 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -9,28 +9,29 @@ Orchestration should be treated as a first class citizen during migrations. Beca their configuration can impact data processing results as much as the logic being orchestrated. While significant tooling exists for code conversion and data reconciliation, migrating from legacy orchestration systems is often manual, time-consuming, and prone to risk. -Flowx was created to automate migrations of data pipelines between various orchestrators. It provides a robust, tested set of capabilities +flowx was created to automate migrations of data pipelines between various orchestrators. It provides a robust, tested set of capabilities to parse existing data pipeline definitions, create migration artifacts, and convert data pipeline definitions to Databricks' [Lakeflow jobs framework](https://docs.databricks.com/aws/en/jobs/). -## How Flowx works +## How flowx works -Flowx is a set of agent skills and deterministic translators. Skills tell agentic tools (e.g. Databricks Genie Code, Claude Code, or any +flowx is a set of agent skills and deterministic translators. Skills tell agentic tools (e.g. Databricks Genie Code, Claude Code, or any agent that supports the open [Agent Skills](https://agentskills.io/) format) how to call deterministic translators that parse, translate, and generate Databricks resources. Translation runs in three phases: -1. `ingest` parses Azure Resource Manager templates (e.g. for Data Factory pipelines, datasets, linked services, and triggers) into an execution -tree and builds an inventory. -2. `translate` processes the inventory and converts each activity into a Databricks-compatible intermediate representation. *Deterministic +1. `discover` parses Azure Resource Manager templates (e.g. for Data Factory pipelines, datasets, linked services, and triggers) into an execution +tree, builds an inventory, and emits a per-pipeline complexity report (`metadata/profile_report.csv`). +2. `convert` processes the inventory and converts each activity into a Databricks-compatible intermediate representation. *Deterministic activities* are translated by Python handlers while *agentic activities* are handed off to an LLM-assisted translator with the right context. *Unsupported activities* are flagged as explicit gaps. -3. `prepare` converts each translated pipeline into a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) that +3. `package` converts each translated pipeline into a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) that can be deployed to a Databricks workspace. Bundles include job configuration files, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). -Each phase can be run independently, maintains its own input/output contract, produces artifacts you can inspect before moving to the next phase. +All three phases write into one shared output directory (default `./flowx_output`): the DAB bundle at the top level, kept artifacts under `metadata/`, and transient intermediates under `.work/` (pruned by `package`). Each phase can be run independently, maintains its own input/output contract, and produces artifacts you can inspect before moving to the next phase. ## Next steps +- **[Architecture](/flowx/docs/architecture)** — understand how flowx is deployed and how it translates - **[Installation](/flowx/docs/installation)** — install the flowx plugin in your agentic tool of choice. - **[Usage Guide](/flowx/docs/guide)** — an end-to-end walkthrough from raw ADF JSON to a deployable bundle. - **[Options](/flowx/docs/options)** — reference documenting options for customizing output when translating pipelines with flowx. diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index 99a4ba5..619c98d 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -5,13 +5,14 @@ description: Install flowx in Databricks Genie Code, Claude Code, or other agent import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; import { Callout } from 'fumadocs-ui/components/callout'; +import { Steps, Step } from 'fumadocs-ui/components/steps'; -Flowx is a set of [agent skills](https://github.com/ghanse/flowx/tree/main/skills) that can be installed and used with AI coding assistants. +flowx is a set of [agent skills](https://github.com/ghanse/flowx/tree/main/skills) that can be installed and used with AI coding assistants. To use these skills, install flowx as a plugin using your AI assistant's preferred installation method. -Clone the flowx repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos), then copy the `skills/` directory into a user-level skills folder: +Clone the flowx repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos). Clone it under **`/Workspace/Shared`** (e.g. `/Workspace/Shared/flowx`) rather than your private `/Workspace/Users/` home — when you later deploy the MCP server, the app's service principal must be able to read the source, and it has no access to private user folders by default. Then copy the `skills/` directory into a user-level skills folder: ```bash databricks workspace import-dir skills /Users//.assistant/skills @@ -24,13 +25,13 @@ databricks workspace import-dir skills /Workspace/.assistant/skills ``` Genie Code picks up skills from these directories automatically. Skills fire automatically when their description matches your request. -To invoke a specific skill, use the `@` prefix (e.g. `@migrate translate the ADF pipelines I exported to /Volumes/main/default/adf_export`). +To invoke a specific skill, use the `@` prefix (e.g. `@flowx-migrate translate the ADF pipelines I exported to /Volumes/main/default/adf_export`). See the [Databricks Genie Code Skills documentation](https://docs.databricks.com/aws/en/genie-code/skills) for more details. -Flowx is packaged as a Claude Code plugin. The plugin manifest lives at [`.claude-plugin/plugin.json`](https://github.com/ghanse/flowx/blob/main/.claude-plugin/plugin.json). To install +flowx is packaged as a Claude Code plugin. The plugin manifest lives at [`.claude-plugin/plugin.json`](https://github.com/ghanse/flowx/blob/main/.claude-plugin/plugin.json). To install flowx, run the following command from a Claude Code session: ```bash @@ -41,21 +42,21 @@ flowx, run the following command from a Claude Code session: You can also copy the skill folders into your local `/.claude/skills` folder: ```bash -cp -R skills/{setup,ingest,translate,prepare,migrate} ~/.claude/skills/ +cp -R skills/{flowx-setup,flowx-discover,flowx-convert,flowx-package,flowx-migrate} ~/.claude/skills/ ``` -Once installed, the skills can be invoked using `/flowx:migrate`, `/flowx:ingest`, etc. Claude Code can also invoke the skills when prompted. See the [Claude Code Plugins documentation](https://docs.claude.com/en/docs/claude-code/plugins) for more details. +Once installed, the skills can be invoked using `/flowx:flowx-migrate`, `/flowx:flowx-discover`, etc. Claude Code can also invoke the skills when prompted. See the [Claude Code Plugins documentation](https://docs.claude.com/en/docs/claude-code/plugins) for more details. Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install the flowx skills. The general pattern: -1. Copy each skill folder (`skills/setup`, `skills/ingest`, `skills/translate`, `skills/prepare`, `skills/migrate`) into the tool's configured skills directory. +1. Copy each skill folder (`skills/flowx-setup`, `skills/flowx-discover`, `skills/flowx-convert`, `skills/flowx-package`, `skills/flowx-migrate`) into the tool's configured skills directory. 2. Make sure the path contains `SKILL.md` directly, 3. Restart the tool if it caches skill metadata at startup. -If your tool expects a single Markdown file instead of a directory tree, use the following command to flatten Flowx's skills files: +If your tool expects a single Markdown file instead of a directory tree, use the following command to flatten flowx's skills files: ```bash cat skills/*/SKILL.md > flowx-skills.md @@ -64,11 +65,113 @@ cat skills/*/SKILL.md > flowx-skills.md -## Setting up the Python environment +## Running flowx as an MCP server -Flowx's skills invoke Python modules that may depend on third-party packages. The `setup` skill provisions an isolated virtual environment with the required dependencies. +flowx's phases are also packaged as [Model Context Protocol](https://modelcontextprotocol.io) tools (in [`src/flowx/mcp/`](https://github.com/ghanse/flowx/tree/main/src/flowx/mcp)) so an agent can invoke them directly instead of shelling out to the CLI. The `setup` skill wires this up automatically based on your environment; you can also do it manually. See [Architecture](/docs/architecture) for how the tool layer maps onto the phases. -Run it **once** after installing the skills, before `ingest`, `translate`, `prepare`, or `migrate`. Just ask your agent: +### Configuring the MCP server for Databricks Genie Code + +Genie Code connects to a **hosted** MCP endpoint, so the flowx tools run as a Databricks App that you add in Genie Code's **Custom MCP server** picker. End to end: + + + +#### Clone flowx + +Follow the **Databricks Genie Code** install steps above to clone the repo and copy `skills/` into your skills folder. + + +The MCP app's service principal cannot read private `/Workspace/Users/` folders by default, and `deploy.sh` deploys the source from `/Workspace/Shared/`. Cloning into `/Workspace/Shared` keeps the repo, the deployed source, and team access all in a location every user and the app's service principal can reach. If your workspace restricts `/Workspace/Shared`, use any other folder all users (and the app service principal) can read and pass it via `APP_SOURCE_PATH`. + + + + +#### Run the setup skill + +Ask your agent to *"set up the flowx environment"* (or run `bash /scripts/bootstrap.sh`). On Databricks the `setup` skill detects the environment and, after creating the venv, runs the app deployment in the next step for you. Run it from a workspace web terminal if your Genie session can't shell out to the Databricks CLI. + + + +#### Deploy the MCP server + +```bash +bash /app/deploy.sh +``` + +`deploy.sh` stages a self-contained bundle (the app entrypoint plus a vendored copy of the flowx source), syncs it to **`/Workspace/Shared/mcp-flowx`** (a location the app's service principal can read — override with `APP_SOURCE_PATH`), and creates/deploys the **`mcp-flowx`** app. The script prints the app URL; the MCP endpoint is **`/mcp`**. + + +`databricks apps` deploy commands require a Databricks CLI session and must be run from the workspace web terminal or a local machine. + + + + +#### Grant access + +- **App access:** grant **Can use** on the `mcp-flowx` app to the users or service principals that will call it (Apps UI → *Permissions*, or `databricks apps set-permissions`). +- **Data access:** grant the app's own service principal access to the catalogs, schemas, and Unity Catalog volumes the migration reads from and writes to, plus any SQL warehouse used by the reporting commands (`flowx(command="record_results")` / `flowx(command="install_dashboard")`). + + + +#### Register the MCP server + +MCP servers are available in Genie Code [Agent mode](https://learn.microsoft.com/en-us/azure/databricks/genie-code/use-genie-code#modes). To add the flowx MCP server: + +1. In the Genie Code panel, click **⚙ Settings**. +2. Under **MCP Servers**, click **+ Add Server**. +3. Choose **Custom MCP server** and select the **`mcp-flowx`** Databricks App. +4. Click **Save**. + +The single `flowx` tool will be available when you use Genie Code in Agent mode. + + +Databricks requires a custom MCP app to be: +* Deployed in the same workspace +* Reachable at `https:///mcp` + +If Genie Code cannot connect to the flowx MCP server, set the app's `FLOWX_ALLOWED_ORIGINS` environment variable to your workspace URL and redeploy. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp). + + + + +#### Verify the MCP Server + +Open the health endpoint `/` (returns `{"status":"ok"}`), or ask Genie Code *"what flowx MCP tools are available?"*. You should see the single `flowx` tool. + + + +### Configuring the MCP server for other agent tools + +Deploy the MCP server locally to use flowx with other agent tools. Install the MCP server stack into a local Python virtual environment and run it over stdio: + +```bash +PY="$(cat /.migration-venv)" +"$PY" -m pip install "mcp>=1.12" "uvicorn>=0.30" "starlette>=0.40" +PYTHONPATH="/src" "$PY" -m flowx.mcp +``` + +Register the server with your MCP client (use the interpreter path from the marker file for `command`): + +```json +{ + "mcpServers": { + "flowx": { + "command": "/.venv/bin/python", + "args": ["-m", "flowx.mcp"], + "env": { "PYTHONPATH": "/src" } + } + } +} +``` + + +If you prefer an installed package over `PYTHONPATH`, run `pip install -e ".[mcp]"` from the plugin root; then `python -m flowx.mcp` works without setting `PYTHONPATH`. + + +## Running flowx as a Python process + +flowx's skills invoke Python modules that may depend on third-party packages. The `setup` skill provisions an isolated virtual environment with the required dependencies. + +Run it **once** after installing the skills, before `discover`, `convert`, `package`, or `migrate`. Just ask your agent: > Set up the flowx environment @@ -81,8 +184,9 @@ bash /scripts/bootstrap.sh Running the setup process will: 1. Check that `python3`, `pip`, and `venv` are available. -2. Create `/.venv` if it doesn't already exist. +2. Create the virtual environment if it doesn't already exist. When running under Databricks (Genie Code or notebooks, detected via `DATABRICKS_RUNTIME_VERSION`), the venv is created at `/Workspace/Users//.migration-skills`; everywhere else it is created at `/.venv`. 3. Install the `requirements.txt` dependencies into your virtual environment using `pip`. +4. Write the resolved interpreter path to the marker file `/.migration-venv`. The environment is created once and reused. Re-running the script simply confirms the venv exists and its dependencies are satisfied. @@ -95,14 +199,15 @@ To install Python in your environment, run one of the following commands: * **Windows:** [python.org/downloads](https://www.python.org/downloads/) (enable "Add python.exe to PATH") -After the venv exists, every Python command the skills run uses the venv interpreter with `src/` on `PYTHONPATH`: +After the venv exists, every Python command the skills run uses the interpreter recorded in the marker file, with `src/` on `PYTHONPATH`. Read the interpreter path from `/.migration-venv` rather than hardcoding it: ```bash export PYTHONPATH="/src" -"/.venv/bin/python" -m flowx.adapter inputs ingest +PY="$(cat /.migration-venv)" +"$PY" -m flowx.adapter inputs discover ``` -On Windows, the interpreter is `\.venv\Scripts\python.exe`. The agent normally runs these commands for you; they are handy for troubleshooting a `ModuleNotFoundError`. +The marker file points at `/.venv` locally or `/Workspace/Users//.migration-skills` on Databricks. The agent normally runs these commands for you; they are handy for troubleshooting a `ModuleNotFoundError`. ## Verifying the installation diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index 73219f4..d253728 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -1,8 +1,9 @@ { - "title": "Flowx", + "title": "flowx", "pages": [ "index", "how-it-works", + "architecture", "installation", "guide", "options", diff --git a/docs/content/docs/options.mdx b/docs/content/docs/options.mdx index 741d581..bbfa4f0 100644 --- a/docs/content/docs/options.mdx +++ b/docs/content/docs/options.mdx @@ -5,8 +5,8 @@ description: Control flowx's translation behavior and outputs import { Callout } from 'fumadocs-ui/components/callout'; -Flowx defers some architectural choices to allow users to specify properties of the output jobs. When options are available for controlling -translation, the agent session prompts the user for their preferences. +flowx defers some architectural choices to allow users to specify properties of the output jobs. When options are available for controlling +translation, the agent session prompts the user for their configuration. ## copy_activity_paradigm @@ -53,7 +53,7 @@ must use a [query-based connector](https://docs.databricks.com/aws/en/ingestion/ For every multi-activity motif the detector matches in a pipeline (`incremental_load_watermark`, `rest_api_pagination`, `metadata_driven_bulk_copy`, ...), flowx raises a per-motif -question with id `consolidate_motif:` so each detected pattern can be approved +option with id `consolidate_motif:` so each detected pattern can be approved or rejected independently. | Value | Default | Behavior | @@ -82,3 +82,46 @@ The following are required to consolidate into a single ingestion pipeline: - Access to query the file or database where metadata is stored or a CSV file containing the exported metadata - The number of metadata rows or objects must be less than 250 + +## Discover complexity report + +The `discover` phase emits `/metadata/profile_report.csv` (default `./flowx_output/metadata/profile_report.csv`), one row per pipeline with the following columns: + +| Column | Description | +|--------|-------------| +| `pipeline` | Pipeline name | +| `activities` | Total activity count | +| `datasets` | Number of referenced datasets | +| `linked_services` | Number of referenced linked services | +| `collapsible_patterns` | Number of detected collapsible (motif) patterns | +| `databricks_native_activities` | Count of Databricks-native activities (notebook/jar/python/job) | +| `control_flow_activities` | Count of control-flow / parameter-setting activities (ForEach/If/Switch/SetVariable/AppendVariable/Filter/Wait/Until) | +| `other_activities` | Count of all other activities (Copy/Web/Lookup/etc.) | +| `complexity_score` | Weighted score (see below) | +| `complexity_size` | T-shirt size (`S`/`M`/`L`/`XL`) derived from `complexity_score` | + +The weighted score is `sum(activity weights) + #datasets + #linked_services + #collapsible_patterns`, where activity weights are: + +- Databricks-native (notebook/jar/python/job) = **1** (simplest) +- control-flow / parameter-setting (ForEach/If/Switch/SetVariable/AppendVariable/Filter/Wait/Until) = **2** +- all other activities (Copy/Web/Lookup/etc.) = **3** (hardest) + +## Coverage results table & dashboard (Genie Code) + +When running with workspace auth (Genie Code, or a configured Databricks profile) the `package` +phase surfaces three optional inputs — `results_table`, `results_warehouse_id`, and +`install_dashboard` — to persist coverage and visualize it: + +- **`record-results`** writes one row **per pipeline per run** to the supplied Unity Catalog + table (`catalog.schema.table`), combining the complexity columns above with the + deterministic/agentic/unsupported coverage breakdown. Every row is stamped with a shared + **`run_id`** (UUID), **`run_date`** (`CURRENT_TIMESTAMP()`), and **`run_by`** + (`CURRENT_USER()`), so coverage is trackable across runs and users. +- **`install-dashboard`** creates and publishes an AI/BI (Lakeview) dashboard over that table — + KPI counters (pipelines, coverage %, deterministic/agentic/unsupported activity totals), a + pipelines-by-complexity bar chart, a coverage-over-runs line, and a per-pipeline coverage + table. + +The SQL warehouse is auto-detected (preferring a running serverless warehouse) when +`results_warehouse_id` is left blank. Both run via the Databricks SDK and degrade gracefully +when workspace auth or a warehouse is unavailable. diff --git a/pyproject.toml b/pyproject.toml index f5b9dac..057e8fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] -name = "flowx" -version = "0.2.0" +name = "databricks-flowx" +version = "0.1.0" description = "ADF to Databricks Lakeflow Jobs translator via Declarative Automation Bundles" readme = "README.md" requires-python = ">=3.12" @@ -26,6 +26,15 @@ dependencies = [ "sqlglot>=25.0", ] +[project.optional-dependencies] +# Install with `pip install -e .[mcp]` to host the flowx MCP tools +# (locally over stdio, or as a streamable-HTTP server / Databricks App). +mcp = [ + "mcp>=1.12", + "uvicorn>=0.30", + "starlette>=0.40", +] + [dependency-groups] dev = [ "pytest>=8.3.3,<9", @@ -50,6 +59,12 @@ python_version = "3.12" mypy_path = "src" exclude = ['venv', '.venv', 'tests/*'] +# Optional `mcp` extra (only installed for hosting the MCP server). Avoid +# missing-stub errors in dev environments that don't install it. +[[tool.mypy.overrides]] +module = ["mcp.*", "starlette.*", "uvicorn.*"] +ignore_missing_imports = true + [tool.pytest.ini_options] testpaths = ["tests"] addopts = "--no-header" diff --git a/requirements.txt b/requirements.txt index 060dab7..a439c35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ charset-normalizer==3.4.7 cryptography==48.0.0 # via google-auth databricks-sdk==0.110.0 - # via flowx + # via databricks-flowx google-auth==2.53.0 # via databricks-sdk idna==3.15 @@ -23,10 +23,10 @@ pyasn1-modules==0.4.2 pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' # via cffi pyyaml==6.0.3 - # via flowx + # via databricks-flowx requests==2.34.2 # via databricks-sdk sqlglot==30.8.0 - # via flowx + # via databricks-flowx urllib3==2.7.0 # via requests diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh old mode 100755 new mode 100644 index badb2ca..ead7187 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -2,16 +2,25 @@ # # Bootstraps a Python environment for the flowx plugin. # -# Creates a virtual environment at /.venv and installs the Python -# dependencies listed in requirements.txt using pip. +# Creates a virtual environment and installs the Python dependencies listed in +# requirements.txt using pip. The venv location depends on the environment: +# * Databricks (Genie Code / notebooks): /Workspace/Users//.migration-skills +# so the environment persists in the workspace and is shared across sessions. +# * Everywhere else: /.venv # # If python3, pip, or the venv module are unavailable, the script prints a clear # warning telling the user what to install and exits non-zero without making changes. # +# On Databricks serverless compute (where ensurepip is unavailable), the script +# falls back to --without-pip + get-pip.py automatically. +# # After bootstrapping, run the plugin's Python code with the venv interpreter and # src/ on PYTHONPATH, e.g.: # -# PYTHONPATH="/src" "/.venv/bin/python" -m flowx.adapter inputs ingest +# PYTHONPATH="/src" "/bin/python" -m flowx.adapter inputs discover +# +# The resolved interpreter path is also written to /.migration-venv +# so the skills can discover it without re-deriving the location. # set -euo pipefail @@ -33,7 +42,7 @@ if ! command -v python3 >/dev/null 2>&1; then cat >&2 <<'EOF' WARNING: python3 was not found on your PATH. -Flowx requires Python 3.12+ to run its translation code. +flowx requires Python 3.12+ to run its translation code. Please install Python (it bundles pip) before continuing: - macOS: brew install python (or https://www.python.org/downloads/) @@ -47,6 +56,43 @@ fi PYTHON_BIN="$(command -v python3)" +# On Databricks (Genie Code / notebooks), create the venv under the current +# user's workspace folder so it persists and is shared across sessions, rather +# than inside the (ephemeral) plugin checkout. Resolve the current user from the +# notebook runtime context, falling back to the workspace SDK. +if [ -n "${DATABRICKS_RUNTIME_VERSION:-}" ]; then + CURRENT_USER="$("$PYTHON_BIN" - <<'PYRESOLVE' 2>/dev/null || true +def _resolve(): + try: + from dbruntime.databricks_repl_context import get_context + ctx = get_context() + for attr in ("userName", "user"): + val = getattr(ctx, attr, None) + if val and "@" in str(val): + return str(val) + except Exception: + pass + try: + from databricks.sdk import WorkspaceClient + name = WorkspaceClient().current_user.me().user_name + if name: + return name + except Exception: + pass + return "" +print(_resolve()) +PYRESOLVE +)" + if [ -n "$CURRENT_USER" ]; then + VENV_DIR="/Workspace/Users/${CURRENT_USER}/.migration-skills" + echo "Databricks runtime detected; using workspace venv for ${CURRENT_USER}:" + echo " $VENV_DIR" + else + echo "Databricks runtime detected but current user could not be resolved;" >&2 + echo " falling back to plugin-local venv at $VENV_DIR" >&2 + fi +fi + # Verify pip is available if ! "$PYTHON_BIN" -m pip --version >/dev/null 2>&1; then cat >&2 <<'EOF' @@ -81,7 +127,20 @@ fi # Create the virtual environment if [ ! -x "$VENV_DIR/bin/python" ]; then echo "Creating virtual environment at $VENV_DIR ..." - "$PYTHON_BIN" -m venv "$VENV_DIR" + if "$PYTHON_BIN" -m venv "$VENV_DIR" 2>/dev/null; then + : # standard venv creation succeeded + else + # Fallback for environments where ensurepip is unavailable (e.g. Databricks + # serverless compute). Create the venv without pip, then bootstrap pip + # via get-pip.py. + echo "Standard venv failed (ensurepip likely missing); trying --without-pip ..." + rm -rf "$VENV_DIR" + "$PYTHON_BIN" -m venv --without-pip "$VENV_DIR" + echo "Bootstrapping pip via get-pip.py ..." + curl -sSL https://bootstrap.pypa.io/get-pip.py -o /tmp/_orchestra_get_pip.py + "$VENV_DIR/bin/python" /tmp/_orchestra_get_pip.py --quiet + rm -f /tmp/_orchestra_get_pip.py + fi else echo "Using existing virtual environment at $VENV_DIR ..." fi @@ -100,12 +159,47 @@ echo "Upgrading pip ..." echo "Installing dependencies from requirements.txt ..." "$VENV_PYTHON" -m pip install -r "$REQUIREMENTS" +# --------------------------------------------------------------------------- +# Databricks runtime: pre-configure workspace auth from the notebook context +# --------------------------------------------------------------------------- +# The venv Python does NOT have dbruntime (it's a system-only package), so +# workspace_downloader.py can't auto-configure auth at runtime. However, the +# system Python ($PYTHON_BIN) DOES have it. Extract host + token here once and +# write ~/.databrickscfg so all subsequent venv invocations find it immediately. +# --------------------------------------------------------------------------- +if [ -n "${DATABRICKS_RUNTIME_VERSION:-}" ]; then + CFG_PATH="${HOME}/.databrickscfg" + if [ -s "$CFG_PATH" ]; then + echo "Databricks auth already configured at $CFG_PATH" + else + echo "Databricks runtime detected; extracting workspace auth ..." + "$PYTHON_BIN" -c " +from dbruntime.databricks_repl_context import get_context +c = get_context() +host = 'https://' + c.browserHostName +token = c.apiToken +if not host or not token: + raise SystemExit('host/token unavailable from runtime context') +import pathlib +p = pathlib.Path('$CFG_PATH') +p.parent.mkdir(parents=True, exist_ok=True) +p.write_text(f'[DEFAULT]\nhost = {host}\ntoken = {token}\n') +print(f' -> {p} written (host={host})') +" 2>/dev/null && true || echo " -> skipped (runtime context unavailable)" + fi +fi + +# Record the resolved interpreter path so the skills can discover it without +# re-deriving the (environment-dependent) venv location. +echo "$VENV_PYTHON" > "$PLUGIN_ROOT/.migration-venv" + cat < + Translate parsed ADF pipeline AST into Databricks IR (intermediate representation). + Runs deterministic translators for known activity types, then invokes agentic skills + from adf-to-databricks-plugin for gaps. +triggers: + - "translate ADF" + - "convert ADF" + - "translate pipelines" + - "convert pipelines" + - "run translation" +--- + +# Convert ADF to Databricks IR + +Convert the parsed ADF inventory into Databricks intermediate representation (IR) using deterministic translators for known types and agentic fallback for unknown types. + +## Context + +This is phase 2 of the flowx migration workflow. It consumes the ADF source (profiled by the `discover` skill) and produces a translation report — a transient intermediate under `/.work/` — that the `package` skill uses to generate Databricks Declarative Automation Bundles. It shares the single migration `` with the other phases. + +The translation follows a **deterministic-first** strategy: +1. Activities with known, well-defined mappings are translated by built-in Python translators +2. Activities that require interpretation, expression conversion, or lack a Python translator are handled by agent skills from the `adf-to-databricks-plugin` + +## How to run this skill — MCP tools or venv CLI + +This phase runs one of two ways; run the **`setup`** skill first if you haven't. + +- **MCP tool (Databricks Genie Code, or a local stdio registration) — the only path in Genie Code:** + call the single **`flowx`** tool (one command per step) and run **no** `python3`/`$PY`/`bash` + commands. The `"$PY" -m …` snippets in the steps below are the **local-CLI fallback only** — ignore + them on this path. Map the steps to: + + ``` + flowx(command="convert", parameters={"output_dir": "", "pipeline": ""}) + # convert reuses the discovered output_dir on the server; only pass "adf_definitions" (inline ARM + # JSON) if you are converting without a prior discover on this server. + flowx(command="inspect", parameters={"report_path": "/.work/translation_report.json", "answers": [...]}) + flowx(command="apply_answers", parameters={"report_path": "...", "answers": ["id=value", ...], "output_dir": "", "lookup_csv": ""}) + flowx(command="merge_agentic", parameters={"report_path": "...", "agentic_results_dir": "", "output_path": ""}) + ``` + + Use the tool results in place of reading the files directly. `command="merge_agentic"` covers the + agentic `--merge-agentic` step shown later in this skill. + +- **venv CLI (local, no MCP server):** ensure the venv exists (`setup` Path B / `bootstrap.sh`), then + run the commands below with the venv interpreter (from the marker file `/.migration-venv`) + and `src/` on `PYTHONPATH` (use `$PY` anywhere a command shows `python3`): + + ```bash + export PYTHONPATH="/src" + PY="$(cat /.migration-venv)" + "$PY" -m flowx.adapter convert --output-dir + ``` + + If Python or pip is missing, `bootstrap.sh` prints a warning telling the user what to install — + relay it and stop until they have Python 3.12+ and pip. + +## Workflow + +Follow these steps in order: + +### Step 0 — Gather phase inputs + +Run the adapter inputs subcommand so the agent surfaces the free-text +options the phase needs (inventory path, ADF source dir, output +directory): + +```bash +"$PY" -m flowx.adapter inputs convert +``` + +The JSON response carries the prompts and defaults; collect answers from the user +(or fall back to the defaults). Keep them in conversation context — the same shared +`` is used by every phase. + +### Step 1 — Locate the inventory + +The discover phase wrote `/metadata/inventory.json` (and `profile_report.csv`). If the +shared `` is not already in conversation context, ask the user: + +> Which migration output directory did the discover phase use? (default: `./flowx_output`) + +Validate `/metadata/inventory.json` exists and is well-formed. + +### Step 2 — Run deterministic translation + +Execute the translation engine on all deterministic activities: + +```bash +# Unified runner (recommended): `"$PY" -m flowx.adapter convert ...` +# forwards to the engine below; --adf-source-path aliases --source-dir. +"$PY" -m flowx.translator.engine \ + --source-dir \ + --output-dir \ + [--pipeline ] +``` + +Where: +- `` is the original ADF JSON directory (the same `--source-dir` used by discover) +- `` is the **shared migration output directory** (default: `./flowx_output`) — the + same one discover used +- `` (optional) — when provided, translates only the named pipeline. **Always pass `--pipeline` when the user has specified a specific pipeline to migrate**, matching the value passed to the discover phase. + +The translation report and intermediate IR are written to the **transient** `/.work/` +folder (`translation_report.json`, per-pipeline IR, `gaps.json`). These are consumed by the steps +below and the package phase, then pruned — they are not kept artifacts. + +### Step 3 — Read the translation report + +Read `/.work/translation_report.json`. It has this structure: + +```json +{ + "inventory_path": "/path/to/inventory.json", + "generated_at": "2026-04-07T12:30:00Z", + "translations": [ + { + "pipeline": "ETL_Main", + "activity": "CopyFromBlob", + "type": "Copy", + "strategy": "deterministic", + "status": "translated", + "ir": { + "task_key": "copy_from_blob", + "task_type": "notebook_task", + "notebook_path": "notebooks/copy_from_blob.py", + "parameters": { "source": "abfss://...", "target": "..." } + } + }, + { + "pipeline": "ETL_Main", + "activity": "TransformData", + "type": "ExecuteDataFlow", + "strategy": "agentic", + "status": "pending", + "raw_activity_json": { "...": "..." }, + "target_skill": "adf-to-databricks:adf-dataflow-converter" + } + ], + "summary": { + "total": 47, + "deterministic_translated": 35, + "agentic_pending": 10, + "failed": 2 + } +} +``` + +### Step 4 — Handle agentic gaps + +For each translation with `"status": "pending"` and `"strategy": "agentic"`, invoke the appropriate skill from the `adf-to-databricks-plugin`. Route by activity type. + +Every agentic gap in the translation report carries the activity's **full ADF/ARM JSON** under `raw_activity_json` (engine field `raw_definition`), and the generated placeholder notebook embeds the same JSON in a fenced `json` block. This holds for nested activities too — an `Until` inside an `IfCondition` / `Switch` / `ForEach` is reported as its own gap. Always translate from this ARM JSON. + +**Until activities (agent-based handler):** +Databricks Lakeflow Jobs have no native repeat-until loop, so translate the `Until` from its ARM JSON into a single Python notebook task implementing a bounded polling loop. From the embedded JSON, read: +- `typeProperties.expression` — the ADF exit condition (e.g. `@or(equals(variables('jobStatus'),'succeeded'), equals(variables('jobStatus'),'failed'))`); convert it into the Python `while not ():` guard. +- `typeProperties.timeout` — wrap the loop in a wall-clock deadline (`time.monotonic()`), raising on timeout. +- `typeProperties.activities` — the loop body (e.g. a `Wait`, a polling `WebActivity`, a `SetVariable` that captures the next status); translate each child inline so the whole loop runs in one notebook. +Read the loop variables from `dbutils.widgets`, surface the final state as a task value, and write the result over the placeholder notebook's `raise NotImplementedError` cell. If the external `adf-to-databricks:adf-pipeline-converter` skill is installed you may delegate to it with the same ARM JSON; otherwise perform the translation directly. + +**ExecuteDataFlow activities:** +Invoke `adf-to-databricks:adf-dataflow-converter` with the raw activity JSON and associated data flow definition. Provide context: +- The raw `typeProperties` from the ADF activity +- The data flow JSON definition (if available in the source directory under `dataflow/`) +- The linked service configurations for source/sink connections +- Target catalog and schema for the SDP pipeline or PySpark notebook output + +**Control flow activities (Switch, Until, Wait, Filter, AppendVariable):** +Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: +- The full pipeline JSON containing the activity +- Any nested activities within the control flow +- Variable definitions from the pipeline +- The desired Databricks task type mapping + +**Stored procedures and external calls (SqlServerStoredProcedure, AzureFunction, WebHook, Custom):** +Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: +- The linked service configuration for the target system +- Connection details and authentication method +- Any parameters or request bodies + +**Complex expressions:** +If any activity (deterministic or agentic) contains ADF expressions that the deterministic translator could not resolve, invoke `adf-to-databricks:adf-expression-translator` with: +- The raw expression string (e.g., `@pipeline().parameters.inputPath`) +- The expression context (pipeline parameters, variables, activity outputs) +- The target format (Python f-string, Spark SQL, task parameter reference) + +**Trigger definitions:** +Invoke `adf-to-databricks:adf-trigger-converter` with: +- The trigger JSON definition +- The associated pipeline references +- Target: Databricks job schedule configuration (quartz_cron_expression, periodic, or file_arrival) + +### Step 5 — Collect agentic results + +Each resolved agentic gap produces one translation result. Write them into +`/agentic_results/` as one JSON file per activity (the filename is +arbitrary, e.g. `__.json`). Each file MUST use this schema: + +```json +{ + "activity_name": "", + "pipeline": "", + "task": { + "type": "NotebookActivity", + "name": "", + "task_key": "", + "notebook_path": "/Workspace/.../your_translated_notebook" + } +} +``` + +- `activity_name` (required) — matches the `name` of the placeholder task in the + report (the merge locates it by name, recursing into IfCondition / ForEach / + Switch containers, so nested gaps like an `Until` are found). +- `pipeline` (optional) — only needed to disambiguate multi-pipeline reports. +- `task` (required) — the replacement IR task. The most portable form is a + `NotebookActivity` whose `notebook_path` points at a notebook you have written + to the workspace; the package phase references it directly. `task_key` and + `depends_on` are inherited from the placeholder when omitted, so dependency + edges are preserved. + +### Step 6 — Merge agentic results + +Fold the results into the translation report (placeholders are replaced in place): + +```bash +"$PY" -m flowx.translator.engine \ + --merge-agentic \ + --report /.work/translation_report.json \ + --agentic-results +``` + +Equivalently via the unified runner: `"$PY" -m flowx.adapter convert --merge-agentic --report /.work/translation_report.json --agentic-results `. Add `--output ` to write a copy instead of overwriting the report. The command exits non-zero if any result could not be matched to a placeholder. + +This updates `/.work/translation_report.json` with the agentic results merged in, changing their status from `pending` to `translated` (or `failed` if the agentic skill could not produce a result). + +### Step 6.1 — Gather just-in-time translation configuration + +Run `inspect` **once** to get the full option schema, then drive the whole question chain yourself — +do **not** re-run `inspect` per follow-up: + +```bash +"$PY" -m flowx.adapter inspect /.work/translation_report.json +``` + +It returns every option the report can raise, each annotated with a `show_when` condition: + +```json +{"pipelines": [{"pipeline_name": "...", "options": [ + {"option_id": "notify_destination", "prompt": "...", "rationale": "...", + "choices": [{"value": "...", "label": "...", "description": "..."}], + "free_text": false, "default": "keep", "show_when": []}, + {"option_id": "notify_slack_url", "prompt": "...", "free_text": true, "default": "", + "show_when": [{"option_id": "notify_destination", "in": ["slack"]}]} +]}]} +``` + +Walk it locally: + +1. **Ask an option only when its `show_when` is satisfied** — every clause `{option_id, in:[values]}` + must match an answer you've already collected (empty `show_when` = always ask). So `notify_slack_url` + surfaces only after `notify_destination=slack`; the metadata-driven `access`/`size`/`lookup_tool` + chain surfaces only after `metadata_driven_consolidate=consolidate`, etc. Present each option's + `prompt`/`rationale` and `choices`; honor the `default`. +2. **Validate each answer** against `choices` (a `free_text` option — empty `choices` — accepts any + value; blank skips an optional one). +3. **Perform data actions inline** when an answer calls for it — e.g. when + `metadata_driven_lookup_tool=have`, run the lookup query with your database tool to get the rows. +4. When every applicable option is answered, apply them **in one `modify` call** (Step 6.2) with all + answers as `--answer OPTION_ID=VALUE` flags. `modify` validates every answer server-side. + +**Activity→Notify (`activity_and_notify`) motifs.** When **any** activity (Copy, +Notebook, Lookup, stored procedure, …) is followed by notification Web +activities, the adapter raises `notify_destination`: +`keep` (default) leaves the Web activities to translate directly — nothing is +collapsed. Any other value (`email`, `slack`, `teams`, `pagerduty`, `webhook`) +collapses the pattern: the upstream activity becomes the task and the +notifications become Databricks job-task `on_success`/`on_failure` notifications +routed to that destination (the ADF Web activity URL/body is not used). The schema includes **one +follow-up per Databricks-SDK field** of each destination, each gated by +`show_when: [{notify_destination, in:[]}]`; ask the chosen destination's fields (required +first) once the user picks it: + +| Destination | Chained field options (SDK arg) | +|-------------|---------------------------------| +| `email` | `notify_email_recipients` (`addresses`, comma-separated) | +| `slack` | `notify_slack_url` (`url`), `notify_slack_channel_id` (`channel_id`, optional), `notify_slack_oauth_token` (`oauth_token`, optional) | +| `teams` | `notify_teams_url` (`url`) | +| `pagerduty` | `notify_pagerduty_integration_key` (`integration_key`) | +| `webhook` | `notify_webhook_url` (`url`), `notify_webhook_username` (`username`, optional), `notify_webhook_password` (`password`, optional) | + +All destinations also take an optional `notify_destination_name` and +`notify_events` (both/on_failure/on_success). Optional fields left blank are +omitted so the SDK applies its defaults. For **non-email** destinations, the +`modify` phase creates (or reuses by display name) the Databricks notification +destination via the SDK **as soon as you submit the answers** — it validates the +config immediately and bakes the resolved destination id into the modified report, +so package just wires `webhook_notifications` to that id (no further SDK call). +This requires workspace auth at `modify` time; if creation fails there, the id is +left unresolved and package retries or emits a `notification_destination` setup task. +**Email** needs no destination — it uses raw `email_notifications` and is never +created via the SDK. + +When the metadata-driven flow ends with `metadata_driven_lookup_tool=have` +and the agent has a database tool (Genie, MCP SQL, or a workspace SDK), +run the lookup query directly to obtain the rows as CSV. When the answer is +`none`, ask the user for a CSV file path or a literal CSV string. Pass it inline +to `modify` via `--lookup-csv` (no intermediate JSON file): + +```bash +"$PY" -m flowx.adapter modify \ + /.work/translation_report.json \ + --output-dir \ + --answer metadata_driven_consolidate=consolidate \ + --answer metadata_driven_access=yes \ + --lookup-csv "" +``` + +When no metadata-driven motif is consolidated, `--lookup-csv` is omitted. In that default +(non-consolidated) case the motif becomes a Databricks **for-each task** that runs one Spark JDBC +read per source table — its iteration inputs are the resolved lookup rows when available, otherwise a +control-table lookup task seeds them at run time. (Consolidating instead emits one managed Lakeflow +Connect ingestion pipeline.) + +#### Legacy flow details + +Before writing the final report, surface any pipeline-modifier options the +IR raises (Copy Data paradigm, non-Databricks task compute, Lakeflow Connect +opt-in, Databricks task compute). Use the adapter CLI bridge: + +```bash +"$PY" -m flowx.adapter inspect /.work/translation_report.json +``` + +The command emits JSON: + +```json +{ + "pipelines": [ + { + "pipeline_name": "ETL_Main", + "options": [ + { + "option_id": "copy_activity_paradigm", + "prompt": "How should Copy Data activities targeting Delta be implemented?", + "rationale": "...", + "options": [{"value": "notebook", "label": "...", "description": "..."}, ...], + "affected_task_keys": ["copy_orders", "copy_customers"], + "default": "notebook" + }, + ... + ] + } + ] +} +``` + +For each option, prompt the user with the rationale, options, and the task keys it +affects. Use the default when the user defers. Then apply the collected answers as +`--answer OPTION_ID=VALUE` flags: + +```bash +"$PY" -m flowx.adapter modify \ + /.work/translation_report.json \ + --output-dir \ + --answer copy_activity_paradigm=sdp \ + --answer non_databricks_task_compute=serverless \ + --answer use_lakeflow_connectors=lakeflow_connect +``` + +`modify` writes two things under the shared ``: +- `.work/translation_report.stamped.json` — the configuration-stamped IR the package phase consumes +- `metadata/configuration.json` — the collected answers, kept as the migration's configuration record + +The package phase (next skill) reads the stamped report from `.work/` automatically. +When no options are raised, the inspect output is `{"pipelines": [{"pipeline_name": "...", "options": []},...]}` — skip `modify`; package falls back to the un-stamped report. + +### Step 7 — Present translation summary + +Display a summary to the user: + +``` +Translation Summary +=================== +Total activities: 47 +Deterministic translated: 35 (74.5%) +Agentic translated: 8 (17.0%) +Failed: 4 ( 8.5%) + +Overall coverage: 91.5% + +Failed translations: + - ETL_Main / RunSSIS (ExecuteSSISPackage) — no translator available + - ETL_Main / CustomTask (Custom) — agentic skill returned error + ... + +Generated artifacts (transient, under /.work/): + - translation_report.json + - per-pipeline IR (43 files) + - gaps.json +``` + +If coverage is below 100%, explain the options for failed translations: +1. Manual notebook creation for unsupported types +2. Retry agentic translation with additional context +3. Skip the activity and add a placeholder task in the DAB + +## Reference + +See `references/activity-mapping.md` for the complete mapping between ADF activity types and translation strategies. + +## Examples + +- "Convert the ADF pipelines" +- "Convert ADF to Databricks" +- "Run the translation on the inventory from the profile step" +- "Convert the parsed pipelines using deterministic + agentic" +- "Convert only the pl_demo_01 pipeline" + +## Output Artifacts + +The convert phase writes only **transient** intermediates, under `/.work/` (consumed +by `modify`/`package`, then pruned — not kept): + +| File | Description | +|---|---| +| `.work/translation_report.json` | Full translation report with IR for all activities | +| `.work/.json` | Per-pipeline Databricks IR | +| `.work/gaps.json` | Agentic gaps awaiting skill conversion | +| `.work/translation_report.stamped.json` | Configuration-stamped report (written by `modify`) | diff --git a/skills/translate/references/activity-mapping.md b/skills/flowx-convert/references/activity-mapping.md similarity index 99% rename from skills/translate/references/activity-mapping.md rename to skills/flowx-convert/references/activity-mapping.md index 137d980..9635220 100644 --- a/skills/translate/references/activity-mapping.md +++ b/skills/flowx-convert/references/activity-mapping.md @@ -163,7 +163,7 @@ The agentic approach trades speed for coverage — it can handle the long tail o ## Expression Function Coverage -Flowx deterministically translates 73 of 84 ADF expression functions to Python notebook code. The remaining 11 functions require agentic translation: +flowx deterministically translates 73 of 84 ADF expression functions to Python notebook code. The remaining 11 functions require agentic translation: - `dataUri`, `dataUriToBinary`, `dataUriToString`, `decodeDataUri` — Data URI encoding/decoding (rare in practice) - `uriComponentToBinary` — URI component to binary conversion diff --git a/skills/translate/references/expression-functions.md b/skills/flowx-convert/references/expression-functions.md similarity index 100% rename from skills/translate/references/expression-functions.md rename to skills/flowx-convert/references/expression-functions.md diff --git a/skills/flowx-discover/SKILL.md b/skills/flowx-discover/SKILL.md new file mode 100644 index 0000000..f33a992 --- /dev/null +++ b/skills/flowx-discover/SKILL.md @@ -0,0 +1,273 @@ +--- +name: flowx-discover +description: > + Load and parse Azure Data Factory pipeline definitions from Unity Catalog volumes or local directories. + Produces a typed inventory that classifies every activity as deterministic, agentic, or unsupported. +triggers: + - "discover ADF" + - "load ADF" + - "parse ADF" + - "import pipelines" + - "load pipelines" + - "parse pipelines" + - "inventory ADF" +--- + +# Discover ADF Pipeline Definitions + +Parse Azure Data Factory pipeline, dataset, linked service, and trigger JSON files into a typed AST and produce a classified inventory. + +## Context + +This is phase 1 of the flowx migration workflow. It takes raw ADF JSON exports and produces an `inventory.json` file that the `convert` skill consumes. The inventory classifies every ADF activity into one of three strategies: + +- **Deterministic** — a built-in translator exists (Copy, DatabricksNotebook, ForEach, IfCondition, etc.) +- **Agentic** — requires LLM-assisted translation via the `adf-to-databricks-plugin` skills (ExecuteDataFlow, Switch, Until, StoredProc, etc.) +- **Unsupported** — no known translation path; requires manual intervention + +## How to run this skill — MCP tool or venv CLI + +This phase runs one of two ways; run the **`setup`** skill first if you haven't. + +- **MCP tool (Databricks Genie Code, or a local stdio registration) — the only path in Genie Code:** + call the single **`flowx`** tool with `command="discover"` and run **no** `python3`/`$PY`/`bash` + commands. The `"$PY" -m …` snippets in the steps below are the **local-CLI fallback only** — ignore + them on this path. + + The hosted server **cannot read your workspace/volume files**, so pass the ADF JSON **inline** as + `adf_definitions` — a mapping of relative path → JSON content mirroring the ADF Git-export layout. + You (the agent) read the ARM JSON files from the source and supply them: + + ``` + flowx(command="discover", parameters={ + "adf_definitions": { + "pipeline/Foo.json": { ...ARM JSON... }, + "dataset/Bar.json": { ... }, + "linkedService/Baz.json": { ... }, + "trigger/Qux.json": { ... } + }, + "output_dir": "", "pipeline": ""}) + ``` + + For **large factories** (hundreds–thousands of pipelines), don't inline — reference the source + instead (inline `adf_definitions` is capped at ~5 MB): pass `"adf_volume_path": + "/Volumes/cat/sch/adf_export"` for a UC Volume (read via the SDK Files API) or + `"adf_workspace_path": "/Workspace/Shared/adf_export"` for an ADF Git folder in the workspace (read + via the SDK Workspace API). Locally, where the server can read + the path, you may instead pass `adf_source_path`. The tool returns the inventory summary + (pipeline/activity counts by strategy and coverage); use it in place of reading the files directly. + +- **venv CLI (local, no MCP server):** ensure the venv exists (`setup` Path B / `bootstrap.sh`), then + run the commands below with the venv interpreter and `src/` on `PYTHONPATH`. The interpreter path is + in the marker file `/.migration-venv` (use `$PY` anywhere a command shows `python3`): + + ```bash + export PYTHONPATH="/src" + PY="$(cat /.migration-venv)" + "$PY" -m flowx.adapter discover --adf-source-path --output-dir + ``` + + If Python or pip is missing, `bootstrap.sh` prints a warning telling the user what to install — + relay it and stop until they have Python 3.12+ and pip. + +## Workflow + +Follow these steps in order: + +### Step 1 — Determine the ADF source path + +Ask the user for the location of their ADF JSON exports. Accept either: +- A Unity Catalog volume path (e.g., `/Volumes/main/default/adf_export`) +- A local directory path (e.g., `./adf_export/` or `/tmp/adf_json/`) + +The directory should contain subdirectories or files for: +- `pipeline/` or `pipelines/` — pipeline definition JSON files +- `dataset/` or `datasets/` — dataset definition JSON files (optional) +- `linkedService/` or `linked_services/` — linked service JSON files (optional) +- `trigger/` or `triggers/` — trigger definition JSON files (optional) + +### Step 2 — Download from UC volumes if needed + +If the source path starts with `/Volumes/`, the files live in a Unity Catalog volume and must be downloaded to a local temp directory first. + +Use the `databricks-execution-compute` skill to run the following on the Databricks workspace: + +```python +import os, json, shutil, tempfile + +volume_path = "" +local_dir = tempfile.mkdtemp(prefix="adf_ingest_") + +# Copy from volume to local +for root, dirs, files in os.walk(volume_path): + for f in files: + if f.endswith(".json"): + src = os.path.join(root, f) + rel = os.path.relpath(src, volume_path) + dst = os.path.join(local_dir, rel) + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + +print(f"Downloaded ADF files to: {local_dir}") +``` + +Alternatively, use the Databricks CLI: +```bash +databricks fs cp -r "dbfs:" "" --overwrite +``` + +Set the working source directory to the local temp path for subsequent steps. + +### Step 3 — Run the deterministic parser + +Run the discover phase via the adapter's unified phase runner (recommended): + +```bash +"$PY" -m flowx.adapter discover \ + --adf-source-path \ + --output-dir \ + [--pipeline ] +``` + +`--adf-source-path` is accepted as an alias of `--source-dir` (it matches the +`adf_source_path` input option). This forwards to, and is equivalent to, running +the loader directly: + +```bash +"$PY" -m flowx.parser.adf_loader \ + --source-dir --output-dir [--pipeline ] +``` + +Where: +- `` is the root of the flowx plugin (the directory containing `src/`) +- `` is the local directory containing ADF JSON files +- `` is the **single shared migration output directory** used by all three phases + (default: `./flowx_output`). Discover writes its artifacts into the `metadata/` subfolder. +- `` (optional) — when provided, filters to only the named pipeline. When omitted, all pipelines in the source directory are included. + +**Always pass `--pipeline` when the user has specified a specific pipeline to migrate.** This ensures the inventory and all downstream phases are scoped to only that pipeline. + +This produces, under `/metadata/`: +- `inventory.json` — the classified activity inventory +- `profile_report.csv` — one row per pipeline with a complexity assessment (see Step 4b) +- `.arm.json` — the verbatim original ADF/ARM source for each pipeline (provenance) + +### Step 4 — Read and validate the inventory + +Read the generated `/metadata/inventory.json` file. It has this structure: + +```json +{ + "source_dir": "/path/to/adf/json", + "generated_at": "2026-04-07T12:00:00Z", + "pipelines": [ + { + "name": "PipelineName", + "file": "pipeline/PipelineName.json", + "activities": [ + { + "name": "CopyFromBlob", + "type": "Copy", + "strategy": "deterministic", + "translator": "copy.py" + }, + { + "name": "RunDataFlow", + "type": "ExecuteDataFlow", + "strategy": "agentic", + "skill": "adf-to-databricks:adf-dataflow-converter" + } + ] + } + ], + "summary": { + "pipeline_count": 12, + "activity_count": 47, + "deterministic_count": 35, + "agentic_count": 10, + "unsupported_count": 2, + "coverage_pct": 95.7 + } +} +``` + +### Step 4b — Review the complexity report + +`/metadata/profile_report.csv` carries one row per pipeline with a migration-complexity +assessment. Columns: + +| Column | Meaning | +|---|---| +| `pipeline` | Pipeline name | +| `activities` | Total activities (including nested ForEach/If/Switch children) | +| `datasets` | Distinct datasets the pipeline references | +| `linked_services` | Distinct linked services (activity-level + via referenced datasets) | +| `collapsible_patterns` | Number of motif patterns detected (auto-collapsible during convert) | +| `databricks_native_activities` | Notebook / SparkJar / SparkPython / Job activities (simplest) | +| `control_flow_activities` | ForEach / If / Switch / SetVariable / AppendVariable / Filter / Wait / Until | +| `other_activities` | Everything else — Copy, Web, Lookup, agentic types (hardest) | +| `complexity_score` | Weighted score: native×1 + control×2 + other×3 + datasets + linked_services + collapsible_patterns | +| `complexity_size` | T-shirt size from the score: **S** ≤5, **M** ≤15, **L** ≤30, **XL** >30 | + +Use it to set expectations: S/M pipelines are largely deterministic; L/XL pipelines (many "other" +activities, datasets, or linked services) warrant closer review and more agentic translation. + +### Step 5 — Present the summary + +Display a summary table to the user: + +``` +ADF Ingestion Summary +===================== +Pipelines parsed: 12 +Total activities: 47 + +Strategy Breakdown: + Deterministic: 35 (74.5%) + Agentic: 10 (21.3%) + Unsupported: 2 ( 4.3%) + +Coverage: 95.7% +``` + +### Step 6 — Detail agentic activities + +For activities classified as `agentic`, explain which skill from the `adf-to-databricks-plugin` will handle each: + +| Activity | Type | Handling Skill | +|---|---|---| +| RunDataFlow | ExecuteDataFlow | `adf-to-databricks:adf-dataflow-converter` | +| BranchLogic | Switch | `adf-to-databricks:adf-pipeline-converter` | +| ... | ... | ... | + +### Step 7 — Warn about unsupported activities + +For activities classified as `unsupported`, warn the user clearly: + +``` +WARNING: The following activities have no automated translation path: + - Pipeline "ETL_Main" / Activity "RunSSIS" (ExecuteSSISPackage) + Recommendation: Manual conversion to PySpark notebook required. +``` + +### Step 8 — Confirm output location + +Tell the user where the metadata files were written (`/metadata/`: inventory.json, profile_report.csv, and the per-pipeline `.arm.json`), summarise the complexity sizes, and confirm they can proceed to the `convert` phase using the same ``. + +## Examples + +- "Discover my ADF pipelines from /Volumes/main/default/adf_export" +- "Parse ADF definitions from ./tests/resources/json/" +- "Load the ADF pipeline JSON files and show me the inventory" +- "Import pipelines from /tmp/customer_adf_export" +- "Discover only the pl_demo_01 pipeline from /Volumes/main/default/adf_export" + +## Output Artifacts + +All under the shared `/metadata/` folder: + +| File | Description | +|---|---| +| `metadata/inventory.json` | Classified activity inventory for the convert phase | +| `metadata/profile_report.csv` | Per-pipeline complexity report (counts + T-shirt size) | +| `metadata/.arm.json` | Verbatim original ADF/ARM source for each pipeline | diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md new file mode 100644 index 0000000..9f895bb --- /dev/null +++ b/skills/flowx-migrate/SKILL.md @@ -0,0 +1,422 @@ +--- +name: flowx-migrate +description: > + End-to-end migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs. + Orchestrates discover, convert, and package phases in sequence. +triggers: + - "migrate ADF" + - "migrate pipelines" + - "ADF to Databricks" + - "migrate to Lakeflow" + - "ADF migration" + - "convert ADF to Lakeflow" + - "migrate data factory" +--- + +# End-to-End ADF to Databricks Migration + +Orchestrate the complete migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. This skill runs all three phases in sequence: discover, convert, package. + +## Context + +This is the top-level orchestration skill. It runs the full migration pipeline: + +1. **Discover** — Parse ADF JSON exports into a typed inventory +2. **Convert** — Convert ADF activities to Databricks IR (deterministic + agentic) +3. **Package** — Generate Databricks Declarative Automation Bundles for deployment + +Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. + +## How to run this skill — MCP tools or venv CLI + +This skill orchestrates all three phases. Run the **`setup`** skill first if you haven't. There are +two execution paths: + +### MCP tools (Databricks Genie Code, or a local stdio registration) + +In Genie Code this is the **only** path — the phases run on the deployed `mcp-flowx` app, so +there is **no venv, no `bootstrap.sh`, and no `.migration-venv`**. Run **no** `python3`/`$PY`/`bash` +commands on this path; the `"$PY" -m …` snippets in the steps below are the **local-CLI fallback +only**. Everything goes through the single **`flowx`** tool, one `command` per step: + +The hosted server **cannot read your workspace/volume files**, so pass the ADF JSON **inline** as +`adf_definitions` (a mapping of relative path → ARM JSON content mirroring the Git-export layout — +`pipeline/…`, `dataset/…`, `linkedService/…`, `trigger/…`). You read those files and supply them. +The **recommended Genie path is a single `migrate` call**, which avoids re-sending the payload per +phase: + +``` +flowx(command="migrate", parameters={ + "adf_definitions": {"pipeline/Foo.json": {...}, "linkedService/Bar.json": {...}, ...}, + "output_dir": ..., "catalog": ..., "schema": ..., "pipeline": ""}) +``` + +**`migrate` is interactive when configuration options exist.** After translating, if the pipeline +raises any configuration options (e.g. how to handle an `activity_and_notify` motif, metadata-driven +bulk-copy consolidation, non-Databricks task compute), it **does not package** — it returns the +**full option schema once**: `{"status": "needs_input", "pending_options": [{"pipeline_name", +"options": [{option_id, prompt, rationale, choices, free_text, default, show_when}, …]}, …], +"report_path", "output_dir"}`. You drive the whole chain locally — no per-answer round trip: + +1. **Ask an option only when its `show_when` is satisfied** — every clause `{option_id, in:[values]}` + must match an answer you've already collected (empty `show_when` = always ask). So + `notify_slack_url` surfaces only after `notify_destination=slack`; the metadata-driven + `access`/`size`/`lookup_tool` chain only after `metadata_driven_consolidate=consolidate`. Present + each option's `prompt`/`rationale`/`choices`; honor the `default`. +2. **Validate** each answer against `choices` (a `free_text` option accepts any value); collect picks + as `"option_id=value"` strings. Run any data action inline (e.g. the lookup query when + `metadata_driven_lookup_tool=have`). +3. When **every applicable option** is answered, call `migrate` **once** with the same parameters plus + `"answers": ["option_id=value", …]`. It applies them and packages (`"status": "completed"`). + +To accept all defaults and skip the prompts, pass `"interactive": false`. (Re-calling `migrate` with +`answers` reuses the existing report and skips re-running discover/convert.) + +> **Large factories (hundreds–thousands of pipelines): do not inline.** Inline `adf_definitions` +> passes through your context window and is capped (~5 MB). Instead point the server at the source by +> reference: either stage the ADF export to a **UC Volume** and pass +> `"adf_volume_path": "/Volumes/cat/sch/adf_export"` (read via the SDK Files API), or pass +> `"adf_workspace_path": "/Workspace/Shared/adf_export"` for an ADF Git folder already in the workspace +> (read via the SDK Workspace API). For output, pass `"output_volume_path": "/Volumes/cat/sch/dab"` +> **or** `"output_workspace_path": "/Workspace/Shared/dab"` so the generated bundle is written to that +> target via the SDK (returned as `bundle_uploaded` instead of inline `bundle`). Grant the +> `mcp-flowx` app's service principal read on the source and write on the output target. + +For step-by-step control, run the commands in order (the app reuses `output_dir` across calls, so +only `discover` needs `adf_definitions`): + +``` +flowx(command="inputs", parameters={"phase": "discover" | "convert" | "package"}) # learn each phase's inputs +flowx(command="discover", parameters={"adf_definitions": {...}, "output_dir": ..., "pipeline": ...}) +flowx(command="convert", parameters={"output_dir": ..., "pipeline": ...}) +flowx(command="merge_agentic", parameters={"report_path": ..., "agentic_results_dir": ..., "output_path": ...}) # if agentic results +flowx(command="inspect", parameters={"report_path": ...}) +flowx(command="apply_answers", parameters={"report_path": ..., "answers": [...], "output_dir": ...}) +flowx(command="package", parameters={"output_dir": ..., "catalog": ..., "schema": ...}) +flowx(command="record_results", parameters={...}) / flowx(command="install_dashboard", parameters={...}) +``` + +The server's `output_dir` is ephemeral and not reachable from your workspace, so **have `migrate`/ +`package` write the DAB to the target via the SDK** — pass `"output_volume_path": "/Volumes/…"` or +`"output_workspace_path": "/Workspace/…"` and the bundle is uploaded there (returned as +`bundle_uploaded`). Only when neither is set is the bundle returned inline as `bundle = {"files": +{relpath: text,…}, "truncated": [...]}` (small bundles), which you must then persist yourself. Either +way the user ends up with the DAB to validate and deploy. Each call returns a structured result +(summaries / file trees); use those in place of reading files. Wherever a step below shows `"$PY" -m flowx.adapter …`, call +`flowx(command="", parameters={...})` instead. + +> `databricks bundle validate` / `deploy` of the *generated* bundle is still a user-driven CLI step +> (web terminal / local / CI-CD); present the bundle for review. + +### venv CLI (local, no MCP server) + +Ensure the venv exists (`setup` Path B / `bootstrap.sh`), then run the commands below with the venv +interpreter (from the marker file `/.migration-venv`) and `src/` on `PYTHONPATH` (use +`$PY` anywhere a command shows `python3`): + +```bash +export PYTHONPATH="/src" +PY="$(cat /.migration-venv)" +"$PY" -m flowx.adapter inputs discover +``` + +If Python or pip is missing, `bootstrap.sh` prints a warning telling the user what to install — relay +it and stop until they have Python 3.12+ and pip. + +## Workflow + +Follow these steps in order: + +### Step 0 — Gather phase inputs via the adapter + +Before invoking discover, run the adapter inputs subcommand once per +phase so the agent surfaces the matching free-text prompts: + +```bash +"$PY" -m flowx.adapter inputs discover +"$PY" -m flowx.adapter inputs convert +"$PY" -m flowx.adapter inputs package +``` + +Each response carries the options for that phase plus their descriptions and +defaults. Collect answers from the user (or accept the defaults) and thread the +values into the downstream CLI calls. All phases share **one** migration +`` (default `./flowx_output`). + +### Step 1 — Gather inputs + +Ask the user for all required inputs upfront: + +| Parameter | Description | Required | Default | +|---|---|---|---| +| ADF source path | UC volume path or local directory with ADF JSON files | Yes | — | +| Output directory | Single shared root for all flowx output (bundle + `metadata/`) | No | `./flowx_output` | +| Target catalog | Unity Catalog catalog for tables/volumes | No | `main` | +| Target schema | Schema within the catalog | No | `default` | +| Bundle name | Name for the generated DABs project | No | derived from pipelines | + +Example prompt: + +> To migrate your ADF pipelines, I need: +> 1. Where are your ADF JSON exports? (UC volume path like `/Volumes/main/default/adf_export` or local directory) +> 2. Where should I write the output? (default: `./flowx_output/`) +> 3. What target catalog and schema? (default: `main.default`) + +### Step 2 — Phase 1: Discover + +Invoke the `flowx:flowx-discover` skill with the ADF source path and `--output-dir ` (the shared migration dir). Profile writes `/metadata/{inventory.json, profile_report.csv, .arm.json}`. + +Wait for discover to complete and present the inventory summary: + +``` +Phase 1: Discover — Complete +========================== +Pipelines parsed: 12 +Total activities: 47 + Deterministic: 35 (74.5%) + Agentic: 10 (21.3%) + Unsupported: 2 ( 4.3%) +Coverage: 95.7% +``` + +### Step 3 — Checkpoint: confirm proceed + +Ask the user to review the inventory and confirm before continuing: + +> The discover phase found 47 activities across 12 pipelines. 95.7% have a translation path (74.5% deterministic, 21.3% agentic). 2 activities are unsupported and will need manual handling. +> +> Proceed to the translation phase? (yes/no) + +If the user says no, explain the options: +- Re-run discover with a different source directory +- Review `/metadata/inventory.json` (and `profile_report.csv`) to understand unsupported activities and pipeline complexity +- Manually classify activities before proceeding + +If the user says yes, proceed to step 4. + +### Step 4 — Phase 2: Convert + +Invoke the `flowx:flowx-convert` skill with: +- ADF source dir: the original ADF source path (same `--source-dir` as discover) +- Output dir: the same shared `` (convert writes its report to `/.work/`) + +Wait for the translation to complete and present the summary: + +``` +Phase 2: Convert — Complete +============================= +Deterministic translated: 35 (74.5%) +Agentic translated: 8 (17.0%) +Failed: 4 ( 8.5%) +Overall coverage: 91.5% +``` + +### Step 5 — Present translation details + +Show the user: +1. What was translated deterministically (bulk — just counts by type) +2. What was translated via agentic skills (list each with the skill used) +3. What failed and why (list each with the failure reason) + +For failures, suggest: +- Manual notebook creation +- Retry with additional context +- Skip and add placeholder + +### Step 5.1 — Gather just-in-time translation configuration + +Run `inspect` **once** to get the full option schema (every option carries a `show_when` condition), +then drive the chain locally — ask an option only when its `show_when` clauses are all satisfied by +the answers collected so far; never re-run `inspect` per follow-up. When the user opts to consolidate +a metadata-driven motif and the agent has a database tool, run the lookup query to get CSV rows; +otherwise prompt the user for a CSV file path or literal CSV string. Apply everything in **one** +`modify` call (all `--answer OPTION_ID=VALUE` flags, plus `--lookup-csv ""` when needed — +no intermediate JSON file). + +#### Legacy flow details + +Before bundle generation, run the adapter inspect CLI on the translation +report to surface any pipeline-modifier options the IR raises: + +```bash +"$PY" -m flowx.adapter inspect /.work/translation_report.json +``` + +`inspect` returns the full option tree at once. Ask each option only when its `show_when` clauses +(`{option_id, in:[values]}`) are all satisfied by the answers collected so far (empty = always); +present its rationale, choices, and affected task keys. Then apply all collected answers in one +`modify` call as repeatable `--answer OPTION_ID=VALUE` flags: + +```bash +"$PY" -m flowx.adapter modify \ + /.work/translation_report.json \ + --output-dir \ + --answer copy_activity_paradigm=sdp \ + --answer non_databricks_task_compute=serverless \ + [--lookup-csv ""] +``` + +`modify` writes the stamped report to `/.work/translation_report.stamped.json` +and the kept answers record to `/metadata/configuration.json`. The package phase +reads the stamped report from `.work/` automatically. When inspect emits no options for any +pipeline, skip `modify` — package falls back to the un-stamped report. + +The options the adapter raises: + +| `option_id` | Allowed values | Default | +|---|---|---| +| `copy_activity_paradigm` | `notebook`, `sdp` | `notebook` | +| `non_databricks_task_compute` | `serverless`, `classic` | `serverless` | +| `use_lakeflow_connectors` | `existing`, `lakeflow_connect` | `existing` | +| `consolidate_motif:` | `keep`, `consolidate` | `keep` | + +DatabricksNotebook and DatabricksSparkPython tasks always inherit the cluster binding derived from +their source linked service. + +For each multi-activity motif the detector matches (rest_api_pagination, +incremental_load_watermark, metadata_driven_bulk_copy, ...) the adapter emits one +`consolidate_motif:` option. The user must explicitly opt in to `consolidate` +for each detected pattern. + +### Step 6 — Checkpoint: confirm proceed to bundle generation + +> Translation is 91.5% complete. 4 activities could not be translated automatically. +> Options: +> 1. Proceed to bundle generation (failed activities will get placeholder tasks) +> 2. Retry failed translations with more context +> 3. Stop here and review the translation report +> +> What would you like to do? + +### Step 6.5 — Detect workspace artifacts and authenticate + +Before invoking the package phase, run the adapter's +`workspace-paths` subcommand to detect any absolute workspace paths +the bundle would need to download: + +```bash +"$PY" -m flowx.adapter workspace-paths \ + /.work/translation_report.stamped.json \ + --source-dir +``` + +When the response carries `needs_auth: true`: + +1. Confirm the workspace host with the user, defaulting to the first + entry in `suggested_hosts` (extracted from the Databricks linked + services in the ADF export). +2. Run `databricks auth login --host ` interactively to set up + a local profile. +3. Pass `--profile ` to the prepare invocation in Step 7 so + flowx downloads the referenced notebooks and downloads them under + `bundle/src/notebooks/` with the task references rewritten to the + relative `../src/notebooks/...` paths. + +Skip this step entirely when `needs_auth` is `false`. + +### Step 7 — Phase 3: Package + +Invoke the `flowx:flowx-package` skill with: +- Output dir: the same shared `` — package reads the stamped report from + `/.work/` automatically (no report path needed) and writes the bundle here +- Catalog: user-specified or `main` +- Schema: user-specified or `default` + +Package prunes the transient `/.work/` after a successful build, leaving the +bundle (databricks.yml, resources/, src/, SETUP.md) plus the kept `metadata/` folder. + +### Step 7.5 — (Optional) Persist coverage results and install a dashboard + +When running with workspace auth (Genie Code or a configured profile), offer to record this +run's migration coverage to a Unity Catalog table and optionally install a coverage dashboard. +The `inputs package` prompts surface `results_table`, `results_warehouse_id`, and +`install_dashboard`. + +If the user supplies a `results_table`: + +```bash +"$PY" -m flowx.adapter record-results \ + --output-dir --results-table [--warehouse-id ] +``` + +Writes one row per pipeline (counts, complexity size, deterministic/agentic/unsupported +coverage), each stamped with a UUID `run_id`, `run_date` (`CURRENT_TIMESTAMP()`), and `run_by` +(`CURRENT_USER()`). If `install_dashboard = yes`: + +```bash +"$PY" -m flowx.adapter install-dashboard --results-table [--warehouse-id ] +``` + +Creates and publishes an AI/BI coverage dashboard over the table and prints its URL. Both +auto-detect a SQL warehouse when `--warehouse-id` is omitted and degrade gracefully without +workspace auth. See the `package` skill (Step 8) for details. + +### Step 8 — Present final summary + +Display the complete migration summary: + +``` +Migration Complete +================== + +Source: /Volumes/main/default/adf_export (12 ADF pipelines) +Output: ./flowx_output/ (bundle + metadata/) + +Coverage: + Total activities: 47 + Successfully translated: 43 (91.5%) + Placeholder tasks: 4 ( 8.5%) + +Generated Files (under ./flowx_output/): + databricks.yml + resources/ (3 job definitions) + src/notebooks/ (12 notebooks) + src/setup/ (3 setup scripts) + SETUP.md + metadata/ inventory.json, profile_report.csv, .arm.json, configuration.json + +Setup Required: + - Run setup/create_volumes.py to create UC volumes + - Run setup/create_secrets.py to configure secrets (review credentials first) + - Run setup/register_connections.py to register external connections + +Next Steps: + 1. cd ./flowx_output/ + 2. Review generated files, especially notebooks and setup scripts + 3. databricks bundle validate --target dev + 4. Run setup scripts on the target workspace + 5. databricks bundle deploy --target dev + 6. databricks bundle run --target dev + 7. Verify job output and promote to staging/prod +``` + +### Step 9 — Offer follow-up actions + +Ask if the user wants to: +1. Validate the bundle now (`databricks bundle validate`) +2. Deploy to dev (`databricks bundle deploy --target dev`) +3. Review specific generated files +4. Re-translate any failed activities +5. Export a migration report for documentation + +## Reference + +See `references/workflow.md` for a detailed description of the three-phase architecture. + +## Examples + +- "Migrate my ADF pipelines to Databricks" +- "Convert ADF to Lakeflow jobs" +- "ADF to Databricks migration from /Volumes/main/default/adf_export" +- "Migrate data factory pipelines to catalog analytics, schema bronze" +- "Run the full ADF migration workflow" + +## Output Artifacts + +All three phases write into a single shared `` (default `./flowx_output`): + +| Path | Phase | Contents | +|---|---|---| +| `metadata/` | Profile + Modify | `inventory.json`, `profile_report.csv`, `.arm.json`, `configuration.json` | +| `databricks.yml`, `resources/`, `src/`, `SETUP.md` | Package | The deployable DAB bundle | +| `.work/` | Convert/Modify (transient) | Translation report + IR; pruned by package | diff --git a/skills/migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md similarity index 93% rename from skills/migrate/references/workflow.md rename to skills/flowx-migrate/references/workflow.md index 93e997d..379a958 100644 --- a/skills/migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -1,16 +1,16 @@ -# Flowx Migration Workflow +# flowx Migration Workflow End-to-end architecture for migrating Azure Data Factory (ADF) pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles (DABs). ## Overview -Flowx follows a three-phase pipeline architecture. Each phase is independently runnable and produces artifacts consumed by the next phase. The design principle is **deterministic-first, agentic fallback**: well-known ADF patterns are translated by fast, reliable Python code, while complex or ambiguous patterns are handled by LLM-assisted skills. +flowx follows a three-phase pipeline architecture. Each phase is independently runnable and produces artifacts consumed by the next phase. The design principle is **deterministic-first, agentic fallback**: well-known ADF patterns are translated by fast, reliable Python code, while complex or ambiguous patterns are handled by LLM-assisted skills. ``` ADF JSON Exports | v - Phase 1: INGEST + Phase 1: PROFILE (parse + classify) | v @@ -35,9 +35,9 @@ ADF JSON Exports databricks bundle deploy ``` -## Phase 1: Ingest +## Phase 1: Discover -**Skill:** `flowx:ingest` +**Skill:** `flowx:flowx-discover` **Input:** Directory of ADF JSON export files (from ARM template export, Azure DevOps, or manual export) @@ -60,9 +60,9 @@ ADF JSON Exports - Datasets and linked services are parsed for context but not independently translated — they inform the activity translators. - Triggers are included in the inventory and translated in phase 2. -## Phase 2: Translate +## Phase 2: Convert -**Skill:** `flowx:translate` +**Skill:** `flowx:flowx-convert` **Input:** `inventory.json` from phase 1 + original ADF JSON files @@ -84,13 +84,13 @@ ADF JSON Exports **Key decisions:** - Deterministic translators run first because they are fast and reliable. Agentic skills are only invoked for gaps. -- The IR is an intermediate format that decouples translation from DABs generation. This allows the prepare phase to target different output formats in the future. +- The IR is an intermediate format that decouples translation from DABs generation. This allows the package phase to target different output formats in the future. - Each deterministic translator is a standalone Python module in `src/flowx/translator/activity_translators/`. Adding support for a new activity type means adding a new module. - Agentic results are saved separately before merging, so they can be inspected, retried, or manually overridden. -## Phase 3: Prepare +## Phase 3: Package -**Skill:** `flowx:prepare` +**Skill:** `flowx:flowx-package` **Input:** `translation_report.json` from phase 2 @@ -140,7 +140,7 @@ The Databricks IR (intermediate representation) sits between ADF semantics and D - **Semantic mapping** — translating ADF concepts to Databricks concepts - **Serialization** — writing DABs YAML and notebooks -This means the prepare phase could target different output formats (Terraform, raw API calls, etc.) without changing the translation logic. +This means the package phase could target different output formats (Terraform, raw API calls, etc.) without changing the translation logic. ## ADF Concepts to Databricks Mapping @@ -164,4 +164,4 @@ This means the prepare phase could target different output formats (Terraform, r - **Parse errors** — logged to `parse_errors.json`, skipped in inventory - **Translation failures** — marked as `failed` in translation report, get placeholder tasks in DABs - **Agentic failures** — saved with error details, can be retried with additional context -- **Unsupported activities** — warned at ingest, get placeholder tasks with TODO comments in DABs +- **Unsupported activities** — warned at discover, get placeholder tasks with TODO comments in DABs diff --git a/skills/flowx-package/SKILL.md b/skills/flowx-package/SKILL.md new file mode 100644 index 0000000..f22769e --- /dev/null +++ b/skills/flowx-package/SKILL.md @@ -0,0 +1,344 @@ +--- +name: flowx-package +description: > + Generate Databricks Declarative Automation Bundles (DABs) from translated IR, + including job definitions, notebooks, and setup scripts. +triggers: + - "package bundles" + - "generate DABs" + - "create bundles" + - "package deployment" + - "generate bundles" + - "build DABs" +--- + +# Package Databricks Declarative Automation Bundles + +Generate deployment-ready Databricks Declarative Automation Bundles (DABs) from the translated intermediate representation, including job definitions, notebooks, and infrastructure setup scripts. + +## Context + +This is phase 3 of the flowx migration workflow. It consumes the `translation_report.json` produced by the `convert` skill and generates a complete DABs project that can be validated and deployed with the Databricks CLI. + +The output is a standard DABs project with: +- `databricks.yml` — the bundle configuration +- `resources/` — job and pipeline YAML definitions +- `src/notebooks/` — generated and helper notebooks +- `setup/` — infrastructure setup scripts (volumes, secrets, connections) + +## How to run this skill — MCP tools or venv CLI + +This phase runs one of two ways; run the **`setup`** skill first if you haven't. + +- **MCP tool (Databricks Genie Code, or a local stdio registration) — the only path in Genie Code:** + call the single **`flowx`** tool (one command per step) and run **no** `python3`/`$PY`/`bash` + commands. The `"$PY" -m …` snippets in the steps below are the **local-CLI fallback only** — ignore + them on this path. Map the steps to: + + ``` + flowx(command="package", parameters={"output_dir": "", "report_path": "", "catalog": "", + "schema": "", "bundle_name": "", "profile": "", + "download_workspace_files": true, + "output_volume_path": "", + "output_workspace_path": ""}) + flowx(command="workspace_paths", parameters={"report_path": "...", "source_dir": ""}) + flowx(command="record_results", parameters={"output_dir": "", "results_table": "catalog.schema.table", "warehouse_id": ""}) + flowx(command="install_dashboard", parameters={"results_table": "catalog.schema.table", "warehouse_id": ""}) + ``` + + The server's `output_dir` is ephemeral and not reachable from your workspace, so **have `package` + write the bundle to the target itself via the SDK** — don't try to persist files yourself. Pass one + of: + - `"output_volume_path": "/Volumes/cat/sch/dab"` — uploads the DAB to that UC Volume (SDK Files API). + - `"output_workspace_path": "/Workspace/Shared/dab"` — uploads it to that workspace folder (SDK + Workspace API; files written verbatim, not imported as notebooks). + + Either returns `bundle_uploaded = {"output_volume_path"|"output_workspace_path", "files", "count"}`. + Only when **neither** is given does `package` return the contents inline as + `bundle = {"files": {relpath: text, …}, "truncated": [...]}` (small bundles only) for you to persist. + Prefer an output path so the bundle lands durably and large bundles aren't capped. Skip the + `"$PY" -m …` commands below. + +- **venv CLI (local, no MCP server):** ensure the venv exists (`setup` Path B / `bootstrap.sh`), then + run the commands below with the venv interpreter (from the marker file `/.migration-venv`) + and `src/` on `PYTHONPATH` (use `$PY` anywhere a command shows `python3`): + + ```bash + export PYTHONPATH="/src" + PY="$(cat /.migration-venv)" + "$PY" -m flowx.adapter package --output-dir --catalog --schema + ``` + + If Python or pip is missing, `bootstrap.sh` prints a warning telling the user what to install — + relay it and stop until they have Python 3.12+ and pip. + +## Workflow + +Follow these steps in order: + +### Step 1 — Locate the translation report + +The translate/modify phases left the stamped report at `/.work/translation_report.stamped.json` +(or `/.work/translation_report.json` when `modify` was not run). If the shared +`` is not in conversation context, ask the user: + +> Which migration output directory should I build the bundle in? (default: `./flowx_output`) + +`package` reads the report from `/.work/` automatically — you do not pass a report path. +Validate that a report exists there and that all required translations have status `translated`. + +### Step 2 — Gather deployment parameters + +Ask the user for the following (provide defaults): + +| Parameter | Description | Default | +|---|---|---| +| Target catalog | Unity Catalog catalog for tables/volumes | `main` | +| Target schema | Schema within the catalog | `default` | +| Output directory | Shared migration dir; bundle + `metadata/` are written here | `./flowx_output` | +| Bundle name | Name for the DABs project | derived from first pipeline name | +| Target environments | Deployment targets to configure | `dev, staging, prod` | +| Warehouse ID | SQL warehouse for SQL tasks (optional) | prompt if SQL tasks exist | +| Databricks CLI profile | Profile used to download workspace-resident notebooks / JARs / Python files (`--profile`). Required only when the bundle references absolute workspace paths. | resolved from `~/.databrickscfg` (auto-prompt if multiple) | + +### Step 2.5 — Detect workspace artifacts and authenticate + +> **Databricks runtime (serverless / cluster):** Authentication is auto-configured +> from the notebook runtime context. The `workspace_downloader` module detects +> `DATABRICKS_RUNTIME_VERSION` in the environment and writes `~/.databrickscfg` +> from `dbruntime.databricks_repl_context` automatically. You can skip the +> interactive `databricks auth login` step — just pass `--profile DEFAULT` (or +> omit `--profile` entirely) and notebook downloading will work. + +Before running the bundle writer, check whether the report references +absolute workspace paths (notebooks under `/Shared/`, SparkPython +files, SparkJar libraries) or DBFS paths that the bundle should +download to be self-contained: + +```bash +"$PY" -m flowx.adapter workspace-paths \ + /.work/translation_report.stamped.json \ + --source-dir +``` + +The command emits: + +```json +{ + "paths": ["/Shared/team/notebook_a", "/Shared/team/notebook_b"], + "suggested_hosts": ["https://adb-1234.5.azuredatabricks.net"], + "needs_auth": true +} +``` + +When `needs_auth` is `true`: + +1. Surface the suggested hosts to the user with `AskUserOption`. Use + the first `suggested_hosts` value as the default; allow the user to + override. When no host is suggested (no Databricks linked service + in the export), prompt for the host with no default. +2. Run the interactive Databricks CLI login command and wait for it to + complete: + + ```bash + databricks auth login --host + ``` + + This writes a profile into `~/.databrickscfg`. When the user has + chosen a specific profile name, append `--profile ` to both + the login and the package invocation below. + +3. Pass the resolved profile to step 3 via `--profile ` (default + profile name is `DEFAULT`). When `needs_auth` is `false` skip steps + 1–2 and omit `--profile` from step 3. + +The `paths` list is informational; you can echo it to the user so they +know which notebooks the bundle will download. + +### Step 3 — Run bundle generation + +Execute the DAB writer: + +```bash +# Unified runner (recommended): `"$PY" -m flowx.adapter package ...` +# forwards to dab_writer below. +"$PY" -m flowx.bundler.dab_writer \ + --output-dir \ + --catalog \ + --schema \ + --bundle-name \ + [--profile ] \ + [--no-download-workspace-files] \ + [--keep-intermediates] +``` + +Where: +- `` is the shared migration directory — `package` defaults `--report` to + `/.work/translation_report.stamped.json` (falling back to the un-stamped report). + Pass `--report ` only to override. +- Other parameters are from step 2 +- After a successful build, `package` **prunes the transient `/.work/`** so the + final tree contains only the bundle and the kept `metadata/` files. Pass `--keep-intermediates` + to retain `.work/` for debugging. + +**Workspace artifact downloading (default: enabled).** When the report references workspace-resident notebooks (`/Shared/...`), DBFS Spark JARs (`dbfs:/...`), or Spark Python files, the preparer downloads them via the Databricks CLI auth so the resulting bundle is self-contained and deployable across environments. Downloaded notebooks are downloaded under `src/notebooks/` and bound to the default `job_cluster` (since they may rely on classic-compute features). The original `notebook_path` in the resource YAML is rewritten to the bundle-relative path `../src/notebooks/.py`. + +If no Databricks CLI auth is detected on the host (`~/.databrickscfg` empty AND no `DATABRICKS_CONFIG_PROFILE` / `DATABRICKS_HOST`+`DATABRICKS_TOKEN` env vars), the CLI prints the workspace paths it was about to download and prompts: + +``` +Workspace downloads are enabled but no Databricks CLI auth was found. + Looked for profiles in: /Users//.databrickscfg + Artifacts to download: /Shared/ETL/transform, … + +To authenticate, run one of: + databricks auth login --host https://.cloud.databricks.com + databricks configure --token + +Continue with placeholders (downloads will be skipped)? [y/N]: +``` + +Answering `n` aborts with exit code 2 so the user can authenticate and re-run. Answering `y` continues with placeholder notebooks (legacy in-place workspace paths). In non-interactive sessions the prompt defaults to placeholders. + +Use `--no-download-workspace-files` to opt out entirely; the bundle then keeps original workspace paths exactly as in the IR. + +### Step 4 — Present the generated file tree + +Show the user what was generated: + +``` +/ # the shared migration directory + databricks.yml + resources/ + etl_main_job.yml + transform_dlt_pipeline.yml + src/ + notebooks/ + copy_from_blob.py + web_activity_call.py + setup/ + create_volumes.py + create_secrets.py + register_connections.py + SETUP.md + metadata/ # kept migration metadata (from discover + modify) + inventory.json + profile_report.csv + .arm.json # verbatim original ADF/ARM source + configuration.json # the collected configuration answers + # .work/ (transient translation report + IR) is pruned after a successful build +``` + +### Step 5 — Explain setup tasks + +If the `setup/` directory was generated, explain what each script does: + +**create_volumes.py** — Creates Unity Catalog volumes required by the migrated jobs. These volumes replace Azure Blob Storage or ADLS references from ADF. Run this once per environment. + +**create_secrets.py** — Creates Databricks secret scopes and secrets for connection credentials that were in ADF linked services. Review the secret values and populate them manually or via your secrets management system. + +**register_connections.py** — Registers Unity Catalog connections for external data sources (SQL Server, REST APIs, etc.) that were referenced in ADF linked services. + +Emphasize that the user should review these scripts before running them, especially `create_secrets.py` which will need actual credential values. + +### Step 6 — Explain the generated bundle structure + +Briefly describe: +- **databricks.yml** — The root bundle config with workspace, target environments (dev/staging/prod), and variable definitions. Variables are parameterized for environment-specific values (catalog, schema, warehouse). +- **resources/*.yml** — One YAML file per Databricks Lakeflow Job (one per ADF pipeline). Each job contains tasks mapped from ADF activities, with dependencies matching the original ADF dependency chains. +- **src/notebooks/*.py** — Python notebooks for activities that translate to notebook_task. These contain the actual data movement or transformation logic. +- **tests/*.py** — Skeleton test files for validating the migrated jobs. + +### Step 7 — Suggest next steps + +Present the following next steps: + +``` +Next Steps +========== +1. Review generated files: + cd + cat databricks.yml + +2. Validate the bundle: + databricks bundle validate --target dev + +3. Run setup scripts (if generated): + databricks bundle run setup_volumes --target dev + +4. Deploy to dev: + databricks bundle deploy --target dev + +5. Test the deployed jobs: + databricks bundle run --target dev + +6. Promote to staging/prod: + databricks bundle deploy --target staging + databricks bundle deploy --target prod +``` + +Recommend running `databricks bundle validate` first to catch any configuration issues before deployment. + +### Step 8 — (Optional) Persist coverage results and install a dashboard + +This step only applies when running with workspace auth (Genie Code, or a configured +Databricks CLI profile). The `inputs package` options surface three optional prompts: +`results_table`, `results_warehouse_id`, and `install_dashboard`. + +**Persist results.** When the user provides a `results_table` (a UC `catalog.schema.table`), +write one migration-coverage row **per pipeline** for this run: + +```bash +"$PY" -m flowx.adapter record-results \ + --output-dir \ + --results-table \ + [--warehouse-id ] +``` + +It reads `/metadata/{inventory.json, profile_report.csv}`, creates the table if +needed, and inserts a row per pipeline with the activity/dataset/linked-service counts, +collapsible-pattern count, complexity size, and the deterministic/agentic/unsupported coverage +breakdown. Every row is stamped with a shared **`run_id`** (UUID for this run), +**`run_date`** (`CURRENT_TIMESTAMP()`), and **`run_by`** (`CURRENT_USER()`). The warehouse is +auto-detected (prefers a running serverless warehouse) when `--warehouse-id` is omitted. The +command prints the `run_id` and row count. + +**Install the dashboard.** When the user answers `install_dashboard = yes`, create and publish +an AI/BI (Lakeview) coverage dashboard over that table: + +```bash +"$PY" -m flowx.adapter install-dashboard \ + --results-table \ + [--warehouse-id ] \ + [--dashboard-name ""] [--parent-path "/Workspace/Users/"] +``` + +It builds the dashboard from a template (KPI counters for pipelines / coverage % / +deterministic-agentic-unsupported activity totals, a complexity-size bar chart, a +coverage-over-runs line, and a per-pipeline coverage table), publishes it, and prints the URL. +Both commands degrade gracefully with an actionable message when workspace auth or a warehouse +is unavailable. + +## Examples + +- "Package the bundles" +- "Generate DABs for the translated pipelines" +- "Create deployment bundles targeting catalog 'analytics' and schema 'bronze'" +- "Build the DABs project in ./output/my_migration/" + +## Output Artifacts + +All under the shared ``: + +| File | Description | +|---|---| +| `databricks.yml` | Root bundle configuration | +| `resources/*.yml` | Job and pipeline YAML definitions | +| `src/notebooks/*.py` | Generated notebooks | +| `src/setup/*.py` | Infrastructure setup scripts | +| `SETUP.md` | Human-readable setup instructions | +| `metadata/inventory.json` | Activity inventory (from discover) | +| `metadata/profile_report.csv` | Per-pipeline complexity report (from profile) | +| `metadata/.arm.json` | Verbatim original ADF/ARM source (from discover) | +| `metadata/configuration.json` | Collected configuration answers (from modify) | + +> **Notification destinations.** When a `activity_and_notify` motif was opted into a Slack/Teams/PagerDuty/Generic-Webhook destination, the destination is created (or reused by display name) via the SDK at **prompt time** (the `modify` phase), and its resolved id is carried in the report; package simply wires that id into the task's `webhook_notifications`. If the report has no pre-resolved id (creation was deferred or failed earlier), package retries the create; failing that — e.g. no workspace auth — a `notification_destination` setup task is emitted in SETUP.md instead and the task ships without notifications. Email destinations use raw `email_notifications` and never create an SDK destination. diff --git a/skills/flowx-setup/SKILL.md b/skills/flowx-setup/SKILL.md new file mode 100644 index 0000000..f319580 --- /dev/null +++ b/skills/flowx-setup/SKILL.md @@ -0,0 +1,172 @@ +--- +name: flowx-setup +description: > + Prepare flowx to run its phases (discover, convert, package, migrate). In Databricks + Genie Code this deploys the phases as an MCP server (a Databricks App) and creates NO virtual + environment — all code runs through MCP. Everywhere else it provisions a Python virtual + environment for the CLI skills, and optionally a local (stdio) MCP server. Run this once before + any other flowx skill, or whenever the environment is missing. +triggers: + - "setup flowx" + - "bootstrap flowx" + - "install flowx dependencies" + - "flowx environment" + - "create flowx venv" + - "ModuleNotFoundError flowx" + - "install flowx mcp" + - "deploy flowx mcp" +--- + +# Set up flowx + +flowx runs its phases (`discover`, `convert`, `package`, `migrate`) in one of two ways. This +skill prepares whichever fits your environment, keyed on `DATABRICKS_RUNTIME_VERSION` (the same +signal the rest of the plugin uses to detect Databricks): + +- **Databricks Genie Code** (`DATABRICKS_RUNTIME_VERSION` *set*) → the phases run as **MCP tools** + hosted on a Databricks App. **No virtual environment is created** — the app vendors its own copy + of the flowx code and dependencies, so the phase skills just call the single `flowx` MCP tool. +- **Local / Claude Code / other agents** (`DATABRICKS_RUNTIME_VERSION` *unset*) → the phases run + from a **Python virtual environment** via the CLI, with an optional local (stdio) MCP server. + +## Step 1 — Pick the path + +```bash +if [ -n "${DATABRICKS_RUNTIME_VERSION:-}" ]; then + echo "Databricks / Genie Code → deploy the MCP server (Path A, no venv)" +else + echo "Local → create the virtual environment (Path B)" +fi +``` + +--- + +## Path A — Databricks Genie Code (MCP, no virtual environment) + +In Genie Code the phases run on the deployed app, so **do not run `bootstrap.sh` and do not create a +venv** — it isn't needed. Deploy the MCP server instead: + +```bash +bash /app/deploy.sh +``` + +`app/deploy.sh` stages a self-contained bundle (the app entrypoint plus a vendored copy of the +flowx source), syncs it to **`/Workspace/Shared/mcp-flowx`**, and creates/deploys the +**`mcp-flowx`** Databricks App. The script prints the app URL; the MCP endpoint is +`/mcp`. It only needs the Databricks CLI and a system `python3` (for parsing CLI output) — +**not** an flowx venv. + +> **Clone into a shared location.** The app's service principal cannot read private +> `/Workspace/Users/` folders by default, so `deploy.sh` deploys the source from +> `/Workspace/Shared/`. Clone flowx into a Git folder under **`/Workspace/Shared`** +> (e.g. `/Workspace/Shared/flowx`), not your user home. If `/Workspace/Shared` is restricted, +> use another all-users location and pass it via `APP_SOURCE_PATH`. + +After it deploys, relay these follow-up steps to the user (the script also prints them): + +1. **App access:** grant **Can use** on `mcp-flowx` to the users / service principals that will + call it (Apps UI → *Permissions*, or `databricks apps set-permissions mcp-flowx ...`). +2. **Data access:** grant the app's service principal access to the catalogs, schemas, and volumes + the migration touches (plus any SQL warehouse used by the reporting tools). +3. **Add it in Genie Code (Agent mode):** open Genie Code **Settings → MCP Servers → Add Server**, + choose **Custom MCP server**, select the `mcp-flowx` app, and **Save**. The single + `flowx` tool then appears (MCP needs Agent mode; it uses one of the 20 tool slots). + Verify via the health endpoint `/`. + +Once added, the `discover`, `convert`, `package`, and `migrate` skills run **entirely through the +`flowx` MCP tool** (`flowx(command="…", parameters={…})`) — there is no venv, no +`bootstrap.sh`, and no `.migration-venv` marker on this path. + +> **Note:** `databricks apps` deploy commands require a Databricks CLI session (workspace web +> terminal or a local machine), not serverless notebook Python. If the Genie session can't shell +> out to the CLI, run `app/deploy.sh` from the web terminal. (Same constraint as `databricks bundle +> deploy`.) If you see `Error: please specify target`, the CLI attached to a stray `databricks.yml`; +> `deploy.sh` already isolates against this, so re-run it as-is. + +--- + +## Path B — Local / Claude Code (virtual environment) + +The flowx code in `src/flowx/` depends on third-party packages (`pyyaml`, `databricks-sdk`, +`sqlglot`); running it against a bare system Python fails with `ModuleNotFoundError`. Provision an +isolated venv (created once and reused). + +### Step B1 — Run the bootstrap script + +```bash +bash /scripts/bootstrap.sh +``` + +Where `` is the flowx plugin root (the directory containing `src/`, `skills/`, and +`requirements.txt`). The script will: + +1. Check that `python3`, `pip`, and the `venv` module are available. +2. Create the venv at `/.venv`. +3. Install `requirements.txt` into that venv using `pip`. +4. Write the resolved interpreter path to `/.migration-venv` for the other skills. + +### Step B2 — Handle a missing Python or pip + +If Python, pip, or the `venv` module are **not** available, the script prints a `WARNING:` block and +exits non-zero **without** creating anything. Do **not** work around it — relay the warning and stop: + +> ⚠️ Python must be installed before I can set up the flowx environment. +> +> * On macOS: `brew install python`. +> * On Debian/Ubuntu: `sudo apt-get install python3 python3-venv python3-pip`. +> +> Let me know once it's installed and I'll re-run setup. + +Re-run this setup skill after the user confirms Python and pip are installed. + +### Step B3 — Confirm success and how to run Python code + +On success, the script writes the interpreter path to `/.migration-venv`. The phase +skills run Python with that interpreter and `src/` on `PYTHONPATH`. Resolve it from the marker file: + +```bash +export PYTHONPATH="/src" +PY="$(cat /.migration-venv)" +"$PY" -m flowx.adapter inputs discover +``` + +`$PY` resolves to `/.venv/bin/python` (on Windows, `\.venv\Scripts\python.exe`). + +### Step B4 — (Optional) Run the MCP server locally + +To drive the phases through MCP tools locally (instead of the CLI), install the MCP server stack into +the venv and register the stdio server with your MCP client: + +```bash +PY="$(cat /.migration-venv)" +"$PY" -m pip install "mcp>=1.12" "uvicorn>=0.30" "starlette>=0.40" +PYTHONPATH="/src" "$PY" -m flowx.mcp # stdio (default) +``` + +```json +{ + "mcpServers": { + "flowx": { + "command": "", + "args": ["-m", "flowx.mcp"], + "env": { "PYTHONPATH": "/src" } + } + } +} +``` + +## Output + +| Path | Artifact | Description | +|---|---|---| +| A (Genie Code) | `mcp-flowx` Databricks App | Hosts the phases as the single `flowx` MCP tool at `/mcp`; **no venv** is created | +| B (local) | venv at `/.venv` | Virtual environment with the installed dependencies | +| B (local) | `/.migration-venv` | Marker file holding the resolved interpreter path | +| B (local, optional) | local MCP server | `mcp` / `uvicorn` / `starlette` installed into the venv; run with `python -m flowx.mcp` | + +## Examples + +- "Set up flowx" (auto-detects Genie Code vs. local) +- "Deploy the flowx MCP server to Databricks / Genie Code" (Path A) +- "Bootstrap flowx so I can run a migration locally" (Path B) +- "I got a ModuleNotFoundError running discover — fix the environment" (Path B) diff --git a/skills/ingest/SKILL.md b/skills/ingest/SKILL.md deleted file mode 100644 index 0466403..0000000 --- a/skills/ingest/SKILL.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -name: ingest -description: > - Load and parse Azure Data Factory pipeline definitions from Unity Catalog volumes or local directories. - Produces a typed inventory that classifies every activity as deterministic, agentic, or unsupported. -triggers: - - "ingest ADF" - - "load ADF" - - "parse ADF" - - "import pipelines" - - "load pipelines" - - "parse pipelines" - - "inventory ADF" ---- - -# Ingest ADF Pipeline Definitions - -Parse Azure Data Factory pipeline, dataset, linked service, and trigger JSON files into a typed AST and produce a classified inventory. - -## Context - -This is phase 1 of the flowx migration workflow. It takes raw ADF JSON exports and produces an `inventory.json` file that the `translate` skill consumes. The inventory classifies every ADF activity into one of three strategies: - -- **Deterministic** — a built-in translator exists (Copy, DatabricksNotebook, ForEach, IfCondition, etc.) -- **Agentic** — requires LLM-assisted translation via the `adf-to-databricks-plugin` skills (ExecuteDataFlow, Switch, Until, StoredProc, etc.) -- **Unsupported** — no known translation path; requires manual intervention - -## Prerequisite — Python environment - -This skill runs the plugin's Python code, which depends on third-party packages. Before running -any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the -**`setup`** skill, or directly: - -```bash -bash /scripts/bootstrap.sh -``` - -This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or -pip is missing, the script prints a warning telling the user what to install — relay it and stop -until they have installed Python 3.12+ and pip. - -Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` -(use it anywhere a command below shows `python3`): - -```bash -export PYTHONPATH="/src" -"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... -``` - -## Workflow - -Follow these steps in order: - -### Step 1 — Determine the ADF source path - -Ask the user for the location of their ADF JSON exports. Accept either: -- A Unity Catalog volume path (e.g., `/Volumes/main/default/adf_export`) -- A local directory path (e.g., `./adf_export/` or `/tmp/adf_json/`) - -The directory should contain subdirectories or files for: -- `pipeline/` or `pipelines/` — pipeline definition JSON files -- `dataset/` or `datasets/` — dataset definition JSON files (optional) -- `linkedService/` or `linked_services/` — linked service JSON files (optional) -- `trigger/` or `triggers/` — trigger definition JSON files (optional) - -### Step 2 — Download from UC volumes if needed - -If the source path starts with `/Volumes/`, the files live in a Unity Catalog volume and must be downloaded to a local temp directory first. - -Use the `databricks-execution-compute` skill to run the following on the Databricks workspace: - -```python -import os, json, shutil, tempfile - -volume_path = "" -local_dir = tempfile.mkdtemp(prefix="adf_ingest_") - -# Copy from volume to local -for root, dirs, files in os.walk(volume_path): - for f in files: - if f.endswith(".json"): - src = os.path.join(root, f) - rel = os.path.relpath(src, volume_path) - dst = os.path.join(local_dir, rel) - os.makedirs(os.path.dirname(dst), exist_ok=True) - shutil.copy2(src, dst) - -print(f"Downloaded ADF files to: {local_dir}") -``` - -Alternatively, use the Databricks CLI: -```bash -databricks fs cp -r "dbfs:" "" --overwrite -``` - -Set the working source directory to the local temp path for subsequent steps. - -### Step 3 — Run the deterministic parser - -Execute the ADF loader to parse all JSON files and produce the inventory: - -```bash -python3 /src/flowx/parser/adf_loader.py \ - --source-dir \ - --output-dir -``` - -Where: -- `` is the root of the flowx plugin (the directory containing `src/`) -- `` is the local directory containing ADF JSON files -- `` is where to write the parsed output (default: `./orchestra_output/ingest/`) - -This produces: -- `inventory.json` — the classified activity inventory -- `ast/` directory — the typed AST for each pipeline -- `parse_errors.json` — any files that failed to parse - -### Step 4 — Read and validate the inventory - -Read the generated `inventory.json` file. It has this structure: - -```json -{ - "source_dir": "/path/to/adf/json", - "generated_at": "2026-04-07T12:00:00Z", - "pipelines": [ - { - "name": "PipelineName", - "file": "pipeline/PipelineName.json", - "activities": [ - { - "name": "CopyFromBlob", - "type": "Copy", - "strategy": "deterministic", - "translator": "copy.py" - }, - { - "name": "RunDataFlow", - "type": "ExecuteDataFlow", - "strategy": "agentic", - "skill": "adf-to-databricks:adf-dataflow-converter" - } - ] - } - ], - "summary": { - "pipeline_count": 12, - "activity_count": 47, - "deterministic_count": 35, - "agentic_count": 10, - "unsupported_count": 2, - "coverage_pct": 95.7 - } -} -``` - -### Step 5 — Present the summary - -Display a summary table to the user: - -``` -ADF Ingestion Summary -===================== -Pipelines parsed: 12 -Total activities: 47 - -Strategy Breakdown: - Deterministic: 35 (74.5%) - Agentic: 10 (21.3%) - Unsupported: 2 ( 4.3%) - -Coverage: 95.7% -``` - -### Step 6 — Detail agentic activities - -For activities classified as `agentic`, explain which skill from the `adf-to-databricks-plugin` will handle each: - -| Activity | Type | Handling Skill | -|---|---|---| -| RunDataFlow | ExecuteDataFlow | `adf-to-databricks:adf-dataflow-converter` | -| BranchLogic | Switch | `adf-to-databricks:adf-pipeline-converter` | -| ... | ... | ... | - -### Step 7 — Warn about unsupported activities - -For activities classified as `unsupported`, warn the user clearly: - -``` -WARNING: The following activities have no automated translation path: - - Pipeline "ETL_Main" / Activity "RunSSIS" (ExecuteSSISPackage) - Recommendation: Manual conversion to PySpark notebook required. -``` - -### Step 8 — Confirm output location - -Tell the user where the inventory and AST files were written, and confirm they can proceed to the `translate` phase. - -## Examples - -- "Ingest my ADF pipelines from /Volumes/main/default/adf_export" -- "Parse ADF definitions from ./tests/resources/json/" -- "Load the ADF pipeline JSON files and show me the inventory" -- "Import pipelines from /tmp/customer_adf_export" - -## Output Artifacts - -| File | Description | -|---|---| -| `inventory.json` | Classified activity inventory for the translate phase | -| `ast/*.json` | Typed AST for each pipeline | -| `parse_errors.json` | Any files that failed to parse | diff --git a/skills/migrate/SKILL.md b/skills/migrate/SKILL.md deleted file mode 100644 index c0f2b9b..0000000 --- a/skills/migrate/SKILL.md +++ /dev/null @@ -1,324 +0,0 @@ ---- -name: migrate -description: > - End-to-end migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs. - Orchestrates ingest, translate, and prepare phases in sequence. -triggers: - - "migrate ADF" - - "migrate pipelines" - - "ADF to Databricks" - - "migrate to Lakeflow" - - "ADF migration" - - "convert ADF to Lakeflow" - - "migrate data factory" ---- - -# End-to-End ADF to Databricks Migration - -Orchestrate the complete migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. This skill runs all three phases in sequence: ingest, translate, prepare. - -## Context - -This is the top-level orchestration skill. It runs the full migration pipeline: - -1. **Ingest** — Parse ADF JSON exports into a typed inventory -2. **Translate** — Convert ADF activities to Databricks IR (deterministic + agentic) -3. **Prepare** — Generate Databricks Declarative Automation Bundles for deployment - -Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. - -## Prerequisite — Python environment - -This skill runs the plugin's Python code, which depends on third-party packages. Before running -any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the -**`setup`** skill, or directly: - -```bash -bash /scripts/bootstrap.sh -``` - -This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or -pip is missing, the script prints a warning telling the user what to install — relay it and stop -until they have installed Python 3.12+ and pip. - -Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` -(use it anywhere a command below shows `python3`): - -```bash -export PYTHONPATH="/src" -"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... -``` - -## Workflow - -Follow these steps in order: - -### Step 0 — Gather phase inputs via the adapter - -Before invoking ingest, run the adapter inputs subcommand once per -phase so the agent surfaces the matching free-text prompts: - -```bash -python3 -m flowx.adapter inputs ingest -python3 -m flowx.adapter inputs translate -python3 -m flowx.adapter inputs prepare -``` - -Each response carries the questions for that phase plus their -descriptions and defaults. Collect answers from the user (or accept -the defaults), persist them to `//inputs.json`, and -thread the values into the downstream CLI calls. - -### Step 1 — Gather inputs - -Ask the user for all required inputs upfront: - -| Parameter | Description | Required | Default | -|---|---|---|---| -| ADF source path | UC volume path or local directory with ADF JSON files | Yes | — | -| Output directory | Root directory for all flowx output | No | `./orchestra_output/` | -| Target catalog | Unity Catalog catalog for tables/volumes | No | `main` | -| Target schema | Schema within the catalog | No | `default` | -| Bundle name | Name for the generated DABs project | No | derived from pipelines | - -Example prompt: - -> To migrate your ADF pipelines, I need: -> 1. Where are your ADF JSON exports? (UC volume path like `/Volumes/main/default/adf_export` or local directory) -> 2. Where should I write the output? (default: `./orchestra_output/`) -> 3. What target catalog and schema? (default: `main.default`) - -### Step 2 — Phase 1: Ingest - -Invoke the `flowx:ingest` skill with the ADF source path and output directory set to `/ingest/`. - -Wait for the ingest to complete and present the inventory summary: - -``` -Phase 1: Ingest — Complete -========================== -Pipelines parsed: 12 -Total activities: 47 - Deterministic: 35 (74.5%) - Agentic: 10 (21.3%) - Unsupported: 2 ( 4.3%) -Coverage: 95.7% -``` - -### Step 3 — Checkpoint: confirm proceed - -Ask the user to review the inventory and confirm before continuing: - -> The ingest phase found 47 activities across 12 pipelines. 95.7% have a translation path (74.5% deterministic, 21.3% agentic). 2 activities are unsupported and will need manual handling. -> -> Proceed to the translation phase? (yes/no) - -If the user says no, explain the options: -- Re-run ingest with a different source directory -- Review the `inventory.json` to understand unsupported activities -- Manually classify activities before proceeding - -If the user says yes, proceed to step 4. - -### Step 4 — Phase 2: Translate - -Invoke the `flowx:translate` skill with: -- Inventory path: `/ingest/inventory.json` -- ADF source dir: the original ADF source path -- Output dir: `/translate/` - -Wait for the translation to complete and present the summary: - -``` -Phase 2: Translate — Complete -============================= -Deterministic translated: 35 (74.5%) -Agentic translated: 8 (17.0%) -Failed: 4 ( 8.5%) -Overall coverage: 91.5% -``` - -### Step 5 — Present translation details - -Show the user: -1. What was translated deterministically (bulk — just counts by type) -2. What was translated via agentic skills (list each with the skill used) -3. What failed and why (list each with the failure reason) - -For failures, suggest: -- Manual notebook creation -- Retry with additional context -- Skip and add placeholder - -### Step 5.1 — Gather just-in-time translation preferences - -Drive the loop multi-pass: re-run `inspect --answers ` -after each batch of answers so the adapter can surface chained -metadata-driven prompts. When the user opts to consolidate a -metadata-driven motif and the agent has a database tool, run the -lookup query directly and persist the rows to -`/translate/lookup_values.json`; otherwise prompt the user -for a CSV file or comma-separated string and run: - -```bash -python3 -m flowx.adapter materialize-lookup "" \ - --out /translate/lookup_values.json -``` - -Pass `--lookup-values` to the modify call when the file exists. - -#### Legacy flow details - -Before bundle generation, run the adapter inspect CLI on the translation -report to surface any pipeline-modifier questions the IR raises: - -```bash -python3 -m flowx.adapter inspect /translate/translation_report.json -``` - -For each question in the JSON output, prompt the user with the rationale, -options, and the affected task keys. Collect answers into -`/translate/answers.json` keyed by `question_id`, then apply -them to a stamped report: - -```bash -python3 -m flowx.adapter modify \ - /translate/translation_report.json \ - /translate/answers.json \ - --out /translate/translation_report.stamped.json -``` - -Use the stamped report (when produced) as the input to the prepare phase. -When inspect emits no questions for any pipeline, skip modify and use the -original report. - -The questions the adapter raises: - -| `question_id` | Allowed values | Default | -|---|---|---| -| `copy_activity_paradigm` | `notebook`, `sdp` | `notebook` | -| `non_databricks_task_compute` | `serverless`, `classic` | `serverless` | -| `use_lakeflow_connectors` | `existing`, `lakeflow_connect` | `existing` | -| `consolidate_motif:` | `keep`, `consolidate` | `keep` | - -DatabricksNotebook and DatabricksSparkPython tasks always inherit the cluster binding derived from -their source linked service. - -For each multi-activity motif the detector matches (rest_api_pagination, -incremental_load_watermark, metadata_driven_bulk_copy, ...) the adapter emits one -`consolidate_motif:` question. The user must explicitly opt in to `consolidate` -for each detected pattern. - -### Step 6 — Checkpoint: confirm proceed to bundle generation - -> Translation is 91.5% complete. 4 activities could not be translated automatically. -> Options: -> 1. Proceed to bundle generation (failed activities will get placeholder tasks) -> 2. Retry failed translations with more context -> 3. Stop here and review the translation report -> -> What would you like to do? - -### Step 6.5 — Detect workspace artifacts and authenticate - -Before invoking the prepare phase, run the adapter's -`workspace-paths` subcommand to detect any absolute workspace paths -the bundle would need to vendor: - -```bash -python3 -m flowx.adapter workspace-paths \ - /translate/translation_report.stamped.json \ - --source-dir -``` - -When the response carries `needs_auth: true`: - -1. Confirm the workspace host with the user, defaulting to the first - entry in `suggested_hosts` (extracted from the Databricks linked - services in the ADF export). -2. Run `databricks auth login --host ` interactively to set up - a local profile. -3. Pass `--profile ` to the prepare invocation in Step 7 so - flowx downloads the referenced notebooks and vendors them under - `bundle/src/notebooks/` with the task references rewritten to the - relative `../src/notebooks/...` paths. - -Skip this step entirely when `needs_auth` is `false`. - -### Step 7 — Phase 3: Prepare - -Invoke the `flowx:prepare` skill with: -- Translation report: `/translate/translation_report.stamped.json` if step 5.5 produced one, otherwise `/translate/translation_report.json` -- Output dir: `/dab_output/` -- Catalog: user-specified or `main` -- Schema: user-specified or `default` - -### Step 8 — Present final summary - -Display the complete migration summary: - -``` -Migration Complete -================== - -Source: /Volumes/main/default/adf_export (12 ADF pipelines) -Output: ./orchestra_output/dab_output/ - -Coverage: - Total activities: 47 - Successfully translated: 43 (91.5%) - Placeholder tasks: 4 ( 8.5%) - -Generated Files: - dab_output/ - databricks.yml - resources/ (3 job definitions) - src/notebooks/ (12 notebooks) - setup/ (3 setup scripts) - tests/ (3 test files) - -Setup Required: - - Run setup/create_volumes.py to create UC volumes - - Run setup/create_secrets.py to configure secrets (review credentials first) - - Run setup/register_connections.py to register external connections - -Next Steps: - 1. cd ./orchestra_output/dab_output/ - 2. Review generated files, especially notebooks and setup scripts - 3. databricks bundle validate --target dev - 4. Run setup scripts on the target workspace - 5. databricks bundle deploy --target dev - 6. databricks bundle run --target dev - 7. Verify job output and promote to staging/prod -``` - -### Step 9 — Offer follow-up actions - -Ask if the user wants to: -1. Validate the bundle now (`databricks bundle validate`) -2. Deploy to dev (`databricks bundle deploy --target dev`) -3. Review specific generated files -4. Re-translate any failed activities -5. Export a migration report for documentation - -## Reference - -See `references/workflow.md` for a detailed description of the three-phase architecture. - -## Examples - -- "Migrate my ADF pipelines to Databricks" -- "Convert ADF to Lakeflow jobs" -- "ADF to Databricks migration from /Volumes/main/default/adf_export" -- "Migrate data factory pipelines to catalog analytics, schema bronze" -- "Run the full ADF migration workflow" - -## Output Artifacts - -All artifacts from all three phases are produced under the output directory: - -| Directory | Phase | Contents | -|---|---|---| -| `ingest/` | Ingest | `inventory.json`, `ast/`, `parse_errors.json` | -| `translate/` | Translate | `translation_report.json`, `ir/`, `notebooks/`, `agentic_results/` | -| `dab_output/` | Prepare | `databricks.yml`, `resources/`, `src/`, `setup/`, `tests/` | diff --git a/skills/prepare/SKILL.md b/skills/prepare/SKILL.md deleted file mode 100644 index b6eed61..0000000 --- a/skills/prepare/SKILL.md +++ /dev/null @@ -1,254 +0,0 @@ ---- -name: prepare -description: > - Generate Databricks Declarative Automation Bundles (DABs) from translated IR, - including job definitions, notebooks, and setup scripts. -triggers: - - "prepare bundles" - - "generate DABs" - - "create bundles" - - "prepare deployment" - - "generate bundles" - - "build DABs" ---- - -# Prepare Databricks Declarative Automation Bundles - -Generate deployment-ready Databricks Declarative Automation Bundles (DABs) from the translated intermediate representation, including job definitions, notebooks, and infrastructure setup scripts. - -## Context - -This is phase 3 of the flowx migration workflow. It consumes the `translation_report.json` produced by the `translate` skill and generates a complete DABs project that can be validated and deployed with the Databricks CLI. - -The output is a standard DABs project with: -- `databricks.yml` — the bundle configuration -- `resources/` — job and pipeline YAML definitions -- `src/notebooks/` — generated and helper notebooks -- `setup/` — infrastructure setup scripts (volumes, secrets, connections) - -## Prerequisite — Python environment - -This skill runs the plugin's Python code, which depends on third-party packages. Before running -any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the -**`setup`** skill, or directly: - -```bash -bash /scripts/bootstrap.sh -``` - -This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or -pip is missing, the script prints a warning telling the user what to install — relay it and stop -until they have installed Python 3.12+ and pip. - -Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` -(use it anywhere a command below shows `python3`): - -```bash -export PYTHONPATH="/src" -"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... -``` - -## Workflow - -Follow these steps in order: - -### Step 1 — Locate the translation report - -Read `translation_report.json` from the translate phase. If the path is not in conversation context, ask the user: - -> Where is the translation_report.json from the translate phase? (default: `./orchestra_output/translate/translation_report.json`) - -Validate the file exists and all required translations have status `translated`. - -### Step 2 — Gather deployment parameters - -Ask the user for the following (provide defaults): - -| Parameter | Description | Default | -|---|---|---| -| Target catalog | Unity Catalog catalog for tables/volumes | `main` | -| Target schema | Schema within the catalog | `default` | -| Output directory | Where to write the DABs project | `./dab_output/` | -| Bundle name | Name for the DABs project | derived from first pipeline name | -| Target environments | Deployment targets to configure | `dev, staging, prod` | -| Warehouse ID | SQL warehouse for SQL tasks (optional) | prompt if SQL tasks exist | -| Databricks CLI profile | Profile used to download workspace-resident notebooks / JARs / Python files (`--profile`). Required only when the bundle references absolute workspace paths. | resolved from `~/.databrickscfg` (auto-prompt if multiple) | - -### Step 2.5 — Detect workspace artifacts and authenticate - -Before running the bundle writer, check whether the report references -absolute workspace paths (notebooks under `/Shared/`, SparkPython -files, SparkJar libraries) or DBFS paths that the bundle should -download to be self-contained: - -```bash -python3 -m flowx.adapter workspace-paths \ - \ - --source-dir -``` - -The command emits: - -```json -{ - "paths": ["/Shared/team/notebook_a", "/Shared/team/notebook_b"], - "suggested_hosts": ["https://adb-1234.5.azuredatabricks.net"], - "needs_auth": true -} -``` - -When `needs_auth` is `true`: - -1. Surface the suggested hosts to the user with `AskUserQuestion`. Use - the first `suggested_hosts` value as the default; allow the user to - override. When no host is suggested (no Databricks linked service - in the export), prompt for the host with no default. -2. Run the interactive Databricks CLI login command and wait for it to - complete: - - ```bash - databricks auth login --host - ``` - - This writes a profile into `~/.databrickscfg`. When the user has - chosen a specific profile name, append `--profile ` to both - the login and the prepare invocation below. - -3. Pass the resolved profile to step 3 via `--profile ` (default - profile name is `DEFAULT`). When `needs_auth` is `false` skip steps - 1–2 and omit `--profile` from step 3. - -The `paths` list is informational; you can echo it to the user so they -know which notebooks the bundle will vendor. - -### Step 3 — Run bundle generation - -Execute the DAB writer: - -```bash -python3 /src/flowx/bundler/dab_writer.py \ - --report \ - --output-dir \ - --catalog \ - --schema \ - --bundle-name \ - [--profile ] \ - [--no-vendor-workspace-files] -``` - -Where: -- `` is the root of the flowx plugin -- `` is the path to `translation_report.json` -- Other parameters are from step 2 - -**Workspace artifact vendoring (default: enabled).** When the report references workspace-resident notebooks (`/Shared/...`), DBFS Spark JARs (`dbfs:/...`), or Spark Python files, the preparer downloads them via the Databricks CLI auth so the resulting bundle is self-contained and deployable across environments. Downloaded notebooks are vendored under `src/notebooks/` and bound to the default `job_cluster` (since they may rely on classic-compute features). The original `notebook_path` in the resource YAML is rewritten to the bundle-relative path `../src/notebooks/.py`. - -If no Databricks CLI auth is detected on the host (`~/.databrickscfg` empty AND no `DATABRICKS_CONFIG_PROFILE` / `DATABRICKS_HOST`+`DATABRICKS_TOKEN` env vars), the CLI prints the workspace paths it was about to download and prompts: - -``` -Workspace downloads are enabled but no Databricks CLI auth was found. - Looked for profiles in: /Users//.databrickscfg - Artifacts to vendor: /Shared/ETL/transform, … - -To authenticate, run one of: - databricks auth login --host https://.cloud.databricks.com - databricks configure --token - -Continue with placeholders (downloads will be skipped)? [y/N]: -``` - -Answering `n` aborts with exit code 2 so the user can authenticate and re-run. Answering `y` continues with placeholder notebooks (legacy in-place workspace paths). In non-interactive sessions the prompt defaults to placeholders. - -Use `--no-vendor-workspace-files` to opt out entirely; the bundle then keeps original workspace paths exactly as in the IR. - -### Step 4 — Present the generated file tree - -Show the user what was generated: - -``` -dab_output/ - databricks.yml - resources/ - etl_main_job.yml - etl_secondary_job.yml - transform_dlt_pipeline.yml - src/ - notebooks/ - copy_from_blob.py - lookup_config.py - web_activity_call.py - set_variable_helper.py - setup/ - create_volumes.py - create_secrets.py - register_connections.py - tests/ - test_etl_main.py -``` - -### Step 5 — Explain setup tasks - -If the `setup/` directory was generated, explain what each script does: - -**create_volumes.py** — Creates Unity Catalog volumes required by the migrated jobs. These volumes replace Azure Blob Storage or ADLS references from ADF. Run this once per environment. - -**create_secrets.py** — Creates Databricks secret scopes and secrets for connection credentials that were in ADF linked services. Review the secret values and populate them manually or via your secrets management system. - -**register_connections.py** — Registers Unity Catalog connections for external data sources (SQL Server, REST APIs, etc.) that were referenced in ADF linked services. - -Emphasize that the user should review these scripts before running them, especially `create_secrets.py` which will need actual credential values. - -### Step 6 — Explain the generated bundle structure - -Briefly describe: -- **databricks.yml** — The root bundle config with workspace, target environments (dev/staging/prod), and variable definitions. Variables are parameterized for environment-specific values (catalog, schema, warehouse). -- **resources/*.yml** — One YAML file per Databricks Lakeflow Job (one per ADF pipeline). Each job contains tasks mapped from ADF activities, with dependencies matching the original ADF dependency chains. -- **src/notebooks/*.py** — Python notebooks for activities that translate to notebook_task. These contain the actual data movement or transformation logic. -- **tests/*.py** — Skeleton test files for validating the migrated jobs. - -### Step 7 — Suggest next steps - -Present the following next steps: - -``` -Next Steps -========== -1. Review generated files: - cd - cat databricks.yml - -2. Validate the bundle: - databricks bundle validate --target dev - -3. Run setup scripts (if generated): - databricks bundle run setup_volumes --target dev - -4. Deploy to dev: - databricks bundle deploy --target dev - -5. Test the deployed jobs: - databricks bundle run --target dev - -6. Promote to staging/prod: - databricks bundle deploy --target staging - databricks bundle deploy --target prod -``` - -Recommend running `databricks bundle validate` first to catch any configuration issues before deployment. - -## Examples - -- "Prepare the bundles" -- "Generate DABs for the translated pipelines" -- "Create deployment bundles targeting catalog 'analytics' and schema 'bronze'" -- "Build the DABs project in ./output/my_migration/" - -## Output Artifacts - -| File | Description | -|---|---| -| `databricks.yml` | Root bundle configuration | -| `resources/*.yml` | Job and pipeline YAML definitions | -| `src/notebooks/*.py` | Generated notebooks | -| `setup/*.py` | Infrastructure setup scripts | -| `tests/*.py` | Skeleton test files | diff --git a/skills/translate/SKILL.md b/skills/translate/SKILL.md deleted file mode 100644 index 0ce95b9..0000000 --- a/skills/translate/SKILL.md +++ /dev/null @@ -1,329 +0,0 @@ ---- -name: translate -description: > - Translate parsed ADF pipeline AST into Databricks IR (intermediate representation). - Runs deterministic translators for known activity types, then invokes agentic skills - from adf-to-databricks-plugin for gaps. -triggers: - - "translate ADF" - - "convert ADF" - - "translate pipelines" - - "convert pipelines" - - "run translation" ---- - -# Translate ADF to Databricks IR - -Convert the parsed ADF inventory into Databricks intermediate representation (IR) using deterministic translators for known types and agentic fallback for unknown types. - -## Context - -This is phase 2 of the flowx migration workflow. It consumes the `inventory.json` produced by the `ingest` skill and produces a `translation_report.json` that the `prepare` skill uses to generate Databricks Declarative Automation Bundles. - -The translation follows a **deterministic-first** strategy: -1. Activities with known, well-defined mappings are translated by built-in Python translators -2. Activities that require interpretation, expression conversion, or lack a Python translator are handled by agent skills from the `adf-to-databricks-plugin` - -## Prerequisite — Python environment - -This skill runs the plugin's Python code, which depends on third-party packages. Before running -any Python commands, ensure the plugin's virtual environment is bootstrapped. Run the -**`setup`** skill, or directly: - -```bash -bash /scripts/bootstrap.sh -``` - -This creates `/.venv` and installs dependencies from `requirements.txt`. If Python or -pip is missing, the script prints a warning telling the user what to install — relay it and stop -until they have installed Python 3.12+ and pip. - -Run **every** Python command in this skill with the venv interpreter and `src/` on `PYTHONPATH` -(use it anywhere a command below shows `python3`): - -```bash -export PYTHONPATH="/src" -"/.venv/bin/python" /src/flowx/parser/adf_loader.py ... -``` - -## Workflow - -Follow these steps in order: - -### Step 0 — Gather phase inputs - -Run the adapter inputs subcommand so the agent surfaces the free-text -questions the phase needs (inventory path, ADF source dir, output -directory): - -```bash -python3 -m flowx.adapter inputs translate -``` - -The JSON response carries the prompts and defaults; collect answers -from the user (or fall back to the defaults) and persist them to -`/translate/inputs.json` so later steps and subsequent -phases can read the same values. - -### Step 1 — Locate the inventory - -Read `inventory.json` from the ingest phase. If the path is not already in conversation context, ask the user: - -> Where is the inventory.json from the ingest phase? (default: `./orchestra_output/ingest/inventory.json`) - -Validate the file exists and is well-formed. - -### Step 2 — Run deterministic translation - -Execute the translation engine on all deterministic activities: - -```bash -python3 /src/flowx/translator/engine.py \ - --inventory \ - --source-dir \ - --output-dir -``` - -Where: -- `` is the root of the flowx plugin -- `` is the path to `inventory.json` -- `` is the original ADF JSON directory (from the ingest phase) -- `` is the translation output path (default: `./orchestra_output/translate/`) - -This produces: -- `translation_report.json` — results for deterministic activities + placeholders for agentic gaps -- `ir/` directory — Databricks IR for each translated activity -- `notebooks/` directory — generated helper notebooks - -### Step 3 — Read the translation report - -Read `translation_report.json`. It has this structure: - -```json -{ - "inventory_path": "/path/to/inventory.json", - "generated_at": "2026-04-07T12:30:00Z", - "translations": [ - { - "pipeline": "ETL_Main", - "activity": "CopyFromBlob", - "type": "Copy", - "strategy": "deterministic", - "status": "translated", - "ir": { - "task_key": "copy_from_blob", - "task_type": "notebook_task", - "notebook_path": "notebooks/copy_from_blob.py", - "parameters": { "source": "abfss://...", "target": "..." } - } - }, - { - "pipeline": "ETL_Main", - "activity": "TransformData", - "type": "ExecuteDataFlow", - "strategy": "agentic", - "status": "pending", - "raw_activity_json": { "...": "..." }, - "target_skill": "adf-to-databricks:adf-dataflow-converter" - } - ], - "summary": { - "total": 47, - "deterministic_translated": 35, - "agentic_pending": 10, - "failed": 2 - } -} -``` - -### Step 4 — Handle agentic gaps - -For each translation with `"status": "pending"` and `"strategy": "agentic"`, invoke the appropriate skill from the `adf-to-databricks-plugin`. Route by activity type: - -**ExecuteDataFlow activities:** -Invoke `adf-to-databricks:adf-dataflow-converter` with the raw activity JSON and associated data flow definition. Provide context: -- The raw `typeProperties` from the ADF activity -- The data flow JSON definition (if available in the source directory under `dataflow/`) -- The linked service configurations for source/sink connections -- Target catalog and schema for the SDP pipeline or PySpark notebook output - -**Control flow activities (Switch, Until, Wait, Filter, AppendVariable):** -Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: -- The full pipeline JSON containing the activity -- Any nested activities within the control flow -- Variable definitions from the pipeline -- The desired Databricks task type mapping - -**Stored procedures and external calls (SqlServerStoredProcedure, AzureFunction, WebHook, Custom):** -Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: -- The linked service configuration for the target system -- Connection details and authentication method -- Any parameters or request bodies - -**Complex expressions:** -If any activity (deterministic or agentic) contains ADF expressions that the deterministic translator could not resolve, invoke `adf-to-databricks:adf-expression-translator` with: -- The raw expression string (e.g., `@pipeline().parameters.inputPath`) -- The expression context (pipeline parameters, variables, activity outputs) -- The target format (Python f-string, Spark SQL, task parameter reference) - -**Trigger definitions:** -Invoke `adf-to-databricks:adf-trigger-converter` with: -- The trigger JSON definition -- The associated pipeline references -- Target: Databricks job schedule configuration (quartz_cron_expression, periodic, or file_arrival) - -### Step 5 — Collect agentic results - -Each agentic skill invocation produces a translation result. Collect all results into `/agentic_results/`: -- Save each result as `__.json` -- Include the generated IR, any notebooks, and metadata - -### Step 6 — Merge agentic results - -Run the merge step to combine deterministic and agentic translations: - -```bash -python3 /src/flowx/translator/engine.py \ - --merge-agentic \ - --report \ - --agentic-results -``` - -This updates `translation_report.json` with the agentic results merged in, changing their status from `pending` to `translated` (or `failed` if the agentic skill could not produce a result). - -### Step 6.1 — Gather just-in-time translation preferences - -The adapter raises several preference questions plus a chained set for -metadata-driven motifs. Every time the user answers a question whose value -gates further prompts, re-run `inspect --answers ` to surface -the next batch. - -When the metadata-driven flow ends with `metadata_driven_lookup_tool=have` -and the agent has a database tool (Genie, MCP SQL, or a workspace SDK), -run the lookup query directly and write the rows to -`/lookup_values.json`. When the answer is `none`, prompt -the user for a CSV file or comma-separated string and call: - -```bash -python3 -m flowx.adapter materialize-lookup "" \ - --out /lookup_values.json -``` - -Then call `modify` with the lookup values: - -```bash -python3 -m flowx.adapter modify \ - \ - /answers.json \ - --lookup-values /lookup_values.json \ - --out /translation_report.stamped.json -``` - -When no metadata-driven motif is consolidated, `--lookup-values` is omitted. - -#### Legacy flow details - -Before writing the final report, surface any pipeline-modifier questions the -IR raises (Copy Data paradigm, non-Databricks task compute, Lakeflow Connect -opt-in, Databricks task compute). Use the adapter CLI bridge: - -```bash -python3 -m flowx.adapter inspect -``` - -The command emits JSON: - -```json -{ - "pipelines": [ - { - "pipeline_name": "ETL_Main", - "questions": [ - { - "question_id": "copy_activity_paradigm", - "prompt": "How should Copy Data activities targeting Delta be implemented?", - "rationale": "...", - "options": [{"value": "notebook", "label": "...", "description": "..."}, ...], - "affected_task_keys": ["copy_orders", "copy_customers"], - "default": "notebook" - }, - ... - ] - } - ] -} -``` - -For each question, prompt the user with the rationale, options, and the -task keys it affects. Use the default when the user defers. Collect the -answers into a JSON file (`/answers.json`) shaped like: - -```json -{ - "copy_activity_paradigm": "sdp", - "non_databricks_task_compute": "serverless", - "use_lakeflow_connectors": "lakeflow_connect" -} -``` - -Then apply the answers to produce a stamped report the prepare phase consumes: - -```bash -python3 -m flowx.adapter modify \ - \ - /answers.json \ - --out /translation_report.stamped.json -``` - -The prepare phase (next skill) must be pointed at the stamped report. -When no questions are raised, the inspect output is `{"pipelines": [{"pipeline_name": "...", "questions": []}, ...]}` — skip the modify step and pass the original report straight through. - -### Step 7 — Present translation summary - -Display a summary to the user: - -``` -Translation Summary -=================== -Total activities: 47 -Deterministic translated: 35 (74.5%) -Agentic translated: 8 (17.0%) -Failed: 4 ( 8.5%) - -Overall coverage: 91.5% - -Failed translations: - - ETL_Main / RunSSIS (ExecuteSSISPackage) — no translator available - - ETL_Main / CustomTask (Custom) — agentic skill returned error - ... - -Generated artifacts: - - translation_report.json - - ir/ (43 files) - - notebooks/ (12 files) -``` - -If coverage is below 100%, explain the options for failed translations: -1. Manual notebook creation for unsupported types -2. Retry agentic translation with additional context -3. Skip the activity and add a placeholder task in the DAB - -## Reference - -See `references/activity-mapping.md` for the complete mapping between ADF activity types and translation strategies. - -## Examples - -- "Translate the ADF pipelines" -- "Convert ADF to Databricks" -- "Run the translation on the inventory from the ingest step" -- "Translate the parsed pipelines using deterministic + agentic" - -## Output Artifacts - -| File | Description | -|---|---| -| `translation_report.json` | Full translation report with IR for all activities | -| `ir/*.json` | Databricks IR for each translated activity | -| `notebooks/*.py` | Generated helper notebooks | -| `agentic_results/*.json` | Raw results from agentic skill invocations | diff --git a/src/AGENTS.md b/src/AGENTS.md index 2198208..e0cd18d 100644 --- a/src/AGENTS.md +++ b/src/AGENTS.md @@ -25,13 +25,15 @@ neighbouring module. This is a companion to the top-level [AGENTS.md](../AGENTS. - First line is **third-person present indicative** describing what the function does: ``"""Converts X to Y."""`` rather than ``"""Convert X..."""`` or ``"""This function converts..."""``. -- Keep docstrings **brief** -- usually one sentence. Add extra detail - only when behaviour is non-obvious (edge cases, surprising - invariants). Don't restate the type signature or repeat parameter - names; ``Args:`` / ``Returns:`` / ``Raises:`` blocks are optional and - should appear only when the type is genuinely ambiguous. -- Don't write multi-paragraph design rationale in docstrings; that - belongs in commit messages or pull-request descriptions. +- **Public** classes and functions use the **Google docstring format**: + a one-line summary, then ``Attributes:`` (classes), ``Args:``, + ``Returns:``, ``Raises:``, and ``Notes:`` sections as applicable. +- Private helpers keep a brief one-line summary; add ``Args:`` / + ``Returns:`` only when the types are genuinely ambiguous. +- Non-obvious design rationale (a constraint, a prior bug, an external + spec) goes in a ``Notes:`` section or a one-line comment that points to + the relevant ``AGENTS.md`` design-notes entry -- never as a + multi-paragraph comment block. ## Comments diff --git a/src/flowx/__init__.py b/src/flowx/__init__.py new file mode 100644 index 0000000..2723f34 --- /dev/null +++ b/src/flowx/__init__.py @@ -0,0 +1,3 @@ +"""flowx - ADF to Databricks translation plugin for Claude Code.""" + +__version__ = "0.1.0" diff --git a/src/flowx/adapter/__init__.py b/src/flowx/adapter/__init__.py new file mode 100644 index 0000000..780bdbd --- /dev/null +++ b/src/flowx/adapter/__init__.py @@ -0,0 +1,75 @@ +"""Agent-facing surfaces and the matching pipeline modifier for flowx translation. + +Two roles live here, kept deliberately separate: the **agent adapter** (:mod:`~flowx.adapter.session` +plus the option shapes in :mod:`~flowx.adapter.models`) converts tool-call arguments into +deterministic calls and maps "need more input" into structured objects, while the **pipeline modifier** +(:mod:`~flowx.adapter.operations`) consumes a validated :class:`TranslationConfiguration` and stamps +concrete decisions onto a Pipeline IR with no awareness of agents. ``constants`` holds shared strings and +``predicates`` holds pure IR predicates used by both ``operations`` and the bundler. +""" + +from __future__ import annotations + +from flowx.adapter.models import ( + DEFAULT_CONFIGURATION, + CopyActivityParadigm, + LakeflowConnectorType, + MetadataDrivenAccess, + MetadataDrivenConsolidate, + MetadataDrivenLookupTool, + MetadataDrivenSize, + MigrationInputOption, + MotifConsolidate, + NonDatabricksTaskCompute, + OptionChoice, + PendingMigrationInputs, + PendingOptions, + TranslationConfiguration, + TranslationOption, + UseLakeflowConnectors, +) +from flowx.adapter.operations import ( + allowed_values_for, + apply_configuration, + collect_workspace_artifact_paths, + detect_databricks_hosts, + enum_for, + gather_options, + validate_answer, +) +from flowx.adapter.session import ( + MigrationInputSession, + TranslationInputRequired, + TranslationSession, + UnknownMigrationPhaseError, +) + +__all__ = [ + "DEFAULT_CONFIGURATION", + "CopyActivityParadigm", + "LakeflowConnectorType", + "MetadataDrivenAccess", + "MetadataDrivenConsolidate", + "MetadataDrivenLookupTool", + "MetadataDrivenSize", + "MigrationInputOption", + "MigrationInputSession", + "MotifConsolidate", + "NonDatabricksTaskCompute", + "PendingMigrationInputs", + "PendingOptions", + "OptionChoice", + "TranslationInputRequired", + "TranslationConfiguration", + "TranslationOption", + "TranslationSession", + "UnknownMigrationPhaseError", + "UseLakeflowConnectors", + "allowed_values_for", + "apply_configuration", + "collect_workspace_artifact_paths", + "detect_databricks_hosts", + "enum_for", + "gather_options", + "validate_answer", +] diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py new file mode 100644 index 0000000..1d214aa --- /dev/null +++ b/src/flowx/adapter/__main__.py @@ -0,0 +1,792 @@ +"""Unified CLI entry point that the flowx skills and MCP tools drive via subprocesses. + +Exposes stateless subcommands -- the ``discover``/``convert``/``package`` phase runners plus +``inspect``, ``modify``, ``inputs``, ``materialize-lookup``, ``workspace-paths``, ``record-results``, +and ``install-dashboard`` -- so each agent turn runs as an independent process holding no session +state across user prompts. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from flowx.adapter.constants import MOTIF_CONSOLIDATE_OPTION_PREFIX +from flowx.adapter.models import ( + DEFAULT_CONFIGURATION, + CopyActivityParadigm, + LakeflowConnectorType, + MetadataDrivenAccess, + MetadataDrivenConsolidate, + MetadataDrivenLookupTool, + MetadataDrivenSize, + MotifConsolidate, + NonDatabricksTaskCompute, + NotifyDestination, + NotifyEvents, + TranslationConfiguration, + UseLakeflowConnectors, +) +from flowx.adapter.operations import ( + apply_configuration, + build_option_schema, + collect_notify_args, + collect_workspace_artifact_paths, + detect_databricks_hosts, + provision_notification_destinations, + validate_answer, +) + +# bundler.dab_writer + translator.engine (sqlglot) are imported lazily inside inspect/modify only, so +# the cheap commands (inputs, phase pass-throughs, materialize-lookup, workspace-paths) skip ~0.15s of +# unused import cost on every adapter subprocess. + +# Maps the unified phase runner subcommands to the module CLI they forward to. +_PHASE_MODULES: dict[str, str] = { + "discover": "flowx.parser.adf_loader", + "convert": "flowx.translator.engine", + "package": "flowx.bundler.dab_writer", +} +# Aliases so the inputs option ids double as CLI flags on the phase runners. +_PHASE_FLAG_ALIASES: dict[str, str] = { + "--adf-source-path": "--source-dir", +} + + +def main(argv: list[str] | None = None) -> int: + """Dispatches an ``inspect`` or ``modify`` subcommand. + + Args: + argv: CLI arguments to parse. Defaults to :data:`sys.argv` when + ``None``. + + Returns: + Exit code (0 on success, non-zero on usage or runtime errors). + """ + raw_args = list(sys.argv[1:]) if argv is None else list(argv) + if raw_args and raw_args[0] in _PHASE_MODULES: + # Phase runners are pure pass-through to the underlying phase CLI; + # bypass argparse so forwarded --flags aren't misparsed at this level. + return _run_phase(raw_args[0], raw_args[1:]) + + parser = _build_parser() + args = parser.parse_args(argv) + if args.command == "inspect": + return _run_inspect(args) + if args.command == "modify": + return _run_modify(args) + if args.command == "materialize-lookup": + return _run_materialize_lookup(args) + if args.command == "inputs": + return _run_inputs(args) + if args.command == "workspace-paths": + return _run_workspace_paths(args) + if args.command == "record-results": + return _run_record_results(args) + if args.command == "install-dashboard": + return _run_install_dashboard(args) + parser.print_help(sys.stderr) + return 2 + + +def _run_record_results(args: argparse.Namespace) -> int: + """Implements ``record-results``: write per-pipeline coverage to a UC table. + + Returns 0 on success, 1 when the metadata cannot be read or the write fails. + """ + from flowx.reporting.results import write_results + + metadata_dir = args.output_dir / "metadata" + if not (metadata_dir / "inventory.json").exists(): + print(f"No inventory.json under {metadata_dir}; run the discover phase first.", file=sys.stderr) + return 1 + try: + run_id, rows = write_results(metadata_dir, args.results_table, warehouse_id=args.warehouse_id) + except Exception as error: # noqa: BLE001 - surface an actionable message to the agent + print(f"Failed to record results to {args.results_table}: {error}", file=sys.stderr) + return 1 + if not rows: + print("No pipelines found to record.", file=sys.stderr) + return 1 + print(f"Recorded {rows} pipeline row(s) to {args.results_table} (run_id={run_id}).") + return 0 + + +def _run_install_dashboard(args: argparse.Namespace) -> int: + """Implements ``install-dashboard``: create + publish the coverage dashboard. + + Returns 0 on success, 1 when the dashboard could not be created. + """ + from flowx.reporting.dashboard import install_dashboard + + try: + dashboard_id, url = install_dashboard( + args.results_table, + warehouse_id=args.warehouse_id, + display_name=args.dashboard_name, + parent_path=args.parent_path, + ) + except Exception as error: # noqa: BLE001 - surface an actionable message to the agent + print(f"Failed to install dashboard for {args.results_table}: {error}", file=sys.stderr) + return 1 + print(f"Installed coverage dashboard (id={dashboard_id}).") + if url: + print(f" {url}") + return 0 + + +def _run_workspace_paths(args: argparse.Namespace) -> int: + """Implements the ``workspace-paths`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``report``, ``source_dir``, + and ``out``. + + Returns: + ``0`` on success. The command always succeeds when the report + can be read; missing or unreadable inputs simply produce empty + path / host lists so the skill can detect the no-op case. + """ + paths = collect_workspace_artifact_paths(args.report) + suggested_hosts = detect_databricks_hosts(args.source_dir) if args.source_dir else [] + payload = { + "paths": paths, + "suggested_hosts": suggested_hosts, + "needs_auth": bool(paths), + } + _emit_json(payload, args.out) + return 0 + + +def _run_inputs(args: argparse.Namespace) -> int: + """Implements the ``inputs`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``phase`` and ``out``. + + Returns: + ``0`` on success. The CLI never raises here because the phase + argument is constrained by argparse. + """ + from flowx.adapter.session import MigrationInputSession + + session = MigrationInputSession(phase=args.phase) + pending = session.pending() + payload = { + "phase": pending.phase, + "options": [ + { + "option_id": option.option_id, + "prompt": option.prompt, + "description": option.description, + "default": option.default, + "required": option.required, + } + for option in pending.options + ], + } + _emit_json(payload, args.out) + return 0 + + +def _build_parser() -> argparse.ArgumentParser: + """Builds the top-level argparse parser with the two subcommands. + + Returns: + Configured :class:`argparse.ArgumentParser`. + """ + parser = argparse.ArgumentParser( + prog="python -m flowx.adapter", + description="Inspect and modify a translated flowx pipeline IR.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + inspect = subparsers.add_parser( + "inspect", + help="Emit the full translation-option schema for a report as JSON.", + ) + inspect.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") + inspect.add_argument( + "--answer", + action="append", + default=[], + metavar="OPTION_ID=VALUE", + help=( + "Deprecated/no-op: the full option tree (every option with a `show_when` condition) is " + "always emitted now, so the agent walks the chain locally. Accepted for back-compat and " + "validated for format only." + ), + ) + inspect.add_argument( + "--out", + type=Path, + default=None, + help="Optional output file; defaults to stdout.", + ) + + modify = subparsers.add_parser( + "modify", + help="Apply collected answers to a translation report and write the stamped IR.", + ) + modify.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") + modify.add_argument( + "--answer", + action="append", + default=[], + metavar="OPTION_ID=VALUE", + help=( + "A collected answer as OPTION_ID=VALUE (e.g. --answer notify_destination=email). " + "Repeatable; pass one per option the user answered. Values may contain '=' (only the " + "first '=' splits the pair)." + ), + ) + modify.add_argument( + "--output-dir", + type=Path, + default=None, + help=( + "Migration output directory. The stamped IR is written to its transient " + ".work/translation_report.stamped.json and the collected answers are written to " + "metadata/configuration.json. Either --output-dir or --out is required." + ), + ) + modify.add_argument( + "--out", + type=Path, + default=None, + help=("Explicit destination for the configuration-stamped IR JSON (overrides the --output-dir convention)."), + ) + modify.add_argument( + "--config-out", + type=Path, + default=None, + help=( + "Explicit destination for configuration.json (the collected answers). Defaults to " + "/metadata/configuration.json." + ), + ) + modify.add_argument( + "--lookup-csv", + type=str, + default=None, + help=( + "Optional CSV file path or literal CSV string of lookup-value rows that consolidated " + "metadata-driven motifs should ingest. The header row names the columns; each " + "subsequent row becomes one dict." + ), + ) + + workspace_paths = subparsers.add_parser( + "workspace-paths", + help=( + "Detect absolute workspace paths in a stamped report and suggest " + "Databricks workspace hosts from the ADF linked services." + ), + ) + workspace_paths.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") + workspace_paths.add_argument( + "--source-dir", + type=Path, + default=None, + help=( + "Optional path to the ADF JSON export directory. When supplied, " + "the command reads ``linked_services/*.json`` to suggest the " + "workspace host that ``databricks auth login --host`` should use." + ), + ) + workspace_paths.add_argument( + "--out", + type=Path, + default=None, + help="Optional output file; defaults to stdout.", + ) + + inputs = subparsers.add_parser( + "inputs", + help="Emit the migration-phase input options for an flowx phase as JSON.", + ) + inputs.add_argument( + "phase", + choices=("discover", "convert", "package"), + help="Migration phase whose input prompts the agent should surface.", + ) + inputs.add_argument( + "--out", + type=Path, + default=None, + help="Optional output file; defaults to stdout.", + ) + + materialize = subparsers.add_parser( + "materialize-lookup", + help="Parse CSV-shaped lookup values into the JSON shape modify consumes.", + ) + materialize.add_argument( + "source", + help=( + "Either a path to a CSV file or a literal CSV string. The first row " + "is treated as headers and every subsequent row is emitted as one dict." + ), + ) + materialize.add_argument( + "--out", + type=Path, + required=True, + help="Destination path for the lookup-values JSON list.", + ) + + record = subparsers.add_parser( + "record-results", + help="Write per-pipeline migration coverage for this run to a Unity Catalog table.", + ) + record.add_argument( + "--output-dir", + type=Path, + required=True, + help="Migration output directory (reads metadata/inventory.json + metadata/profile_report.csv).", + ) + record.add_argument( + "--results-table", + type=str, + required=True, + help="Target UC table as catalog.schema.table.", + ) + record.add_argument( + "--warehouse-id", + type=str, + default=None, + help="SQL warehouse id for the write. Auto-detected (prefers running serverless) when omitted.", + ) + + dashboard = subparsers.add_parser( + "install-dashboard", + help="Create and publish an AI/BI dashboard visualizing coverage from the results table.", + ) + dashboard.add_argument( + "--results-table", + type=str, + required=True, + help="UC table the dashboard reads (catalog.schema.table).", + ) + dashboard.add_argument( + "--warehouse-id", + type=str, + default=None, + help="SQL warehouse backing the dashboard. Auto-detected when omitted.", + ) + dashboard.add_argument( + "--dashboard-name", + type=str, + default=None, + help="Dashboard display name (defaults to 'Migration Coverage \u2014
').", + ) + dashboard.add_argument( + "--parent-path", + type=str, + default=None, + help="Workspace folder for the dashboard (defaults to the current user's home).", + ) + + # Unified phase runners: `adapter -- ` forwards to the phase CLI (one entry point); + # --adf-source-path is accepted as an alias of the loader/translator --source-dir flag. + for _phase in ("discover", "convert", "package"): + _runner = subparsers.add_parser( + _phase, + help=f"Run the {_phase} phase (forwards flags to the underlying phase CLI).", + ) + _runner.add_argument( + "forward", + nargs=argparse.REMAINDER, + help="Flags forwarded to the phase CLI (e.g. --adf-source-path/--source-dir, --output-dir, --pipeline).", + ) + + return parser + + +def _run_phase(phase: str, forward: list[str]) -> int: + """Forward a phase runner subcommand to the underlying phase module, **in-process**. + + ``python -m flowx.adapter discover --adf-source-path X --output-dir Y`` runs + ``flowx.parser.adf_loader.main(["--source-dir", "X", "--output-dir", "Y"])`` in this same + interpreter -- no second ``python -m`` spawn. The module's ``main(argv)`` reuses the existing, + tested phase CLI surface, so there is a single entry point with no argument-surface duplication. + Collapsing the former double-spawn (adapter process -> module process) shaves an interpreter + start + re-import off every ``discover``/``convert``/``package`` call. + + Args: + phase: One of ``"discover"`` / ``"convert"`` / ``"package"``. + forward: Tokens after the phase name (flags for the phase CLI). + + Returns: + The phase's exit code (0 on success). + """ + import importlib + + module = importlib.import_module(_PHASE_MODULES[phase]) + mapped = [_PHASE_FLAG_ALIASES.get(token, token) for token in (forward or [])] + try: + return module.main(mapped) or 0 + except SystemExit as exit_signal: # e.g. argparse usage error -> parser.error() raises SystemExit + code = exit_signal.code + if isinstance(code, int): + return code + return 0 if code is None else 1 + + +def _run_inspect(args: argparse.Namespace) -> int: + """Implements the ``inspect`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``report``, ``answers``, and + ``out``. + + Returns: + ``0`` when the report was inspected successfully, ``1`` when the + report could not be loaded. + """ + pipelines = _load_pipelines(args.report) + if pipelines is None: + return 1 + try: + # The agent now walks the full option tree locally, so --answer no longer filters the output; + # we still validate its format so a malformed pair is reported rather than silently ignored. + _parse_answer_args(getattr(args, "answer", []) or []) + except ValueError as error: + print(f"Invalid --answer: {error}", file=sys.stderr) + return 2 + payload = { + "pipelines": [ + {"pipeline_name": pipeline.name, "options": build_option_schema(pipeline)} for pipeline in pipelines + ], + } + _emit_json(payload, args.out) + return 0 + + +def _parse_answer_args(pairs: list[str]) -> dict[str, str]: + """Parses repeatable ``--answer OPTION_ID=VALUE`` CLI args into a mapping. + + Only the first ``=`` splits each pair, so values may themselves contain + ``=`` (e.g. a query string or base64 token). + + Args: + pairs: Raw ``OPTION_ID=VALUE`` strings from argparse. + + Returns: + Mapping of option_id to answer string (later values win on duplicates). + + Raises: + ValueError: When a token has no ``=`` or an empty option id. + """ + answers: dict[str, str] = {} + for pair in pairs: + if "=" not in pair: + raise ValueError(f"expected OPTION_ID=VALUE, got {pair!r}") + key, value = pair.split("=", 1) + key = key.strip() + if not key: + raise ValueError(f"empty option id in {pair!r}") + answers[key] = value + return answers + + +def _resolve_modify_outputs(args: argparse.Namespace) -> tuple[Path, Path] | None: + """Resolves the (stamped_report_path, configuration_json_path) for ``modify``. + + Honors explicit ``--out`` / ``--config-out`` overrides, otherwise derives both + from ``--output-dir`` (stamped -> ``.work/``, configuration.json -> ``metadata/``). + Returns ``None`` when neither ``--output-dir`` nor ``--out`` was supplied. + """ + output_dir: Path | None = args.output_dir + stamped = args.out + if stamped is None: + if output_dir is None: + return None + stamped = output_dir / ".work" / "translation_report.stamped.json" + config_out = args.config_out + if config_out is None: + if output_dir is not None: + config_out = output_dir / "metadata" / "configuration.json" + elif stamped.parent.name == ".work": + config_out = stamped.parent.parent / "metadata" / "configuration.json" + else: + config_out = stamped.parent / "configuration.json" + return stamped, config_out + + +def _run_modify(args: argparse.Namespace) -> int: + """Implements the ``modify`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``report``, ``answers``, + ``out``, and the optional ``lookup_values``. + + Returns: + ``0`` when the modified IR was written successfully, ``1`` when + the report could not be loaded, ``2`` when the answers failed + validation. + """ + outputs = _resolve_modify_outputs(args) + if outputs is None: + print("modify requires --output-dir (or an explicit --out)", file=sys.stderr) + return 2 + stamped_out, config_out = outputs + + pipelines = _load_pipelines(args.report) + if pipelines is None: + return 1 + try: + answers = _parse_answer_args(args.answer or []) + configuration = _configuration_from_answers(answers) + except ValueError as error: + print(f"Invalid answers: {error}", file=sys.stderr) + return 2 + try: + lookup_values = _parse_csv_source(args.lookup_csv) if args.lookup_csv else [] + except ValueError as error: + print(f"Invalid --lookup-csv: {error}", file=sys.stderr) + return 2 + + stamped_pipelines = [ + _stamp_lookup_values_into_metadata_driven_motifs(apply_configuration(pipeline, configuration), lookup_values) + for pipeline in pipelines + ] + # Prompt-time provisioning: create/reuse the Databricks notification destination for any non-email + # activity_and_notify spec now, so its resolved id is baked into the report (email needs none). + provisioned_pipelines = [] + for pipeline in stamped_pipelines: + provisioned, messages = provision_notification_destinations(pipeline) + provisioned_pipelines.append(provisioned) + for message in messages: + print(message, file=sys.stderr) + from flowx.translator.engine import _pipeline_to_dict # lazy: heavy import (sqlglot) + + modified = [_pipeline_to_dict(pipeline) for pipeline in provisioned_pipelines] + _write_modified_report(args.report, modified, stamped_out) + + # Persist the collected answers as the kept configuration record. + config_out.parent.mkdir(parents=True, exist_ok=True) + config_out.write_text(json.dumps(answers, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"Wrote stamped IR to {stamped_out}", file=sys.stderr) + print(f"Wrote configuration to {config_out}", file=sys.stderr) + return 0 + + +def _run_materialize_lookup(args: argparse.Namespace) -> int: + """Implements the ``materialize-lookup`` subcommand. + + Args: + args: Parsed CLI namespace carrying ``source`` (file path or + literal CSV string) and ``out``. + + Returns: + ``0`` when the JSON was written successfully, ``2`` when the + source could not be parsed as CSV. + """ + try: + rows = _parse_csv_source(args.source) + except ValueError as error: + print(f"Invalid CSV source: {error}", file=sys.stderr) + return 2 + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _parse_csv_source(source: str) -> list[dict[str, str]]: + """Parses a CSV file path or literal CSV string into a list of row dicts. + + Args: + source: Either a path to a CSV file or a literal CSV string with + a header row. + + Returns: + List of dicts, one per data row, keyed by the header names. + + Raises: + ValueError: When the CSV has no header row or is empty. + """ + import csv + + source_path = Path(source) + text = source_path.read_text(encoding="utf-8") if source_path.exists() else source + reader = csv.DictReader(text.splitlines()) + if reader.fieldnames is None: + raise ValueError("Source CSV is empty or missing a header row") + return [dict(row) for row in reader] + + +def _stamp_lookup_values_into_metadata_driven_motifs(pipeline, lookup_values: list[dict[str, Any]]): + """Stamps lookup values onto every metadata-driven motif marked for consolidation. + + Args: + pipeline: Configuration-stamped pipeline IR. + lookup_values: Rows materialised by the agent or the user. + + Returns: + A new :class:`Pipeline` whose metadata-driven motif activities + carry the supplied lookup rows. When *lookup_values* is empty + the pipeline is returned unchanged. + """ + if not lookup_values: + return pipeline + import dataclasses as _dataclasses + + from flowx.models.ir import MotifActivity as _MotifActivity + + stamped_tasks = [] + for task in pipeline.tasks: + if isinstance(task, _MotifActivity) and task.consolidate_metadata_driven: + stamped_tasks.append(_dataclasses.replace(task, lookup_values=list(lookup_values))) + else: + stamped_tasks.append(task) + return _dataclasses.replace(pipeline, tasks=stamped_tasks) + + +def _load_pipelines(report_path: Path) -> list[Any] | None: + """Loads every pipeline IR contained in a report file. + + Args: + report_path: Path to a translation report or pipeline IR JSON. + + Returns: + List of rehydrated :class:`Pipeline` objects, or ``None`` when + the file could not be parsed. + """ + from flowx.bundler.dab_writer import pipeline_dict_to_ir # lazy: heavy import, only inspect/modify + + try: + raw = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + print(f"Failed to read {report_path}: {error}", file=sys.stderr) + return None + pipeline_dicts = _extract_pipeline_dicts(raw) + return [pipeline_dict_to_ir(pipeline_dict)[0] for pipeline_dict in pipeline_dicts] + + +def _extract_pipeline_dicts(raw: Any) -> list[dict[str, Any]]: + """Normalises a translation report into a list of pipeline IR dicts. + + Args: + raw: Parsed JSON content from a report file. + + Returns: + List of dicts, each in the shape ``engine._pipeline_to_dict`` + produces. Empty when *raw* does not contain a recognisable + pipeline payload. + """ + # Single pipeline IR dict (has both "tasks" and "name" at top level) + if isinstance(raw, dict) and "tasks" in raw and "name" in raw: + return [raw] + # Multi-pipeline wrapper written by engine.py ({"pipelines": [...]}) + if isinstance(raw, dict) and "pipelines" in raw and isinstance(raw["pipelines"], list): + return [p for p in raw["pipelines"] if isinstance(p, dict) and "tasks" in p and "name" in p] + # Legacy aggregated translation report shape + if isinstance(raw, dict) and "translations" in raw: + return [ + {"name": entry["pipeline"], **entry["ir"]} + for entry in raw.get("translations", []) + if entry.get("status") == "translated" and entry.get("ir") + ] + return [] + + +def _configuration_from_answers(answers: dict[str, str]) -> TranslationConfiguration: + """Builds a :class:`TranslationConfiguration` from a validated answers dict. + + Args: + answers: Validated mapping of option_id to answer string. + + Returns: + Configuration with every answered field overridden and every + unanswered field defaulted. + + Raises: + ValueError: When an answer is not in the allowed set for its + option. + """ + validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} + motif_consolidations: dict[str, MotifConsolidate] = {} + for qid, value in validated.items(): + if qid.startswith(MOTIF_CONSOLIDATE_OPTION_PREFIX): + motif_consolidations[qid[len(MOTIF_CONSOLIDATE_OPTION_PREFIX) :]] = MotifConsolidate(value) + return TranslationConfiguration( + copy_activity_paradigm=CopyActivityParadigm( + validated.get("copy_activity_paradigm", DEFAULT_CONFIGURATION.copy_activity_paradigm) + ), + non_databricks_task_compute=NonDatabricksTaskCompute( + validated.get("non_databricks_task_compute", DEFAULT_CONFIGURATION.non_databricks_task_compute) + ), + use_lakeflow_connectors=UseLakeflowConnectors( + validated.get("use_lakeflow_connectors", DEFAULT_CONFIGURATION.use_lakeflow_connectors) + ), + lakeflow_connector_type=LakeflowConnectorType( + validated.get("lakeflow_connector_type", DEFAULT_CONFIGURATION.lakeflow_connector_type) + ), + metadata_driven_consolidate=MetadataDrivenConsolidate( + validated.get("metadata_driven_consolidate", DEFAULT_CONFIGURATION.metadata_driven_consolidate) + ), + metadata_driven_access=MetadataDrivenAccess( + validated.get("metadata_driven_access", DEFAULT_CONFIGURATION.metadata_driven_access) + ), + metadata_driven_size=MetadataDrivenSize( + validated.get("metadata_driven_size", DEFAULT_CONFIGURATION.metadata_driven_size) + ), + metadata_driven_lookup_tool=MetadataDrivenLookupTool( + validated.get("metadata_driven_lookup_tool", DEFAULT_CONFIGURATION.metadata_driven_lookup_tool) + ), + notify_destination=NotifyDestination( + validated.get("notify_destination", DEFAULT_CONFIGURATION.notify_destination) + ), + notify_events=NotifyEvents(validated.get("notify_events", DEFAULT_CONFIGURATION.notify_events)), + notify_destination_name=validated.get("notify_destination_name", ""), + notify_args=collect_notify_args(validated), + motif_consolidations=motif_consolidations, + ) + + +def _emit_json(payload: dict[str, Any], out: Path | None) -> None: + """Writes a JSON payload to a file or to stdout. + + Args: + payload: JSON-serialisable mapping to emit. + out: Destination path; ``None`` selects stdout. + """ + encoded = json.dumps(payload, indent=2, default=str) + if out is None: + print(encoded) + return + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(encoded + "\n", encoding="utf-8") + + +def _write_modified_report(report_path: Path, pipelines: list[dict[str, Any]], out: Path) -> None: + """Writes the configuration-stamped IR to *out* using the input report's shape. + + Args: + report_path: Path the modified report was sourced from. Used + only to detect whether the input was a single pipeline IR + or an aggregated translation report. + pipelines: Stamped pipeline IR dicts to write. + out: Destination path for the modified report. + """ + raw = json.loads(report_path.read_text(encoding="utf-8")) + if isinstance(raw, dict) and "translations" in raw: + by_name = {pipeline["name"]: pipeline for pipeline in pipelines} + for entry in raw.get("translations", []): + stamped = by_name.get(entry.get("pipeline")) + if stamped is not None and entry.get("ir") is not None: + entry["ir"] = {key: value for key, value in stamped.items() if key != "name"} + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(raw, indent=2, default=str) + "\n", encoding="utf-8") + return + payload = pipelines[0] if len(pipelines) == 1 else {"pipelines": pipelines} + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/orchestra/adapter/constants.py b/src/flowx/adapter/constants.py similarity index 51% rename from src/orchestra/adapter/constants.py rename to src/flowx/adapter/constants.py index 40c3e3c..02e30aa 100644 --- a/src/orchestra/adapter/constants.py +++ b/src/flowx/adapter/constants.py @@ -1,36 +1,31 @@ """String constants shared across the adapter and its bundler consumers. -Every adapter-side string the modifier stamps onto an IR field or that -the bundler reads back from one is defined here. Modules in -``flowx.adapter``, ``flowx.bundler``, and the test suite import -from this module to avoid string-literal drift between the producer and -consumer ends of the same value. +Defining each stamped/read-back value here keeps the producer (the modifier) and consumer (the +bundler) ends from drifting on string literals. """ from __future__ import annotations from typing import Final -QUESTION_COPY_ACTIVITY_PARADIGM: Final[str] = "copy_activity_paradigm" -QUESTION_NON_DATABRICKS_TASK_COMPUTE: Final[str] = "non_databricks_task_compute" -QUESTION_USE_LAKEFLOW_CONNECTORS: Final[str] = "use_lakeflow_connectors" -QUESTION_LAKEFLOW_CONNECTOR_TYPE: Final[str] = "lakeflow_connector_type" -QUESTION_METADATA_DRIVEN_CONSOLIDATE: Final[str] = "metadata_driven_consolidate" -QUESTION_METADATA_DRIVEN_ACCESS: Final[str] = "metadata_driven_access" -QUESTION_METADATA_DRIVEN_SIZE: Final[str] = "metadata_driven_size" -QUESTION_METADATA_DRIVEN_LOOKUP_TOOL: Final[str] = "metadata_driven_lookup_tool" - -# Per-detected-motif consolidation question_ids carry the motif_id as a suffix -# (e.g. ``consolidate_motif:rest_api_pagination``) so each detected motif gets -# its own question. Validation strips the prefix and validates the answer -# against the :class:`MotifConsolidate` enum. -MOTIF_CONSOLIDATE_QUESTION_PREFIX: Final[str] = "consolidate_motif:" +OPTION_COPY_ACTIVITY_PARADIGM: Final[str] = "copy_activity_paradigm" +OPTION_NON_DATABRICKS_TASK_COMPUTE: Final[str] = "non_databricks_task_compute" +OPTION_USE_LAKEFLOW_CONNECTORS: Final[str] = "use_lakeflow_connectors" +OPTION_LAKEFLOW_CONNECTOR_TYPE: Final[str] = "lakeflow_connector_type" +OPTION_METADATA_DRIVEN_CONSOLIDATE: Final[str] = "metadata_driven_consolidate" +OPTION_METADATA_DRIVEN_ACCESS: Final[str] = "metadata_driven_access" +OPTION_METADATA_DRIVEN_SIZE: Final[str] = "metadata_driven_size" +OPTION_METADATA_DRIVEN_LOOKUP_TOOL: Final[str] = "metadata_driven_lookup_tool" + +# Per-motif consolidation option ids suffix the motif_id (e.g. consolidate_motif:rest_api_pagination) +# so each motif gets its own option; validation strips this prefix and checks against MotifConsolidate. +MOTIF_CONSOLIDATE_OPTION_PREFIX: Final[str] = "consolidate_motif:" METADATA_DRIVEN_MOTIF_ID: Final[str] = "metadata_driven_bulk_copy" -PHASE_INGEST: Final[str] = "ingest" -PHASE_TRANSLATE: Final[str] = "translate" -PHASE_PREPARE: Final[str] = "prepare" +PHASE_DISCOVER: Final[str] = "discover" +PHASE_CONVERT: Final[str] = "convert" +PHASE_PACKAGE: Final[str] = "package" INPUT_ADF_SOURCE_PATH: Final[str] = "adf_source_path" INPUT_ADF_RESOURCE_URL: Final[str] = "adf_resource_url" @@ -42,6 +37,9 @@ INPUT_SCHEMA: Final[str] = "schema" INPUT_BUNDLE_NAME: Final[str] = "bundle_name" INPUT_DATABRICKS_PROFILE: Final[str] = "databricks_profile" +INPUT_RESULTS_TABLE: Final[str] = "results_table" +INPUT_RESULTS_WAREHOUSE: Final[str] = "results_warehouse_id" +INPUT_INSTALL_DASHBOARD: Final[str] = "install_dashboard" LAKEFLOW_CONNECTOR_TYPE_QUERY_BASED: Final[str] = "query_based" LAKEFLOW_CONNECTOR_TYPE_CDC: Final[str] = "cdc" @@ -77,3 +75,9 @@ ) DATABASE_SOURCE_TYPE_HINT: Final[str] = "database" + +OPTION_NOTIFY_DESTINATION: Final[str] = "notify_destination" +OPTION_NOTIFY_EVENTS: Final[str] = "notify_events" +OPTION_NOTIFY_DESTINATION_NAME: Final[str] = "notify_destination_name" +# Per-field notification follow-up option ids (e.g. notify_email_recipients, +# notify_slack_url) are defined by the _NOTIFY_FIELDS registry in operations.py. diff --git a/src/orchestra/adapter/models.py b/src/flowx/adapter/models.py similarity index 72% rename from src/orchestra/adapter/models.py rename to src/flowx/adapter/models.py index 70f5abc..a54e626 100644 --- a/src/orchestra/adapter/models.py +++ b/src/flowx/adapter/models.py @@ -34,7 +34,7 @@ class LakeflowConnectorType(StrEnum): Used only when ``use_lakeflow_connectors`` is ``lakeflow_connect``. The modifier still routes Copy activities that read from a SQL query into - the query-based connector regardless of this preference; this enum + the query-based connector regardless of this configuration; this enum controls the default for table-based Copy activities. """ @@ -91,6 +91,32 @@ class MotifConsolidate(StrEnum): CONSOLIDATE = "consolidate" +class NotifyDestination(StrEnum): + """How an activity->Notify (activity_and_notify) motif's notifications are handled. + + ``KEEP`` preserves the current behaviour (the WebActivity notify + activities translate directly; the motif is not collapsed). Any other + value collapses the motif: the upstream activity (Copy, Notebook, Lookup, + …) becomes the task and the downstream notifications become Databricks + job-task notifications routed to the chosen destination. + """ + + KEEP = "keep" + EMAIL = "email" + SLACK = "slack" + TEAMS = "teams" + PAGERDUTY = "pagerduty" + WEBHOOK = "webhook" + + +class NotifyEvents(StrEnum): + """Which job-task events fire the collapsed notification.""" + + ON_FAILURE = "on_failure" + ON_SUCCESS = "on_success" + BOTH = "both" + + FIELD_TO_ENUM: Final[MappingProxyType[str, type[StrEnum]]] = MappingProxyType( { "copy_activity_paradigm": CopyActivityParadigm, @@ -101,12 +127,14 @@ class MotifConsolidate(StrEnum): "metadata_driven_access": MetadataDrivenAccess, "metadata_driven_size": MetadataDrivenSize, "metadata_driven_lookup_tool": MetadataDrivenLookupTool, + "notify_destination": NotifyDestination, + "notify_events": NotifyEvents, } ) @dataclass(frozen=True, slots=True, kw_only=True) -class TranslationPreferences: +class TranslationConfiguration: """Snapshot of user choices that shape downstream IR transformations. Each field accepts either a raw string or the corresponding enum @@ -122,11 +150,10 @@ class TranslationPreferences: value is a partial mapping of the fields above; only the keys present win over the pipeline-wide defaults. - ADF DatabricksNotebook and DatabricksSparkPython tasks always keep - the cluster binding derived from the source linked service -- the - serverless replacement option was removed because it silently - discarded init scripts and DBR-version constraints that the source - pipeline relied on. + Notes: + ADF DatabricksNotebook and DatabricksSparkPython tasks always keep the cluster binding from + the source linked service; the serverless-replacement option was removed because it silently + discarded init scripts and DBR-version constraints the source pipeline relied on. """ copy_activity_paradigm: CopyActivityParadigm = CopyActivityParadigm.NOTEBOOK @@ -137,6 +164,12 @@ class TranslationPreferences: metadata_driven_access: MetadataDrivenAccess = MetadataDrivenAccess.NO metadata_driven_size: MetadataDrivenSize = MetadataDrivenSize.LARGE metadata_driven_lookup_tool: MetadataDrivenLookupTool = MetadataDrivenLookupTool.NONE + notify_destination: NotifyDestination = NotifyDestination.KEEP + notify_events: NotifyEvents = NotifyEvents.BOTH + notify_destination_name: str = "" + # SDK config kwargs for the chosen destination (e.g. addresses/url/integration_key), keyed by SDK + # arg name; populated from the per-field follow-up answers via collect_notify_args. + notify_args: dict[str, str] = field(default_factory=dict) motif_consolidations: dict[str, MotifConsolidate] = field(default_factory=dict) per_task: dict[str, dict[str, str]] = field(default_factory=dict) @@ -151,29 +184,27 @@ def __post_init__(self) -> None: value = getattr(self, field_name) if not isinstance(value, enum_cls): object.__setattr__(self, field_name, enum_cls(value)) - # motif_consolidations is keyed by dynamic motif_id rather than a - # fixed field name, so it is not in FIELD_TO_ENUM. Coerce its - # values to MotifConsolidate members here. + # motif_consolidations is keyed by dynamic motif_id (not in FIELD_TO_ENUM), so coerce here. coerced: dict[str, MotifConsolidate] = {} for motif_id, choice in self.motif_consolidations.items(): coerced[motif_id] = choice if isinstance(choice, MotifConsolidate) else MotifConsolidate(choice) object.__setattr__(self, "motif_consolidations", coerced) - def effective_for(self, task_key: str) -> TranslationPreferences: - """Returns a preferences view where per-task overrides for *task_key* win. + def effective_for(self, task_key: str) -> TranslationConfiguration: + """Returns a configuration view where per-task overrides for *task_key* win. Args: task_key: Sanitised task key of the activity being prepared. Returns: - A new :class:`TranslationPreferences` with overrides for + A new :class:`TranslationConfiguration` with overrides for *task_key* applied on top of the pipeline-wide values, or ``self`` unchanged when no overrides exist for *task_key*. """ override = self.per_task.get(task_key) if not override: return self - return TranslationPreferences( + return TranslationConfiguration( copy_activity_paradigm=CopyActivityParadigm( override.get("copy_activity_paradigm", self.copy_activity_paradigm) ), @@ -201,12 +232,12 @@ def effective_for(self, task_key: str) -> TranslationPreferences: ) -DEFAULT_PREFERENCES: Final[TranslationPreferences] = TranslationPreferences() +DEFAULT_CONFIGURATION: Final[TranslationConfiguration] = TranslationConfiguration() @dataclass(frozen=True, slots=True, kw_only=True) -class QuestionOption: - """One allowed answer to a :class:`TranslationQuestion`. +class OptionChoice: + """One allowed answer to a :class:`TranslationOption`. Attributes: value: Machine-readable identifier matching the backing enum member. @@ -221,63 +252,63 @@ class QuestionOption: @dataclass(frozen=True, slots=True, kw_only=True) -class TranslationQuestion: - """A single just-in-time question raised by the IR inspector. +class TranslationOption: + """A single just-in-time option raised by the IR inspector. Attributes: - question_id: Stable identifier matching the preferences field. - prompt: Human-readable question text. - rationale: One- or two-sentence explanation of why the question + option_id: Stable identifier matching the configuration field. + prompt: Human-readable option text. + rationale: One- or two-sentence explanation of why the option is being raised. options: Allowed answers; the first option is the conservative default and is also exposed via ``default``. affected_task_keys: Activity task keys impacted by the answer. - default: Default value applied when the caller skips the question. - conditions: Tuples of ``(question_id, expected_value)`` that must + default: Default value applied when the caller skips the option. + conditions: Tuples of ``(option_id, expected_value)`` that must already be answered with the expected value before this - question surfaces. An empty tuple means the question is + option surfaces. An empty tuple means the option is evaluated solely on its IR/motif preconditions. """ - question_id: str + option_id: str prompt: str rationale: str - options: tuple[QuestionOption, ...] + options: tuple[OptionChoice, ...] affected_task_keys: tuple[str, ...] default: str conditions: tuple[tuple[str, str], ...] = () @dataclass(slots=True, kw_only=True) -class PendingQuestions: - """Outstanding questions for a single pipeline translation. +class PendingOptions: + """Outstanding options for a single pipeline translation. Attributes: - pipeline_name: Name of the pipeline these questions belong to. - questions: Ordered list of questions still awaiting an answer. + pipeline_name: Name of the pipeline these options belong to. + options: Ordered list of options still awaiting an answer. """ pipeline_name: str - questions: list[TranslationQuestion] = field(default_factory=list) + options: list[TranslationOption] = field(default_factory=list) @dataclass(frozen=True, slots=True, kw_only=True) -class MigrationInputQuestion: +class MigrationInputOption: """A free-text input gathered before an flowx phase runs. Attributes: - question_id: Stable identifier the skill uses to key the answer. - prompt: Human-readable question text. + option_id: Stable identifier the skill uses to key the answer. + prompt: Human-readable option text. description: One-sentence explanation of what the value is used for and what shape is expected (path, URL, identifier). default: Default value applied when the caller skips the - question; ``None`` when the field is required and has no + option; ``None`` when the field is required and has no sensible default. required: When ``True`` the skill must collect a value; when ``False`` the default (which may be ``None``) is permitted. """ - question_id: str + option_id: str prompt: str description: str default: str | None = None @@ -286,13 +317,13 @@ class MigrationInputQuestion: @dataclass(slots=True, kw_only=True) class PendingMigrationInputs: - """Outstanding migration-phase input questions for a single phase. + """Outstanding migration-phase input options for a single phase. Attributes: - phase: The migration phase name (``"ingest"``, ``"translate"``, - ``"prepare"``). - questions: Ordered list of questions still awaiting an answer. + phase: The migration phase name (``"discover"``, ``"convert"``, + ``"package"``). + options: Ordered list of options still awaiting an answer. """ phase: str - questions: list[MigrationInputQuestion] = field(default_factory=list) + options: list[MigrationInputOption] = field(default_factory=list) diff --git a/src/orchestra/adapter/operations.py b/src/flowx/adapter/operations.py similarity index 51% rename from src/orchestra/adapter/operations.py rename to src/flowx/adapter/operations.py index fe21cbc..fb292fe 100644 --- a/src/orchestra/adapter/operations.py +++ b/src/flowx/adapter/operations.py @@ -1,7 +1,7 @@ -"""Standalone operations: question gathering, validation, and IR modification. +"""Standalone operations: option gathering, validation, and IR modification. The agent adapter and the CLI bridge call into these functions; nothing -here is stateful. Preference dataclasses, StrEnums, and question shapes +here is stateful. Configuration dataclasses, StrEnums, and option shapes live in :mod:`flowx.adapter.models`. """ @@ -23,14 +23,17 @@ LAKEFLOW_CONNECT_REPLACEMENT, LAKEFLOW_CONNECTOR_TYPE_QUERY_BASED, METADATA_DRIVEN_MOTIF_ID, - MOTIF_CONSOLIDATE_QUESTION_PREFIX, - QUESTION_COPY_ACTIVITY_PARADIGM, - QUESTION_METADATA_DRIVEN_ACCESS, - QUESTION_METADATA_DRIVEN_CONSOLIDATE, - QUESTION_METADATA_DRIVEN_LOOKUP_TOOL, - QUESTION_METADATA_DRIVEN_SIZE, - QUESTION_NON_DATABRICKS_TASK_COMPUTE, - QUESTION_USE_LAKEFLOW_CONNECTORS, + MOTIF_CONSOLIDATE_OPTION_PREFIX, + OPTION_COPY_ACTIVITY_PARADIGM, + OPTION_METADATA_DRIVEN_ACCESS, + OPTION_METADATA_DRIVEN_CONSOLIDATE, + OPTION_METADATA_DRIVEN_LOOKUP_TOOL, + OPTION_METADATA_DRIVEN_SIZE, + OPTION_NON_DATABRICKS_TASK_COMPUTE, + OPTION_NOTIFY_DESTINATION, + OPTION_NOTIFY_DESTINATION_NAME, + OPTION_NOTIFY_EVENTS, + OPTION_USE_LAKEFLOW_CONNECTORS, ) from flowx.adapter.models import ( FIELD_TO_ENUM, @@ -42,10 +45,12 @@ MetadataDrivenSize, MotifConsolidate, NonDatabricksTaskCompute, - PendingQuestions, - QuestionOption, - TranslationPreferences, - TranslationQuestion, + NotifyDestination, + NotifyEvents, + OptionChoice, + PendingOptions, + TranslationConfiguration, + TranslationOption, UseLakeflowConnectors, ) from flowx.adapter.predicates import ( @@ -59,65 +64,73 @@ from flowx.models.ir import ( Activity, CopyActivity, + Dependency, ForEachActivity, IfConditionActivity, MotifActivity, Pipeline, SwitchActivity, SwitchCase, + WebActivity, ) from flowx.models.motifs import MOTIF_LAKEFLOW_CONNECT_DATABASE +# Free-text option ids (no backing enum) that validate_answer accepts with any value, vs a genuinely +# unknown id which it rejects -- the set is the NOTIFY_FREE_TEXT_OPTION_IDS registry defined below. -def enum_for(question_id: str) -> type[StrEnum] | None: - """Returns the enum class backing a preference field. + +def enum_for(option_id: str) -> type[StrEnum] | None: + """Returns the enum class backing a configuration field. Args: - question_id: Field name (e.g. ``"copy_activity_paradigm"``) or + option_id: Field name (e.g. ``"copy_activity_paradigm"``) or per-motif id (e.g. ``"consolidate_motif:rest_api_pagination"``). Returns: The :class:`StrEnum` subclass that defines the allowed values, or - ``None`` when the question_id is unknown. + ``None`` when the option_id is unknown. """ - if question_id.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): + if option_id.startswith(MOTIF_CONSOLIDATE_OPTION_PREFIX): return MotifConsolidate - return FIELD_TO_ENUM.get(question_id) + return FIELD_TO_ENUM.get(option_id) -def allowed_values_for(question_id: str) -> tuple[str, ...]: - """Returns the allowed string values for a preference field. +def allowed_values_for(option_id: str) -> tuple[str, ...]: + """Returns the allowed string values for a configuration field. Args: - question_id: Field name (e.g. ``"copy_activity_paradigm"``). + option_id: Field name (e.g. ``"copy_activity_paradigm"``). Returns: Tuple of allowed string values in declaration order. Empty when the field is unknown. """ - enum_cls = enum_for(question_id) + enum_cls = enum_for(option_id) return tuple(member.value for member in enum_cls) if enum_cls else () -def validate_answer(question_id: str, value: str) -> str: - """Returns *value* when it is an allowed answer for *question_id*. +def validate_answer(option_id: str, value: str) -> str: + """Returns *value* when it is an allowed answer for *option_id*. Args: - question_id: Stable question identifier. + option_id: Stable option identifier. value: Caller-supplied answer string. Returns: The validated value, unchanged. Raises: - ValueError: When *question_id* is not known or *value* is not in - the allowed set for the question. + ValueError: When *option_id* is not known or *value* is not in + the allowed set for the option. """ - allowed = allowed_values_for(question_id) - if not allowed: - raise ValueError(f"Unknown question_id {question_id!r}") - if value not in allowed: - raise ValueError(f"Invalid answer {value!r} for {question_id!r}; allowed: {sorted(allowed)}") + allowed = allowed_values_for(option_id) + if allowed: + if value not in allowed: + raise ValueError(f"Invalid answer {value!r} for {option_id!r}; allowed: {sorted(allowed)}") + return value + if option_id in NOTIFY_FREE_TEXT_OPTION_IDS: + return value + raise ValueError(f"Unknown option_id {option_id!r}") return value @@ -213,118 +226,539 @@ def _walk_workspace_paths(tasks: list[dict[str, Any]] | None, candidates: list[s _walk_workspace_paths(task.get("default_activities"), candidates) -def gather_questions( +def gather_options( pipeline: Pipeline, motifs: list | None = None, *, answers: dict[str, str] | None = None, -) -> PendingQuestions: - """Walks the IR and returns the questions that apply to *pipeline*. +) -> PendingOptions: + """Walks the IR and returns the options that apply to *pipeline*. Args: pipeline: Translated pipeline IR after motif collapsing. motifs: Detected motifs, used to surface the Lakeflow Connect - question for multi-step database ingestion patterns. - answers: Answers the caller has already collected. Questions - whose ``question_id`` is in this mapping are filtered out, - and questions whose ``conditions`` reference earlier answers + option for multi-step database ingestion patterns. + answers: Answers the caller has already collected. Options + whose ``option_id`` is in this mapping are filtered out, + and options whose ``conditions`` reference earlier answers are evaluated against this mapping. Returns: - A :class:`PendingQuestions` instance carrying the questions whose + A :class:`PendingOptions` instance carrying the options whose IR preconditions and answer-dependent conditions are met but - whose ``question_id`` has not yet been answered. + whose ``option_id`` has not yet been answered. """ motif_list = motifs or [] answer_map = answers or {} builders = ( - _build_use_lakeflow_connectors_question, - _build_lakeflow_connector_type_question, - _build_copy_activity_paradigm_question, - _build_non_databricks_task_compute_question, - _build_metadata_driven_consolidate_question, - _build_metadata_driven_access_question, - _build_metadata_driven_size_question, - _build_metadata_driven_lookup_tool_question, + _build_use_lakeflow_connectors_option, + _build_lakeflow_connector_type_option, + _build_copy_activity_paradigm_option, + _build_non_databricks_task_compute_option, + _build_metadata_driven_consolidate_option, + _build_metadata_driven_access_option, + _build_metadata_driven_size_option, + _build_metadata_driven_lookup_tool_option, ) candidates = (builder(pipeline, motif_list, answers=answer_map) for builder in builders) pending = [ - question - for question in candidates - if question is not None - and question.question_id not in answer_map - and _conditions_met(question.conditions, answer_map) + option + for option in candidates + if option is not None and option.option_id not in answer_map and _conditions_met(option.conditions, answer_map) ] - # Per-motif "consolidate?" questions: one per detected motif. Each - # gets its own question_id ``consolidate_motif:`` so the - # adapter can solicit and validate them independently. Default is - # ``keep`` -- nothing is collapsed without an explicit yes. - for motif_question in _build_motif_consolidation_questions(motif_list): - if motif_question.question_id in answer_map: + # Per-motif consolidate options: one per detected motif (consolidate_motif:), default keep. + for motif_option in _build_motif_consolidation_options(motif_list): + if motif_option.option_id in answer_map: + continue + pending.append(motif_option) + # activity->Notify chain: destination choice, then one follow-up per SDK field once chosen. + for notify_option in _build_notify_options(pipeline, answer_map): + if notify_option.option_id in answer_map: continue - pending.append(motif_question) - return PendingQuestions(pipeline_name=pipeline.name, questions=pending) + pending.append(notify_option) + return PendingOptions(pipeline_name=pipeline.name, options=pending) def _conditions_met(conditions: tuple[tuple[str, str], ...], answers: dict[str, str]) -> bool: """Returns True when every condition is satisfied by *answers*. Args: - conditions: Tuples of ``(question_id, expected_value)`` from a - :class:`TranslationQuestion`. - answers: Mapping of question_id to the caller-supplied answer. + conditions: Tuples of ``(option_id, expected_value)`` from a + :class:`TranslationOption`. + answers: Mapping of option_id to the caller-supplied answer. Returns: - ``True`` when every condition's question has been answered with + ``True`` when every condition's option has been answered with the expected value (or when *conditions* is empty); ``False`` otherwise. """ return all(answers.get(qid) == expected for qid, expected in conditions) -def apply_preferences(pipeline: Pipeline, pipeline_preferences: TranslationPreferences) -> Pipeline: - """Returns a copy of *pipeline* with preferences stamped onto each activity. +# Notify destinations other than KEEP — the destination-name and events follow-ups apply to all of them. +_NOTIFY_NON_KEEP: tuple[str, ...] = tuple(d.value for d in NotifyDestination if d is not NotifyDestination.KEEP) + + +def _show_when_from_conditions(conditions: tuple[tuple[str, str], ...]) -> list[dict[str, Any]]: + """Translates a TranslationOption's equality ``conditions`` into ``show_when`` clauses.""" + return [{"option_id": qid, "in": [value]} for qid, value in conditions] + + +def _option_schema(option: TranslationOption, show_when: list[dict[str, Any]]) -> dict[str, Any]: + """Serialises one option into a declarative schema entry the agent can walk locally.""" + return { + "option_id": option.option_id, + "prompt": option.prompt, + "rationale": option.rationale, + "choices": [ + {"value": choice.value, "label": choice.label, "description": choice.description} + for choice in option.options + ], + "free_text": not option.options, + "default": option.default, + "affected_task_keys": list(option.affected_task_keys), + "show_when": show_when, + } + + +def _build_notify_schema(pipeline: Pipeline) -> list[dict[str, Any]]: + """The full activity_and_notify chain as schema entries (every destination's follow-ups).""" + affected = _notify_present(pipeline) + if not affected: + return [] + entries = [_option_schema(_build_notify_destination_option(affected), [])] + for dest, fields in _NOTIFY_FIELDS.items(): + dest_clause = [{"option_id": OPTION_NOTIFY_DESTINATION, "in": [dest]}] + entries.extend(_option_schema(_notify_field_option(dest, field, affected), dest_clause) for field in fields) + non_keep_clause = [{"option_id": OPTION_NOTIFY_DESTINATION, "in": list(_NOTIFY_NON_KEEP)}] + entries.append(_option_schema(_notify_name_option(affected), non_keep_clause)) + entries.append(_option_schema(_build_notify_events_option(affected), non_keep_clause)) + return entries + + +def build_option_schema(pipeline: Pipeline, motifs: list | None = None) -> list[dict[str, Any]]: + """Returns the full declarative option tree for *pipeline* as schema dicts. + + Unlike :func:`gather_options` -- which filters to the options *currently* pending given the + answers collected so far -- this returns **every** option the pipeline can raise, each annotated + with a ``show_when`` condition: a conjunction of ``{"option_id", "in": [values]}`` clauses (empty + list = always shown). The agent walks this tree locally, asking only the options whose + ``show_when`` clauses are all satisfied by the answers gathered so far, so a multi-step chain + (e.g. notify destination -> per-field follow-ups, or metadata-driven consolidate -> access -> + size -> lookup-tool) needs no per-follow-up round trip back to the server. ``apply_answers`` + remains the single validate-and-apply path. + """ + motif_list = motifs or [] + schema: list[dict[str, Any]] = [] + builders = ( + _build_use_lakeflow_connectors_option, + _build_lakeflow_connector_type_option, + _build_copy_activity_paradigm_option, + _build_non_databricks_task_compute_option, + _build_metadata_driven_consolidate_option, + _build_metadata_driven_access_option, + _build_metadata_driven_size_option, + _build_metadata_driven_lookup_tool_option, + ) + for builder in builders: + option = builder(pipeline, motif_list, answers={}) + if option is not None: + schema.append(_option_schema(option, _show_when_from_conditions(option.conditions))) + for motif_option in _build_motif_consolidation_options(motif_list): + schema.append(_option_schema(motif_option, _show_when_from_conditions(motif_option.conditions))) + schema.extend(_build_notify_schema(pipeline)) + return schema + + +def apply_configuration(pipeline: Pipeline, pipeline_configuration: TranslationConfiguration) -> Pipeline: + """Returns a copy of *pipeline* with configuration stamped onto each activity. Args: pipeline: Translated pipeline IR after motif collapsing. - pipeline_preferences: Validated pipeline-wide preferences. + pipeline_configuration: Validated pipeline-wide configuration. Returns: A new :class:`Pipeline` whose activities carry concrete decisions about compute, target format, and Lakeflow Connect replacement. The input pipeline is not mutated. """ - stamped_tasks = [_stamp_activity(activity, pipeline_preferences) for activity in pipeline.tasks] - return dataclasses.replace( + stamped_tasks = [_stamp_activity(activity, pipeline_configuration) for activity in pipeline.tasks] + stamped = dataclasses.replace( pipeline, tasks=stamped_tasks, - translation_preferences=pipeline_preferences, + translation_configuration=pipeline_configuration, + ) + if pipeline_configuration.notify_destination is not NotifyDestination.KEEP: + stamped = _collapse_notify(stamped, pipeline_configuration) + return stamped + + +def provision_notification_destinations(pipeline: Pipeline) -> tuple[Pipeline, list[str]]: + """Create the Databricks notification destinations for *pipeline* at prompt time. + + Walks the collapsed tasks and, for every non-email ``activity_and_notify`` + notification spec, creates (or reuses) the destination via the SDK now and stamps + the resolved ``destination_id`` back onto the task. Email specs are left untouched + -- email wires raw ``email_notifications`` and needs no destination. This runs in + the adapter ``modify`` phase so the destination exists (and validates) as soon as + the user answers, rather than at prepare time. + + Returns: + ``(pipeline, messages)`` -- a copy of *pipeline* with resolved ids stamped in, + and human-readable status lines (one per destination) for surfacing to the user. + When no non-email notifications are present the pipeline is returned unchanged + with an empty message list and no SDK call is made. + """ + from flowx.preparer.notifications import provision_destination + + messages: list[str] = [] + new_tasks: list = [] + changed = False + for task in pipeline.tasks: + spec = getattr(task, "notifications", None) + if spec and spec.get("destination") not in (None, "", "email"): + new_spec, message = provision_destination(spec) + if message: + messages.append(message) + if new_spec is not spec: + task = dataclasses.replace(task, notifications=new_spec) + changed = True + new_tasks.append(task) + if not changed: + return pipeline, messages + return dataclasses.replace(pipeline, tasks=new_tasks), messages + + +_NOTIFY_WEBHOOK_DESTS: frozenset[str] = frozenset({"slack", "teams", "webhook"}) + + +@dataclasses.dataclass(frozen=True, slots=True) +class _NotifyField: + """One Databricks-SDK config field of a notification destination. + + Surfaced as a chained follow-up option after the user picks a destination. + + Attributes: + option_id: Adapter option id for the follow-up question. + sdk_arg: The kwarg on the SDK config class (e.g. ``url``, ``addresses``). + prompt: The question text. + required: Whether the SDK config needs this field for the destination. + is_list: When True the comma-separated answer is split into a list + (used for email ``addresses``). + """ + + option_id: str + sdk_arg: str + prompt: str + required: bool = True + is_list: bool = False + + +# Per-destination SDK config fields (required first), from the SDK notification-destination config +# classes; the adapter chains one follow-up option per field after the destination is chosen. +_NOTIFY_FIELDS: dict[str, tuple[_NotifyField, ...]] = { + NotifyDestination.EMAIL.value: ( + _NotifyField( + "notify_email_recipients", + "addresses", + "Recipient email address(es)? (comma-separated)", + is_list=True, + ), + ), + NotifyDestination.SLACK.value: ( + _NotifyField("notify_slack_url", "url", "Slack incoming webhook URL?"), + _NotifyField("notify_slack_channel_id", "channel_id", "Slack channel id? (optional)", required=False), + _NotifyField("notify_slack_oauth_token", "oauth_token", "Slack OAuth token? (optional)", required=False), + ), + NotifyDestination.TEAMS.value: (_NotifyField("notify_teams_url", "url", "Microsoft Teams incoming webhook URL?"),), + NotifyDestination.PAGERDUTY.value: ( + _NotifyField("notify_pagerduty_integration_key", "integration_key", "PagerDuty integration key?"), + ), + NotifyDestination.WEBHOOK.value: ( + _NotifyField("notify_webhook_url", "url", "Generic webhook URL?"), + _NotifyField("notify_webhook_username", "username", "Webhook basic-auth username? (optional)", required=False), + _NotifyField("notify_webhook_password", "password", "Webhook basic-auth password? (optional)", required=False), + ), +} + +# Free-text follow-up option ids (no backing enum). ``validate_answer`` accepts +# any value for these, distinct from a genuinely unknown option id. +NOTIFY_FREE_TEXT_OPTION_IDS: frozenset[str] = frozenset( + {field.option_id for fields in _NOTIFY_FIELDS.values() for field in fields} | {OPTION_NOTIFY_DESTINATION_NAME} +) + + +def _find_notify_groups(tasks: list) -> dict[str, list[tuple[Any, str]]]: + """Find activity->notify groups in the IR: any non-Web task with WebActivity dependents. + + Returns a mapping of ``upstream_task_key`` -> list of ``(web_activity, outcome)`` where + outcome is the dependency condition (``Succeeded`` / ``Failed`` / ...). + + Any activity -- Copy, Notebook, Lookup, SparkPython, a stored procedure, etc. -- that is directly + followed by one or more WebActivity calls is a candidate: collapsing turns those Web calls into + native Databricks job-task notifications on the upstream task. The upstream is any non-Web task + (a WebActivity that merely follows another WebActivity is not treated as a notify target). + Mirrors the activity_and_notify motif on the (non-collapsed) IR. + """ + upstream = {t.task_key for t in tasks if not isinstance(t, WebActivity)} + groups: dict[str, list[tuple[Any, str]]] = {} + for task in tasks: + if isinstance(task, WebActivity) and task.depends_on: + for dep in task.depends_on: + if dep.task_key in upstream: + groups.setdefault(dep.task_key, []).append((task, dep.outcome or "Succeeded")) + break + return groups + + +def _notify_present(pipeline: Pipeline) -> tuple[str, ...]: + """Task keys of activities that have notify WebActivity dependents.""" + return tuple(sorted(_find_notify_groups(pipeline.tasks).keys())) + + +def _notify_dest(answers: dict[str, str] | None) -> str: + return (answers or {}).get(OPTION_NOTIFY_DESTINATION, "") + + +def _freetext_option(option_id: str, prompt: str, rationale: str, affected: tuple[str, ...]) -> TranslationOption: + """Builds a free-text option (no enum choices).""" + return TranslationOption( + option_id=option_id, + prompt=prompt, + rationale=rationale, + options=(), + affected_task_keys=affected, + default="", ) -def _build_copy_activity_paradigm_question( +def _build_notify_destination_option(affected: tuple[str, ...]) -> TranslationOption: + """Asks whether/how to route an activity->Notify motif to a Databricks destination.""" + return TranslationOption( + option_id=OPTION_NOTIFY_DESTINATION, + prompt=( + "One or more activities are followed by notification Web activities. " + "Route these to a Databricks destination?" + ), + rationale=( + "Choosing a destination collapses the pattern: the upstream activity becomes the task " + "and the downstream notifications become Databricks job-task success/failure " + "notifications (email_notifications or webhook_notifications). The ADF Web activity's own " + "URL/body is not used. Keeping preserves the current per-activity Web activity translation." + ), + options=( + OptionChoice( + value=NotifyDestination.KEEP.value, + label="Keep current behavior", + description="Do not collapse; translate the Web activities directly.", + ), + OptionChoice( + value=NotifyDestination.EMAIL.value, + label="Email", + description="Wire email_notifications with recipient addresses.", + ), + OptionChoice( + value=NotifyDestination.SLACK.value, + label="Slack", + description="Create a Slack notification destination and wire webhook_notifications.", + ), + OptionChoice( + value=NotifyDestination.TEAMS.value, + label="Microsoft Teams", + description="Create a Teams notification destination and wire webhook_notifications.", + ), + OptionChoice( + value=NotifyDestination.PAGERDUTY.value, + label="PagerDuty", + description="Create a PagerDuty notification destination and wire webhook_notifications.", + ), + OptionChoice( + value=NotifyDestination.WEBHOOK.value, + label="Generic Webhook", + description="Create a generic webhook destination and wire webhook_notifications.", + ), + ), + affected_task_keys=affected, + default=NotifyDestination.KEEP.value, + ) + + +def _build_notify_events_option(affected: tuple[str, ...]) -> TranslationOption: + return TranslationOption( + option_id=OPTION_NOTIFY_EVENTS, + prompt="Which events should notify?", + rationale="Defaults to both (whatever the source notify activities covered). Restrict if desired.", + options=( + OptionChoice( + value=NotifyEvents.BOTH.value, + label="Both success and failure", + description="Wire on_success and on_failure (as the source activities had).", + ), + OptionChoice( + value=NotifyEvents.ON_FAILURE.value, label="On failure only", description="Only wire on_failure." + ), + OptionChoice( + value=NotifyEvents.ON_SUCCESS.value, label="On success only", description="Only wire on_success." + ), + ), + affected_task_keys=affected, + default=NotifyEvents.BOTH.value, + ) + + +def _notify_field_option(dest: str, field: _NotifyField, affected: tuple[str, ...]) -> TranslationOption: + """Builds the free-text follow-up for one SDK config field of a notification destination.""" + suffix = "" if field.required else " (optional -- leave blank to skip)" + return _freetext_option( + field.option_id, + field.prompt, + f"Maps to the Databricks SDK {dest} config field `{field.sdk_arg}`.{suffix}", + affected, + ) + + +def _notify_name_option(affected: tuple[str, ...]) -> TranslationOption: + """Builds the optional destination-display-name follow-up.""" + return _freetext_option( + OPTION_NOTIFY_DESTINATION_NAME, + "Display name for the notification destination? (optional; default derived)", + "Reused if a destination with this name already exists, so prepare is idempotent.", + affected, + ) + + +def _build_notify_options(pipeline: Pipeline, answers: dict[str, str] | None) -> list[TranslationOption]: + """Returns the chained activity_and_notify options. + + The first prompt is the destination choice. Once a (non-keep) destination is + answered, one follow-up option is surfaced per SDK field of that destination + (required first; optional fields flagged), so the agent prompts for each field + sequentially, followed by an optional display name and the events selector. + """ + affected = _notify_present(pipeline) + if not affected: + return [] + options: list[TranslationOption] = [_build_notify_destination_option(affected)] + dest = _notify_dest(answers) + if dest in ("", NotifyDestination.KEEP.value): + return options + options.extend(_notify_field_option(dest, field, affected) for field in _NOTIFY_FIELDS.get(dest, ())) + options.append(_notify_name_option(affected)) + options.append(_build_notify_events_option(affected)) + return options + + +def collect_notify_args(answers: dict[str, str]) -> dict[str, str]: + """Collect the per-field notification answers into ``{sdk_arg: value}``. + + Reads only the fields belonging to the chosen destination so that, e.g., the + Slack ``url`` answer and a Webhook ``url`` answer never collide. + """ + dest = answers.get(OPTION_NOTIFY_DESTINATION, "") + args: dict[str, str] = {} + for field in _NOTIFY_FIELDS.get(dest, ()): + value = answers.get(field.option_id) + if value: + args[field.sdk_arg] = value + return args + + +def _notification_spec(config: TranslationConfiguration) -> dict[str, Any]: + """Builds the notification spec stamped onto the collapsed Copy task. + + ``args`` carries the resolved SDK config kwargs for the destination; email + ``addresses`` is split into a list, other fields pass through as strings. + """ + dest = config.notify_destination.value + spec: dict[str, Any] = { + "destination": dest, + "destination_name": config.notify_destination_name or f"flowx-{dest}", + "args": {}, + } + for field in _NOTIFY_FIELDS.get(dest, ()): + raw = config.notify_args.get(field.sdk_arg, "") + if not raw: + continue + spec["args"][field.sdk_arg] = ( + [item.strip() for item in raw.split(",") if item.strip()] if field.is_list else raw + ) + return spec + + +def _collapse_notify(pipeline: Pipeline, config: TranslationConfiguration) -> Pipeline: + """Collapse activity->notify groups: drop the notify Web activities and stamp a + notification spec (events + chosen destination) onto each upstream task. + + Works for any upstream activity type (Copy, Notebook, Lookup, …). Dependents of a dropped + notify activity are rewired to the upstream task so the DAG stays connected. The destination is + created (and its id resolved) later, at prepare time. + """ + groups = _find_notify_groups(pipeline.tasks) + if not groups: + return pipeline + spec_base = _notification_spec(config) + notify_to_upstream: dict[str, str] = {} + events_by_task: dict[str, list[str]] = {} + drop: set[str] = set() + for upstream_key, web_list in groups.items(): + events: set[str] = set() + for web, outcome in web_list: + events.add("on_failure" if outcome == "Failed" else "on_success") + drop.add(web.task_key) + notify_to_upstream[web.task_key] = upstream_key + chosen = config.notify_events + if chosen is NotifyEvents.ON_FAILURE: + events &= {"on_failure"} + elif chosen is NotifyEvents.ON_SUCCESS: + events &= {"on_success"} + events_by_task[upstream_key] = sorted(events) or ["on_failure"] + + new_tasks: list = [] + for task in pipeline.tasks: + if task.task_key in drop: + continue + deps = task.depends_on + if deps: + rewired: list = [] + seen: set[str] = set() + for dep in deps: + key = notify_to_upstream.get(dep.task_key, dep.task_key) + if key not in seen: + seen.add(key) + rewired.append(Dependency(task_key=key, outcome=dep.outcome)) + deps = rewired + if task.task_key in events_by_task: + spec = {**spec_base, "events": events_by_task[task.task_key]} + task = dataclasses.replace(task, depends_on=deps, notifications=spec) + else: + task = dataclasses.replace(task, depends_on=deps) + new_tasks.append(task) + return dataclasses.replace(pipeline, tasks=new_tasks) + + +def _build_copy_activity_paradigm_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None -) -> TranslationQuestion | None: - """Builds the SDP-vs-notebook question for Copy activities targeting Delta. +) -> TranslationOption | None: + """Builds the SDP-vs-notebook option for Copy activities targeting Delta. Args: pipeline: Translated pipeline IR. motifs: Detected motifs (unused; accepted for builder uniformity). answers: Answers already supplied for prior prompts. When the - user opted into Lakeflow Connect, this question only fires + user opted into Lakeflow Connect, this option only fires for Copy activities that are *not* LFC-eligible -- the paradigm choice is moot for Copies that will become managed LFC pipelines. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when no + The constructed :class:`TranslationOption`, or ``None`` when no Copy activity needs a paradigm choice. Copies whose source query is unfit for both LFC and SDP (joins, aggregates, etc.) are forced to PySpark notebook and excluded from the affected set. """ answers = answers or {} - going_to_lfc = answers.get(QUESTION_USE_LAKEFLOW_CONNECTORS) == UseLakeflowConnectors.LAKEFLOW_CONNECT.value + going_to_lfc = answers.get(OPTION_USE_LAKEFLOW_CONNECTORS) == UseLakeflowConnectors.LAKEFLOW_CONNECT.value affected = tuple( activity.task_key for activity in walk_activities(pipeline.tasks) @@ -334,8 +768,8 @@ def _build_copy_activity_paradigm_question( ) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_COPY_ACTIVITY_PARADIGM, + return TranslationOption( + option_id=OPTION_COPY_ACTIVITY_PARADIGM, prompt="How should Copy Data activities targeting Delta be implemented?", rationale=( "One or more Copy Data activities write to a Delta table. " @@ -343,12 +777,12 @@ def _build_copy_activity_paradigm_question( "a PySpark notebook stays closer to the original ADF activity shape." ), options=( - QuestionOption( + OptionChoice( value=CopyActivityParadigm.NOTEBOOK.value, label="PySpark notebook", description="Generates a notebook task that reads the source and writes Delta directly.", ), - QuestionOption( + OptionChoice( value=CopyActivityParadigm.SDP.value, label="Lakeflow Spark Declarative Pipeline", description="Emits an SDP pipeline resource with declarative table definitions.", @@ -359,10 +793,10 @@ def _build_copy_activity_paradigm_question( ) -def _build_non_databricks_task_compute_question( +def _build_non_databricks_task_compute_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None -) -> TranslationQuestion | None: - """Builds the serverless-vs-classic question for non-Databricks tasks. +) -> TranslationOption | None: + """Builds the serverless-vs-classic option for non-Databricks tasks. Args: pipeline: Translated pipeline IR. @@ -372,12 +806,12 @@ def _build_non_databricks_task_compute_question( because LFC pipelines always use serverless compute. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when + The constructed :class:`TranslationOption`, or ``None`` when every non-Databricks task in the pipeline is going to LFC (no compute choice to make). """ answers = answers or {} - going_to_lfc = answers.get(QUESTION_USE_LAKEFLOW_CONNECTORS) == UseLakeflowConnectors.LAKEFLOW_CONNECT.value + going_to_lfc = answers.get(OPTION_USE_LAKEFLOW_CONNECTORS) == UseLakeflowConnectors.LAKEFLOW_CONNECT.value affected = tuple( activity.task_key for activity in walk_activities(pipeline.tasks) @@ -385,8 +819,8 @@ def _build_non_databricks_task_compute_question( ) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_NON_DATABRICKS_TASK_COMPUTE, + return TranslationOption( + option_id=OPTION_NON_DATABRICKS_TASK_COMPUTE, prompt="What compute should the non-Databricks tasks use?", rationale=( "Tasks such as Copy Data, Web, Lookup, and Wait can run on serverless " @@ -394,12 +828,12 @@ def _build_non_databricks_task_compute_question( "tasks and a larger fixed-size cluster for Copy Data." ), options=( - QuestionOption( + OptionChoice( value=NonDatabricksTaskCompute.SERVERLESS.value, label="Serverless", description="Runs every non-Databricks task on serverless compute.", ), - QuestionOption( + OptionChoice( value=NonDatabricksTaskCompute.CLASSIC.value, label="Classic job_cluster", description="Provisions classic job_clusters sized per task type.", @@ -410,24 +844,24 @@ def _build_non_databricks_task_compute_question( ) -def _build_use_lakeflow_connectors_question( +def _build_use_lakeflow_connectors_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None -) -> TranslationQuestion | None: - """Builds the Lakeflow Connect question for eligible database ingestions. +) -> TranslationOption | None: + """Builds the Lakeflow Connect option for eligible database ingestions. Args: pipeline: Translated pipeline IR. motifs: Detected motifs, scanned for database-source ingestion patterns. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when no + The constructed :class:`TranslationOption`, or ``None`` when no Copy activity or motif qualifies for Lakeflow Connect. """ affected = _affected_task_keys_for_lakeflow_connect(pipeline, motifs) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_USE_LAKEFLOW_CONNECTORS, + return TranslationOption( + option_id=OPTION_USE_LAKEFLOW_CONNECTORS, prompt="Migrate eligible SQL Server, MySQL, and PostgreSQL ingestions to Lakeflow Connect?", rationale=( "One or more Copy Data activities ingest from SQL Server, MySQL, or " @@ -436,12 +870,12 @@ def _build_use_lakeflow_connectors_question( "the ADF-shaped activity intact." ), options=( - QuestionOption( + OptionChoice( value=UseLakeflowConnectors.EXISTING.value, label="Keep existing translation", description="Preserves the Copy Data activity as a notebook or SDP task.", ), - QuestionOption( + OptionChoice( value=UseLakeflowConnectors.LAKEFLOW_CONNECT.value, label="Use Lakeflow Connect", description="Replaces eligible ingestions with a managed Lakeflow Connect pipeline.", @@ -452,10 +886,10 @@ def _build_use_lakeflow_connectors_question( ) -def _build_lakeflow_connector_type_question( +def _build_lakeflow_connector_type_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None -) -> TranslationQuestion | None: - """Builds the CDC-vs-query connector question, suppressed when not actionable. +) -> TranslationOption | None: + """Builds the CDC-vs-query connector option, suppressed when not actionable. Args: pipeline: Translated pipeline IR. @@ -468,7 +902,7 @@ def _build_lakeflow_connector_type_question( (table-based reads → CDC because the query-based connector requires a cursor column; queries with a cursor → query-based because CDC requires direct table access). A pipeline-wide - preference between CDC and query-based therefore has no + configuration between CDC and query-based therefore has no actionable effect; the modifier picks the eligible connector per Copy. """ @@ -476,42 +910,42 @@ def _build_lakeflow_connector_type_question( return None -def _build_metadata_driven_consolidate_question( +def _build_metadata_driven_consolidate_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None, -) -> TranslationQuestion | None: - """Builds the consolidate-or-keep question for metadata-driven motifs. +) -> TranslationOption | None: + """Builds the consolidate-or-keep option for metadata-driven motifs. Args: pipeline: Translated pipeline IR. - motifs: Detected motifs; the question only surfaces when at + motifs: Detected motifs; the option only surfaces when at least one matches the metadata-driven bulk copy pattern. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when + The constructed :class:`TranslationOption`, or ``None`` when the pipeline contains no metadata-driven motif. """ affected = _metadata_driven_motif_task_keys(pipeline, motifs) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_METADATA_DRIVEN_CONSOLIDATE, + return TranslationOption( + option_id=OPTION_METADATA_DRIVEN_CONSOLIDATE, prompt="Consolidate the metadata-driven ingestions into one managed pipeline?", rationale=( "A Lookup feeds a ForEach that copies each row's table. Consolidating " "replaces this loop with a single Lakeflow Connect or Lakeflow Spark " "Declarative Pipeline whose objects list materialises each source as " - "its own streaming table. Keeping the loop preserves the existing " - "per-row Copy translation." + "its own streaming table. Keeping it emits a Databricks for-each task " + "that runs one Spark JDBC read per source table (no managed pipeline)." ), options=( - QuestionOption( + OptionChoice( value=MetadataDrivenConsolidate.KEEP.value, - label="Keep the per-row loop", - description="Preserves the ForEach + Copy translation as a motif scaffold.", + label="Keep the per-table loop", + description="Emits a for-each task running one Spark JDBC read per source table.", ), - QuestionOption( + OptionChoice( value=MetadataDrivenConsolidate.CONSOLIDATE.value, label="Consolidate into one pipeline", description="Emits one pipeline resource that ingests every source from the lookup.", @@ -522,26 +956,26 @@ def _build_metadata_driven_consolidate_question( ) -def _build_metadata_driven_access_question( +def _build_metadata_driven_access_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None, -) -> TranslationQuestion | None: - """Builds the metadata-source access question, gated on consolidate=yes. +) -> TranslationOption | None: + """Builds the metadata-source access option, gated on consolidate=yes. Args: pipeline: Translated pipeline IR. motifs: Detected motifs. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when no + The constructed :class:`TranslationOption`, or ``None`` when no metadata-driven motif applies. """ affected = _metadata_driven_motif_task_keys(pipeline, motifs) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_METADATA_DRIVEN_ACCESS, + return TranslationOption( + option_id=OPTION_METADATA_DRIVEN_ACCESS, prompt="Do you have access to query the metadata source and approve doing so?", rationale=( "Consolidating a metadata-driven ingestion requires materialising the " @@ -550,12 +984,12 @@ def _build_metadata_driven_access_question( "translation pass; answering no falls back to the per-row scaffold." ), options=( - QuestionOption( + OptionChoice( value=MetadataDrivenAccess.YES.value, label="Yes, query is allowed", description="The metadata source is reachable and approved for read during translation.", ), - QuestionOption( + OptionChoice( value=MetadataDrivenAccess.NO.value, label="No, skip materialising the lookup", description="Keeps the per-row motif scaffold without inlining the configuration.", @@ -563,30 +997,30 @@ def _build_metadata_driven_access_question( ), affected_task_keys=affected, default=MetadataDrivenAccess.NO.value, - conditions=((QUESTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value),), + conditions=((OPTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value),), ) -def _build_metadata_driven_size_question( +def _build_metadata_driven_size_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None, -) -> TranslationQuestion | None: - """Builds the t-shirt sizing question, gated on consolidate=yes. +) -> TranslationOption | None: + """Builds the t-shirt sizing option, gated on consolidate=yes. Args: pipeline: Translated pipeline IR. motifs: Detected motifs. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when no + The constructed :class:`TranslationOption`, or ``None`` when no metadata-driven motif applies. """ affected = _metadata_driven_motif_task_keys(pipeline, motifs) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_METADATA_DRIVEN_SIZE, + return TranslationOption( + option_id=OPTION_METADATA_DRIVEN_SIZE, prompt="Roughly how many configuration rows feed the metadata-driven ingestion?", rationale=( "The size determines whether the modifier inlines every lookup row into " @@ -595,17 +1029,17 @@ def _build_metadata_driven_size_question( "avoid generating an unwieldy pipeline definition." ), options=( - QuestionOption( + OptionChoice( value=MetadataDrivenSize.SMALL.value, label="S (under 50 rows)", description="Lookup feeds fewer than 50 ingestion targets.", ), - QuestionOption( + OptionChoice( value=MetadataDrivenSize.MEDIUM.value, label="M (under 250 rows)", description="Lookup feeds 50 to 249 ingestion targets.", ), - QuestionOption( + OptionChoice( value=MetadataDrivenSize.LARGE.value, label="L (250 or more rows)", description="Lookup feeds 250+ targets; skip inline consolidation.", @@ -613,30 +1047,30 @@ def _build_metadata_driven_size_question( ), affected_task_keys=affected, default=MetadataDrivenSize.LARGE.value, - conditions=((QUESTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value),), + conditions=((OPTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value),), ) -def _build_metadata_driven_lookup_tool_question( +def _build_metadata_driven_lookup_tool_option( pipeline: Pipeline, motifs: list, answers: dict[str, str] | None = None, -) -> TranslationQuestion | None: - """Builds the agent-tool question for the lookup query, gated on size != L. +) -> TranslationOption | None: + """Builds the agent-tool option for the lookup query, gated on size != L. Args: pipeline: Translated pipeline IR. motifs: Detected motifs. Returns: - The constructed :class:`TranslationQuestion`, or ``None`` when no + The constructed :class:`TranslationOption`, or ``None`` when no metadata-driven motif applies. """ affected = _metadata_driven_motif_task_keys(pipeline, motifs) if not affected: return None - return TranslationQuestion( - question_id=QUESTION_METADATA_DRIVEN_LOOKUP_TOOL, + return TranslationOption( + option_id=OPTION_METADATA_DRIVEN_LOOKUP_TOOL, prompt="Does the agent have a tool that can run the lookup query?", rationale=( "When the agent has a Genie skill, an MCP database tool, or a SQL " @@ -646,12 +1080,12 @@ def _build_metadata_driven_lookup_tool_question( "comma-separated string of values and the modifier ingests that." ), options=( - QuestionOption( + OptionChoice( value=MetadataDrivenLookupTool.HAVE.value, label="Yes, the agent can run the lookup", description="Agent executes the lookup query via its own tool.", ), - QuestionOption( + OptionChoice( value=MetadataDrivenLookupTool.NONE.value, label="No, ask the user for the values", description="Agent prompts the user for a CSV file or string of values.", @@ -660,22 +1094,22 @@ def _build_metadata_driven_lookup_tool_question( affected_task_keys=affected, default=MetadataDrivenLookupTool.NONE.value, conditions=( - (QUESTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value), - (QUESTION_METADATA_DRIVEN_ACCESS, MetadataDrivenAccess.YES.value), + (OPTION_METADATA_DRIVEN_CONSOLIDATE, MetadataDrivenConsolidate.CONSOLIDATE.value), + (OPTION_METADATA_DRIVEN_ACCESS, MetadataDrivenAccess.YES.value), ), ) -def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuestion]: - """Builds one ``consolidate_motif:`` question per detected motif. +def _build_motif_consolidation_options(motifs: list) -> list[TranslationOption]: + """Builds one ``consolidate_motif:`` option per detected motif. Args: motifs: Detected :class:`~flowx.models.motifs.DetectedMotif` instances from :func:`flowx.motifs.detector.detect_motifs`. Returns: - A list of :class:`TranslationQuestion` instances, one per - detected motif. Each question uses a unique question_id of the + A list of :class:`TranslationOption` instances, one per + detected motif. Each option uses a unique option_id of the form ``consolidate_motif:`` so multiple distinct motif types in the same pipeline (e.g. ``rest_api_pagination`` *and* ``metadata_driven_bulk_copy``) each get their own prompt. @@ -688,7 +1122,7 @@ def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuesti the safer default. - When the same motif type is detected more than once in the same pipeline (rare in practice but possible) the builder - emits a single question covering all instances of that type. + emits a single option covering all instances of that type. Per-instance overrides can still be expressed by adding more fine-grained gating in :class:`MotifActivity`. - The ``affected_task_keys`` field lists the *underlying* ADF @@ -699,7 +1133,7 @@ def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuesti if not motifs: return [] seen: set[str] = set() - questions: list[TranslationQuestion] = [] + options: list[TranslationOption] = [] for motif in motifs: definition = motif.definition motif_id = definition.motif_id @@ -707,13 +1141,13 @@ def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuesti continue seen.add(motif_id) affected = tuple(motif.matched_activities) - question_id = f"{MOTIF_CONSOLIDATE_QUESTION_PREFIX}{motif_id}" + option_id = f"{MOTIF_CONSOLIDATE_OPTION_PREFIX}{motif_id}" confidence_suffix = "" if motif.confidence_notes: confidence_suffix = " Detector notes: " + " | ".join(motif.confidence_notes) - questions.append( - TranslationQuestion( - question_id=question_id, + options.append( + TranslationOption( + option_id=option_id, prompt=f"Consolidate the {definition.display_name!r} motif into a single task?", rationale=( f"{definition.description} " @@ -722,12 +1156,12 @@ def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuesti f"them with a single {definition.databricks_replacement!r} task." ), options=( - QuestionOption( + OptionChoice( value=MotifConsolidate.KEEP.value, label="Keep individual activities", description="Preserves the per-activity translation; no motif collapse.", ), - QuestionOption( + OptionChoice( value=MotifConsolidate.CONSOLIDATE.value, label="Consolidate into one task", description=f"Replaces matched activities with a {definition.databricks_replacement!r} task.", @@ -737,7 +1171,7 @@ def _build_motif_consolidation_questions(motifs: list) -> list[TranslationQuesti default=MotifConsolidate.KEEP.value, ) ) - return questions + return options def _metadata_driven_motif_task_keys( @@ -790,7 +1224,7 @@ def _copy_paradigm_decided_by_lfc(activity: CopyActivity, going_to_lfc: bool) -> Args: activity: Copy activity to inspect. - going_to_lfc: ``True`` when the caller answered the LFC question + going_to_lfc: ``True`` when the caller answered the LFC option with ``lakeflow_connect``. Returns: @@ -809,7 +1243,7 @@ def _task_compute_decided_by_lfc(activity, going_to_lfc: bool) -> bool: Args: activity: Activity to inspect. - going_to_lfc: ``True`` when the caller answered the LFC question + going_to_lfc: ``True`` when the caller answered the LFC option with ``lakeflow_connect``. Returns: @@ -862,13 +1296,13 @@ def _motif_task_keys_for_lakeflow_connect( ] -def _stamp_activity(activity: Activity, pipeline_preferences: TranslationPreferences) -> Activity: - """Stamps preference-derived decisions onto an activity. +def _stamp_activity(activity: Activity, pipeline_configuration: TranslationConfiguration) -> Activity: + """Stamps configuration-derived decisions onto an activity. Args: activity: Source activity from the IR. - pipeline_preferences: Pipeline-wide preferences; per-task overrides - apply via :meth:`TranslationPreferences.effective_for`. + pipeline_configuration: Pipeline-wide configuration; per-task overrides + apply via :meth:`TranslationConfiguration.effective_for`. Returns: A new activity instance with ``compute_mode``, ``target_format``, @@ -876,32 +1310,32 @@ def _stamp_activity(activity: Activity, pipeline_preferences: TranslationPrefere flow activities are recursed into so their inner bodies are stamped too. """ - activity_preferences = pipeline_preferences.effective_for(activity.task_key) + activity_configuration = pipeline_configuration.effective_for(activity.task_key) if isinstance(activity, ForEachActivity): - return _stamp_for_each_activity(activity, pipeline_preferences, activity_preferences) + return _stamp_for_each_activity(activity, pipeline_configuration, activity_configuration) if isinstance(activity, IfConditionActivity): - return _stamp_if_condition_activity(activity, pipeline_preferences, activity_preferences) + return _stamp_if_condition_activity(activity, pipeline_configuration, activity_configuration) if isinstance(activity, SwitchActivity): - return _stamp_switch_activity(activity, pipeline_preferences, activity_preferences) + return _stamp_switch_activity(activity, pipeline_configuration, activity_configuration) if isinstance(activity, CopyActivity): - return _stamp_copy_activity(activity, activity_preferences) + return _stamp_copy_activity(activity, activity_configuration) if isinstance(activity, MotifActivity): - return _stamp_motif_activity(activity, activity_preferences) - return dataclasses.replace(activity, compute_mode=_resolve_compute_mode(activity, activity_preferences)) + return _stamp_motif_activity(activity, activity_configuration) + return dataclasses.replace(activity, compute_mode=_resolve_compute_mode(activity, activity_configuration)) def _stamp_for_each_activity( activity: ForEachActivity, - pipeline_preferences: TranslationPreferences, - activity_preferences: TranslationPreferences, + pipeline_configuration: TranslationConfiguration, + activity_configuration: TranslationConfiguration, ) -> ForEachActivity: """Stamps a ForEach activity and recurses into its inner body. Args: activity: Source ForEach activity. - pipeline_preferences: Pipeline-wide preferences threaded into + pipeline_configuration: Pipeline-wide configuration threaded into inner activities so they re-resolve their own overrides. - activity_preferences: Preferences after per-task overrides for + activity_configuration: Configuration after per-task overrides for *activity*. Returns: @@ -909,23 +1343,23 @@ def _stamp_for_each_activity( """ return dataclasses.replace( activity, - inner_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.inner_activities], - compute_mode=_resolve_compute_mode(activity, activity_preferences), + inner_activities=[_stamp_activity(inner, pipeline_configuration) for inner in activity.inner_activities], + compute_mode=_resolve_compute_mode(activity, activity_configuration), ) def _stamp_if_condition_activity( activity: IfConditionActivity, - pipeline_preferences: TranslationPreferences, - activity_preferences: TranslationPreferences, + pipeline_configuration: TranslationConfiguration, + activity_configuration: TranslationConfiguration, ) -> IfConditionActivity: """Stamps an IfCondition activity and recurses into both branches. Args: activity: Source IfCondition activity. - pipeline_preferences: Pipeline-wide preferences threaded into + pipeline_configuration: Pipeline-wide configuration threaded into inner activities so they re-resolve their own overrides. - activity_preferences: Preferences after per-task overrides for + activity_configuration: Configuration after per-task overrides for *activity*. Returns: @@ -933,24 +1367,24 @@ def _stamp_if_condition_activity( """ return dataclasses.replace( activity, - if_true_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.if_true_activities], - if_false_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.if_false_activities], - compute_mode=_resolve_compute_mode(activity, activity_preferences), + if_true_activities=[_stamp_activity(inner, pipeline_configuration) for inner in activity.if_true_activities], + if_false_activities=[_stamp_activity(inner, pipeline_configuration) for inner in activity.if_false_activities], + compute_mode=_resolve_compute_mode(activity, activity_configuration), ) def _stamp_switch_activity( activity: SwitchActivity, - pipeline_preferences: TranslationPreferences, - activity_preferences: TranslationPreferences, + pipeline_configuration: TranslationConfiguration, + activity_configuration: TranslationConfiguration, ) -> SwitchActivity: """Stamps a Switch activity and recurses into every case and the default. Args: activity: Source Switch activity. - pipeline_preferences: Pipeline-wide preferences threaded into + pipeline_configuration: Pipeline-wide configuration threaded into inner activities so they re-resolve their own overrides. - activity_preferences: Preferences after per-task overrides for + activity_configuration: Configuration after per-task overrides for *activity*. Returns: @@ -959,46 +1393,46 @@ def _stamp_switch_activity( stamped_cases = [ SwitchCase( value=case.value, - activities=[_stamp_activity(inner, pipeline_preferences) for inner in case.activities], + activities=[_stamp_activity(inner, pipeline_configuration) for inner in case.activities], ) for case in activity.cases ] return dataclasses.replace( activity, cases=stamped_cases, - default_activities=[_stamp_activity(inner, pipeline_preferences) for inner in activity.default_activities], - compute_mode=_resolve_compute_mode(activity, activity_preferences), + default_activities=[_stamp_activity(inner, pipeline_configuration) for inner in activity.default_activities], + compute_mode=_resolve_compute_mode(activity, activity_configuration), ) def _stamp_copy_activity( activity: CopyActivity, - activity_preferences: TranslationPreferences, + activity_configuration: TranslationConfiguration, ) -> CopyActivity: """Stamps a Copy activity with paradigm, compute, and Lakeflow Connect flags. Args: activity: Source Copy activity. - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. Returns: A new :class:`CopyActivity` whose ``target_format``, ``compute_mode``, and ``use_lakeflow_connector`` fields reflect the user's choices. Copies whose query is unfit for LFC and SDP (joins, aggregates, etc.) are forced to the notebook - paradigm regardless of preference because the alternative + paradigm regardless of configuration because the alternative paradigms cannot represent arbitrary SQL. """ - user_picked_lfc = activity_preferences.use_lakeflow_connectors is UseLakeflowConnectors.LAKEFLOW_CONNECT + user_picked_lfc = activity_configuration.use_lakeflow_connectors is UseLakeflowConnectors.LAKEFLOW_CONNECT use_lakeflow_connector = user_picked_lfc and copy_eligible_for_any_lfc_connector(activity) - paradigm = _resolve_paradigm(activity, activity_preferences, use_lakeflow_connector) + paradigm = _resolve_paradigm(activity, activity_configuration, use_lakeflow_connector) connector_type = ( - _resolve_lakeflow_connector_type(activity, activity_preferences) if use_lakeflow_connector else None + _resolve_lakeflow_connector_type(activity, activity_configuration) if use_lakeflow_connector else None ) return dataclasses.replace( activity, target_format=paradigm.value, - compute_mode=_resolve_compute_mode(activity, activity_preferences), + compute_mode=_resolve_compute_mode(activity, activity_configuration), use_lakeflow_connector=use_lakeflow_connector, lakeflow_connector_type=connector_type, ) @@ -1006,14 +1440,14 @@ def _stamp_copy_activity( def _resolve_paradigm( activity: CopyActivity, - activity_preferences: TranslationPreferences, + activity_configuration: TranslationConfiguration, use_lakeflow_connector: bool, ) -> CopyActivityParadigm: """Resolves the paradigm (notebook vs SDP) for a Copy that won't go to LFC. Args: activity: Source Copy activity. - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. use_lakeflow_connector: ``True`` when the modifier already routed the Copy to a managed LFC pipeline; the paradigm is informational in that case. @@ -1029,20 +1463,20 @@ def _resolve_paradigm( return CopyActivityParadigm.NOTEBOOK if not copy_targets_delta(activity): return CopyActivityParadigm.NOTEBOOK - return activity_preferences.copy_activity_paradigm + return activity_configuration.copy_activity_paradigm -def _resolve_lakeflow_connector_type(activity: CopyActivity, activity_preferences: TranslationPreferences) -> str: +def _resolve_lakeflow_connector_type(activity: CopyActivity, activity_configuration: TranslationConfiguration) -> str: """Resolves which Lakeflow Connect connector to use for an eligible Copy. Args: activity: Source Copy activity (already known to be LFC-eligible). - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. Returns: Always the connector flavour the Copy is actually eligible for. Query-based eligibility (parseable query + cursor column) wins - over the user's CDC preference because no cursor candidate + over the user's CDC configuration because no cursor candidate exists for a CDC connector to use on a query-only Copy. Table-based Copies route to CDC because the query-based connector requires a cursor column and there is none. @@ -1054,13 +1488,13 @@ def _resolve_lakeflow_connector_type(activity: CopyActivity, activity_preference def _stamp_motif_activity( activity: MotifActivity, - activity_preferences: TranslationPreferences, + activity_configuration: TranslationConfiguration, ) -> MotifActivity: """Stamps a Motif activity, swapping in Lakeflow Connect when eligible. Args: activity: Source motif activity. - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. Returns: A new :class:`MotifActivity` whose ``databricks_replacement`` is @@ -1071,7 +1505,7 @@ def _stamp_motif_activity( consolidation, granted access, and the size bucket is S or M. """ qualifies_for_lakeflow_connect = ( - activity_preferences.use_lakeflow_connectors is UseLakeflowConnectors.LAKEFLOW_CONNECT + activity_configuration.use_lakeflow_connectors is UseLakeflowConnectors.LAKEFLOW_CONNECT and activity.source_type_hint == DATABASE_SOURCE_TYPE_HINT ) replacement = LAKEFLOW_CONNECT_REPLACEMENT if qualifies_for_lakeflow_connect else activity.databricks_replacement @@ -1080,25 +1514,25 @@ def _stamp_motif_activity( if qualifies_for_lakeflow_connect else activity.notebook_template ) - consolidate = _should_consolidate_metadata_driven(activity, activity_preferences) + consolidate = _should_consolidate_metadata_driven(activity, activity_configuration) return dataclasses.replace( activity, databricks_replacement=replacement, notebook_template=notebook_template, - compute_mode=_resolve_compute_mode(activity, activity_preferences), + compute_mode=_resolve_compute_mode(activity, activity_configuration), consolidate_metadata_driven=consolidate, ) def _should_consolidate_metadata_driven( activity: MotifActivity, - activity_preferences: TranslationPreferences, + activity_configuration: TranslationConfiguration, ) -> bool: """Returns True when the modifier should consolidate a metadata-driven motif. Args: activity: Source motif activity. - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. Returns: ``True`` when the motif matches the metadata-driven bulk-copy @@ -1109,19 +1543,19 @@ def _should_consolidate_metadata_driven( """ if activity.motif_id != "metadata_driven_bulk_copy": return False - if activity_preferences.metadata_driven_consolidate is not MetadataDrivenConsolidate.CONSOLIDATE: + if activity_configuration.metadata_driven_consolidate is not MetadataDrivenConsolidate.CONSOLIDATE: return False - if activity_preferences.metadata_driven_access is not MetadataDrivenAccess.YES: + if activity_configuration.metadata_driven_access is not MetadataDrivenAccess.YES: return False - return activity_preferences.metadata_driven_size is not MetadataDrivenSize.LARGE + return activity_configuration.metadata_driven_size is not MetadataDrivenSize.LARGE -def _resolve_compute_mode(activity: Activity, activity_preferences: TranslationPreferences) -> str: +def _resolve_compute_mode(activity: Activity, activity_configuration: TranslationConfiguration) -> str: """Resolves the compute mode an activity should run on. Args: activity: Source activity. - activity_preferences: Effective preferences for this activity. + activity_configuration: Effective configuration for this activity. Returns: One of :data:`COMPUTE_MODE_SERVERLESS`, @@ -1135,7 +1569,7 @@ def _resolve_compute_mode(activity: Activity, activity_preferences: TranslationP """ if not is_non_databricks_task(activity): return COMPUTE_MODE_INHERIT - if activity_preferences.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS: + if activity_configuration.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS: return COMPUTE_MODE_SERVERLESS if isinstance(activity, CopyActivity): return COMPUTE_MODE_CLASSIC_MULTI_NODE diff --git a/src/orchestra/adapter/predicates.py b/src/flowx/adapter/predicates.py similarity index 99% rename from src/orchestra/adapter/predicates.py rename to src/flowx/adapter/predicates.py index aa211e9..7c69bac 100644 --- a/src/orchestra/adapter/predicates.py +++ b/src/flowx/adapter/predicates.py @@ -182,7 +182,7 @@ def copy_query_unfit_for_lfc(activity: CopyActivity) -> bool: contains JOIN, GROUP BY, aggregates, UNION, window functions, subqueries, or column expressions). Such Copies should be translated through PySpark notebooks regardless of paradigm - preference because LFC's query-based connector and SDP's + configuration because LFC's query-based connector and SDP's declarative table form both reject the query. """ if not copy_has_source_query(activity): diff --git a/src/flowx/adapter/session.py b/src/flowx/adapter/session.py new file mode 100644 index 0000000..2b7509c --- /dev/null +++ b/src/flowx/adapter/session.py @@ -0,0 +1,486 @@ +"""Agent adapter that drives the ask-validate-resume loop. + +:class:`TranslationSession` is the entry point an agent uses to +translate tool-call arguments into validated configuration. When the IR +raises options the agent cannot answer from context alone, the +session surfaces them as structured :class:`TranslationOption` +objects (and, via :exc:`TranslationInputRequired`, as exceptions) so +the agent can route them back to the user. The pipeline modifier is +invoked only once every option has an answer. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from flowx.adapter.constants import ( + INPUT_ADF_RESOURCE_URL, + INPUT_ADF_SOURCE_PATH, + INPUT_BUNDLE_NAME, + INPUT_CATALOG, + INPUT_DATABRICKS_PROFILE, + INPUT_INSTALL_DASHBOARD, + INPUT_INVENTORY_PATH, + INPUT_OUTPUT_BUNDLE_PATH, + INPUT_OUTPUT_DIR, + INPUT_RESULTS_TABLE, + INPUT_RESULTS_WAREHOUSE, + INPUT_SCHEMA, + INPUT_TRANSLATION_REPORT_PATH, + MOTIF_CONSOLIDATE_OPTION_PREFIX, + PHASE_CONVERT, + PHASE_DISCOVER, + PHASE_PACKAGE, +) +from flowx.adapter.models import ( + DEFAULT_CONFIGURATION, + CopyActivityParadigm, + LakeflowConnectorType, + MetadataDrivenAccess, + MetadataDrivenConsolidate, + MetadataDrivenLookupTool, + MetadataDrivenSize, + MigrationInputOption, + MotifConsolidate, + NonDatabricksTaskCompute, + PendingMigrationInputs, + PendingOptions, + TranslationConfiguration, + TranslationOption, + UseLakeflowConnectors, +) +from flowx.adapter.operations import ( + apply_configuration, + gather_options, + validate_answer, +) +from flowx.models.ir import Pipeline +from flowx.models.motifs import DetectedMotif + + +class TranslationInputRequired(Exception): + """Raised by :meth:`TranslationSession.run` when answers are still missing. + + Attributes: + pending: The outstanding options the agent should route to the + user before retrying :meth:`TranslationSession.run`. + """ + + def __init__(self, pending: PendingOptions) -> None: + """Stores the pending options on the exception. + + Args: + pending: Outstanding options surfaced by the session. + """ + super().__init__( + f"{len(pending.options)} translation option(s) require user input for pipeline {pending.pipeline_name!r}" + ) + self.pending = pending + + +@dataclass(slots=True, kw_only=True) +class TranslationSession: + """Coordinates the ask-validate-resume loop for one translated pipeline. + + A session is single-use: the caller drives it by either polling via + :meth:`pending` and :meth:`answer`, or calling :meth:`run` and + handling :exc:`TranslationInputRequired`. When every option is + answered, :meth:`run` (or :meth:`resume`) returns the + configuration-stamped pipeline. + + Attributes: + pipeline: Translated pipeline IR after motif collapsing. + motifs: Detected motifs for the pipeline. Optional; only used to + decide whether the Lakeflow Connect option applies. + defaults: Baseline configuration applied when the caller skips a + option. Per-task overrides on this object are preserved + verbatim when :meth:`build_configuration` composes the final + snapshot. + """ + + pipeline: Pipeline + motifs: list[DetectedMotif] = field(default_factory=list) + defaults: TranslationConfiguration = DEFAULT_CONFIGURATION + _answers: dict[str, str] = field(default_factory=dict) + + def pending(self) -> PendingOptions: + """Returns the options still awaiting an answer. + + Returns: + A :class:`PendingOptions` instance containing only the + options whose preconditions are met by the IR and whose + IDs are not yet in the answer set. + """ + return gather_options( + self.pipeline, + self.motifs, + answers=self._answers, + ) + + def answer(self, option_id: str, value: str) -> None: + """Validates and records a single answer. + + Args: + option_id: Stable option identifier from + :class:`TranslationOption`. + value: Caller-supplied answer string. + + Raises: + ValueError: When *option_id* is unknown or *value* is not + in the allowed set for the option. + """ + self._answers[option_id] = validate_answer(option_id, value) + + def answer_many(self, answers: dict[str, str]) -> None: + """Validates and records multiple answers atomically. + + Args: + answers: Mapping of option_id to the caller-supplied answer. + + Raises: + ValueError: When any pair fails validation. No answers from + the batch are recorded when the call raises. + """ + validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} + self._answers.update(validated) + + def find_option(self, option_id: str) -> TranslationOption | None: + """Looks up a pending option by its identifier. + + Args: + option_id: Stable option identifier. + + Returns: + The matching :class:`TranslationOption` if it is still + pending, otherwise ``None``. + """ + return next( + (option for option in self.pending().options if option.option_id == option_id), + None, + ) + + def build_configuration(self) -> TranslationConfiguration: + """Composes the validated configuration snapshot from collected answers. + + Returns: + A :class:`TranslationConfiguration` where every answered field + takes the caller-supplied value and every unanswered field + falls back to the corresponding value on ``defaults``. + """ + return TranslationConfiguration( + copy_activity_paradigm=CopyActivityParadigm( + self._answers.get("copy_activity_paradigm", self.defaults.copy_activity_paradigm) + ), + non_databricks_task_compute=NonDatabricksTaskCompute( + self._answers.get("non_databricks_task_compute", self.defaults.non_databricks_task_compute) + ), + use_lakeflow_connectors=UseLakeflowConnectors( + self._answers.get("use_lakeflow_connectors", self.defaults.use_lakeflow_connectors) + ), + lakeflow_connector_type=LakeflowConnectorType( + self._answers.get("lakeflow_connector_type", self.defaults.lakeflow_connector_type) + ), + metadata_driven_consolidate=MetadataDrivenConsolidate( + self._answers.get("metadata_driven_consolidate", self.defaults.metadata_driven_consolidate) + ), + metadata_driven_access=MetadataDrivenAccess( + self._answers.get("metadata_driven_access", self.defaults.metadata_driven_access) + ), + metadata_driven_size=MetadataDrivenSize( + self._answers.get("metadata_driven_size", self.defaults.metadata_driven_size) + ), + metadata_driven_lookup_tool=MetadataDrivenLookupTool( + self._answers.get("metadata_driven_lookup_tool", self.defaults.metadata_driven_lookup_tool) + ), + motif_consolidations=self._collect_motif_consolidations(), + per_task=self.defaults.per_task, + ) + + def resume(self) -> Pipeline: + """Returns the configuration-stamped pipeline IR. + + Returns: + A new :class:`Pipeline` produced by applying the composed + configuration to ``self.pipeline``. The input pipeline is not + mutated. + """ + return apply_configuration(self.pipeline, self.build_configuration()) + + def run(self) -> Pipeline: + """Returns the modified pipeline, raising when input is still required. + + Returns: + The configuration-stamped pipeline IR when every applicable + option has an answer. + + Raises: + TranslationInputRequired: When one or more options are + still outstanding. The exception carries the pending + options so the agent can route them to the user. + """ + pending = self.pending() + if pending.options: + raise TranslationInputRequired(pending) + return self.resume() + + def _collect_motif_consolidations(self) -> dict[str, MotifConsolidate]: + """Returns the per-motif consolidation answers gathered so far. + + Returns: + Dict mapping ``motif_id`` to the user's :class:`MotifConsolidate` answer. Motifs the user + did not answer fall back to ``self.defaults`` (default :data:`MotifConsolidate.KEEP`). + """ + consolidations: dict[str, MotifConsolidate] = dict(self.defaults.motif_consolidations) + for option_id, answer in self._answers.items(): + if not option_id.startswith(MOTIF_CONSOLIDATE_OPTION_PREFIX): + continue + motif_id = option_id[len(MOTIF_CONSOLIDATE_OPTION_PREFIX) :] + consolidations[motif_id] = MotifConsolidate(answer) + return consolidations + + +_DISCOVER_OPTIONS: tuple[MigrationInputOption, ...] = ( + MigrationInputOption( + option_id=INPUT_ADF_SOURCE_PATH, + prompt="Where are the ADF JSON exports?", + description=( + "Unity Catalog volume path (``/Volumes///``) " + "or a local directory containing the ADF ARM/JSON export." + ), + required=True, + ), + MigrationInputOption( + option_id=INPUT_ADF_RESOURCE_URL, + prompt="ADF resource URL?", + description=( + "Azure portal URL of the source Data Factory. Captured for " + "traceability and surfaced in the generated bundle README; " + "leave blank when the source is exported from a local copy." + ), + default="", + required=False, + ), + MigrationInputOption( + option_id=INPUT_OUTPUT_DIR, + prompt="Which migration output directory should flowx use?", + description=( + "Single shared migration directory used by every phase (default ``./flowx_output``). " + "Discover writes ``metadata/inventory.json``, ``metadata/profile_report.csv``, and the " + "verbatim ``metadata/.arm.json`` into it." + ), + default="./flowx_output", + required=False, + ), +) + +_CONVERT_OPTIONS: tuple[MigrationInputOption, ...] = ( + MigrationInputOption( + option_id=INPUT_INVENTORY_PATH, + prompt="Path to the inventory.json from the discover phase?", + description="Inventory produced by the discover phase (under the shared migration dir's metadata/).", + default="./flowx_output/metadata/inventory.json", + required=False, + ), + MigrationInputOption( + option_id=INPUT_ADF_SOURCE_PATH, + prompt="Path to the ADF JSON exports?", + description="Same source directory the discover phase consumed; needed for cross-references.", + required=True, + ), + MigrationInputOption( + option_id=INPUT_OUTPUT_DIR, + prompt="Which migration output directory should flowx use?", + description=( + "The same shared migration directory the discover phase used (default ``./flowx_output``). " + "Convert writes its transient report and IR to the directory's ``.work/`` subfolder." + ), + default="./flowx_output", + required=False, + ), +) + +_PACKAGE_OPTIONS: tuple[MigrationInputOption, ...] = ( + MigrationInputOption( + option_id=INPUT_TRANSLATION_REPORT_PATH, + prompt="Path to the translation report?", + description=( + "Configuration-stamped report from `python -m flowx.adapter modify`, " + "or the raw convert-phase report when no configuration were applied." + ), + default="./flowx_output/.work/translation_report.stamped.json", + required=False, + ), + MigrationInputOption( + option_id=INPUT_OUTPUT_BUNDLE_PATH, + prompt="Which migration output directory should flowx use?", + description=( + "The same shared migration directory used by discover/convert (default ``./flowx_output``). " + "Package writes the DAB bundle at its top level and prunes the transient ``.work/`` folder." + ), + default="./flowx_output", + required=False, + ), + MigrationInputOption( + option_id=INPUT_CATALOG, + prompt="Target Unity Catalog catalog?", + description="Default ``catalog`` bundle variable used by emitted notebooks and pipelines.", + default="main", + required=False, + ), + MigrationInputOption( + option_id=INPUT_SCHEMA, + prompt="Target Unity Catalog schema?", + description="Default ``schema`` bundle variable used by emitted notebooks and pipelines.", + default="default", + required=False, + ), + MigrationInputOption( + option_id=INPUT_BUNDLE_NAME, + prompt="Bundle name override?", + description="Defaults to the first translated pipeline's resource key when blank.", + default="", + required=False, + ), + MigrationInputOption( + option_id=INPUT_DATABRICKS_PROFILE, + prompt="Databricks CLI profile?", + description=( + "Profile used to download workspace-resident notebooks during the " + "package phase. Leave blank to use the default profile from " + "``~/.databrickscfg`` or the active ``DATABRICKS_*`` env vars." + ), + default="", + required=False, + ), + MigrationInputOption( + option_id=INPUT_RESULTS_TABLE, + prompt="Record migration coverage to a Unity Catalog table? If so, the table (catalog.schema.table)?", + description=( + "Optional. When set, the package phase writes one coverage row per pipeline to this UC " + "table, stamped with a UUID run_id, run_date (CURRENT_TIMESTAMP()), and run_by " + "(CURRENT_USER()). Leave blank to skip. Requires workspace auth (Genie Code / a profile)." + ), + default="", + required=False, + ), + MigrationInputOption( + option_id=INPUT_RESULTS_WAREHOUSE, + prompt="SQL warehouse id for writing the results table / backing the dashboard?", + description=( + "Optional. Warehouse used to run the CREATE/INSERT and back the dashboard. Leave blank to " + "auto-detect (prefers a running, serverless warehouse)." + ), + default="", + required=False, + ), + MigrationInputOption( + option_id=INPUT_INSTALL_DASHBOARD, + prompt="Install a published AI/BI coverage dashboard over the results table? (yes/no)", + description=( + "Optional. When 'yes' (and a results table is set), installs and publishes a Lakeview " + "dashboard that visualizes migration coverage from the table." + ), + default="no", + required=False, + ), +) + +_OPTIONS_BY_PHASE: dict[str, tuple[MigrationInputOption, ...]] = { + PHASE_DISCOVER: _DISCOVER_OPTIONS, + PHASE_CONVERT: _CONVERT_OPTIONS, + PHASE_PACKAGE: _PACKAGE_OPTIONS, +} + + +class UnknownMigrationPhaseError(ValueError): + """Raised when a MigrationInputSession is constructed with an unrecognised phase.""" + + +@dataclass(slots=True, kw_only=True) +class MigrationInputSession: + """Coordinates the free-text input prompts at the top of an flowx phase. + + A session is single-use: the caller drives it by polling + :meth:`pending` and recording answers via :meth:`answer`, then reads + them out with :meth:`collected` once every required input has a + value. The session is intentionally distinct from + :class:`TranslationSession` because the inputs it gathers are + free-text paths and identifiers rather than enum-backed choices. + + Attributes: + phase: One of ``"discover"``, ``"convert"``, ``"package"``. + """ + + phase: str + _answers: dict[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Validates that *phase* is one of the supported migration phases. + + Raises: + UnknownMigrationPhaseError: When *phase* is not registered in + :data:`_OPTIONS_BY_PHASE`. + """ + if self.phase not in _OPTIONS_BY_PHASE: + raise UnknownMigrationPhaseError( + f"Unknown migration phase {self.phase!r}; expected one of {sorted(_OPTIONS_BY_PHASE)}" + ) + + def pending(self) -> PendingMigrationInputs: + """Returns the input options still awaiting an answer. + + Returns: + A :class:`PendingMigrationInputs` with the unanswered + options for ``self.phase`` in registration order. + """ + options = [option for option in _OPTIONS_BY_PHASE[self.phase] if option.option_id not in self._answers] + return PendingMigrationInputs(phase=self.phase, options=options) + + def answer(self, option_id: str, value: str) -> None: + """Records an answer to one input option. + + Args: + option_id: Stable identifier of the option. + value: Caller-supplied string value. + + Raises: + ValueError: When *option_id* is not a known input for the + session's phase. + """ + if not any(option.option_id == option_id for option in _OPTIONS_BY_PHASE[self.phase]): + raise ValueError(f"Unknown input option {option_id!r} for phase {self.phase!r}") + self._answers[option_id] = value + + def answer_many(self, answers: dict[str, str]) -> None: + """Records multiple input answers atomically. + + Args: + answers: Mapping of option_id to the caller-supplied value. + + Raises: + ValueError: When any pair references an unknown option. + No answers are recorded when the call raises. + """ + known_ids = {option.option_id for option in _OPTIONS_BY_PHASE[self.phase]} + unknown = set(answers) - known_ids + if unknown: + raise ValueError(f"Unknown input options for phase {self.phase!r}: {sorted(unknown)}") + self._answers.update(answers) + + def collected(self) -> dict[str, str]: + """Returns the collected answers merged with each option's default. + + Returns: + A dict keyed by option_id covering every option for the + phase: caller-supplied answers take precedence; otherwise + the option's ``default`` value (which may be the empty + string) is used. Required options whose answers are + missing are omitted so the caller can detect them. + """ + collected: dict[str, str] = {} + for option in _OPTIONS_BY_PHASE[self.phase]: + if option.option_id in self._answers: + collected[option.option_id] = self._answers[option.option_id] + elif option.default is not None: + collected[option.option_id] = option.default + return collected diff --git a/src/orchestra/bundler/__init__.py b/src/flowx/bundler/__init__.py similarity index 100% rename from src/orchestra/bundler/__init__.py rename to src/flowx/bundler/__init__.py diff --git a/src/orchestra/bundler/constants.py b/src/flowx/bundler/constants.py similarity index 100% rename from src/orchestra/bundler/constants.py rename to src/flowx/bundler/constants.py diff --git a/src/orchestra/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py similarity index 79% rename from src/orchestra/bundler/dab_writer.py rename to src/flowx/bundler/dab_writer.py index 05aa8b2..362d48a 100644 --- a/src/orchestra/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -66,15 +66,13 @@ class _BundleYamlDumper(yaml.SafeDumper): # Module-level warnings collector — reset per write_bundle call. _bundle_warnings: list[str] = [] -# Cross-bundle ExecutePipeline refs seen while translating: variable_name → -# target pipeline name. Reset per write_bundle call and surfaced via the -# bundle's ``variables`` block + SETUP.md. +# Cross-bundle ExecutePipeline refs seen while translating (variable_name -> target pipeline). Reset per +# write_bundle call and surfaced via the bundle's ``variables`` block + SETUP.md. _cross_bundle_variables: dict[str, str] = {} -# C-43 (CF5-001 / CF5-002): condition_task operands the dangling-ref safety -# net had to blank. Each entry is {task_key, field, original_ref}. Reset -# per write_bundle call and surfaced as a SETUP.md section so a neutralised -# branch predicate (always-true) is never silent. +# C-43 (CF5-001 / CF5-002): condition_task operands the dangling-ref safety net blanked +# ({task_key, field, original_ref}). Reset per write_bundle call and surfaced in SETUP.md so a +# neutralised (always-true) branch predicate is never silent. _neutralized_conditions: list[dict[str, str]] = [] _WIDGET_REFERENCE = re.compile(r"""dbutils\.widgets\.get\(\s*["']([^"']+)["']\s*\)""") @@ -99,9 +97,8 @@ def write_bundle( Returns: List of absolute paths to all created files. """ - # Reset module-level accumulators so successive ``write_bundle`` calls - # (CLI loops, library users, integration tests) don't carry warnings or - # cross-bundle variables from one bundle into the next. + # Reset module-level accumulators so successive write_bundle calls (CLI loops, tests) don't carry + # warnings or cross-bundle variables from one bundle into the next. _bundle_warnings.clear() _cross_bundle_variables.clear() _neutralized_conditions.clear() @@ -113,11 +110,8 @@ def write_bundle( resource_key = normalize_task_key(workflow.name) effective_name = bundle_name or resource_key - # Bind clusters across the parent workflow and any inner workflows up - # front so we can decide whether the bundle needs cluster-related - # tunables in ``databricks.yml`` at all. Binding is idempotent, so the - # subsequent ``_build_job_resource`` calls re-checking the same tasks is - # harmless. + # Bind clusters across the parent and inner workflows up front to decide whether databricks.yml needs + # cluster tunables at all. Binding is idempotent, so _build_job_resource re-checking these is harmless. _bind_cluster_to_notebook_tasks(workflow.tasks) for inner in workflow.inner_workflows: _bind_cluster_to_notebook_tasks(inner.tasks) @@ -125,11 +119,11 @@ def write_bundle( _any_task_uses_classic_cluster(inner.tasks) for inner in workflow.inner_workflows ) - # 1. Write databricks.yml. When at least one task runs on classic - # compute, defaults for spark_version / node_type_id come from the - # ADF linked service configs on the tasks so the emitted cluster - # matches the source-of-truth runtime. When every task is - # serverless, those variables are omitted entirely. + pipeline_resources = _collect_pipeline_resources(workflow) + pipeline_variable_declarations = _build_pipeline_variable_declarations(pipeline_resources, catalog, schema) + + # 1. Write databricks.yml. When any task runs on classic compute, spark_version / node_type_id + # defaults come from the ADF linked-service configs; when every task is serverless, they're omitted. databricks_yml_path = output_dir / "databricks.yml" inferred_spark_version, inferred_node_type_id = _infer_bundle_cluster_defaults(workflow) databricks_yml_dict = _build_databricks_yml( @@ -139,6 +133,7 @@ def write_bundle( spark_version=inferred_spark_version, node_type_id=inferred_node_type_id, include_cluster_variables=bundle_uses_classic_cluster, + extra_variables=pipeline_variable_declarations, ) databricks_yml_path.write_text( yaml.dump( @@ -152,10 +147,8 @@ def write_bundle( ) created_files.append(databricks_yml_path.resolve()) - # 2. Write job resource YAML. Strip broken base_parameters from - # existing-notebook tasks before serialising — these are surfaced - # in SETUP.md (§Existing-notebook parameter handling) further down - # and shouldn't ship in the YAML as malformed widget values. + # 2. Write job resource YAML. Strip broken base_parameters from existing-notebook tasks first — + # they're surfaced in SETUP.md further down and shouldn't ship in the YAML as malformed values. manual_parameters: list[ManualParameter] = _extract_manual_parameters_from_existing_notebook_tasks(workflow.tasks) for inner in workflow.inner_workflows: manual_parameters.extend(_extract_manual_parameters_from_existing_notebook_tasks(inner.tasks)) @@ -172,9 +165,8 @@ def write_bundle( ) created_files.append(job_yml_path.resolve()) - # Write inner workflows as additional resource files. Inner tasks reuse - # notebooks that live in the parent workflow's notebooks list, so pass - # those in so the inner job's widget auto-augmentation can see them. + # Write inner workflows as additional resource files. Inner tasks reuse notebooks from the parent's + # list, so pass those in for the inner job's widget auto-augmentation. for inner in workflow.inner_workflows: inner_key = normalize_task_key(inner.name) inner_yml_path = resources_dir / f"{inner_key}.yml" @@ -191,14 +183,10 @@ def write_bundle( ) created_files.append(inner_yml_path.resolve()) - # 2b. Write Lakeflow pipeline resources (Lakeflow Connect ingestion - # definitions emitted by the Copy preparer's LFC branch). Each - # resource lives in its own YAML so the bundle parser merges them - # alongside the job resources via the ``include`` glob. - pipelines_dir = resources_dir / "pipelines" - for resource in _collect_pipeline_resources(workflow): - pipelines_dir.mkdir(parents=True, exist_ok=True) - resource_yml_path = pipelines_dir / f"{resource['resource_key']}.yml" + # 2b. Write Lakeflow pipeline resources (Lakeflow Connect ingestion defs from the Copy preparer's LFC + # branch). Each lives in its own YAML so the bundle parser merges them via the ``include`` glob. + for resource in pipeline_resources: + resource_yml_path = resources_dir / f"{resource['resource_key']}.yml" resource_yml_path.write_text( yaml.dump( _wrap_pipeline_resource(resource), @@ -216,9 +204,8 @@ def write_bundle( if workflow.notebooks: created_files.extend(write_notebooks(workflow.notebooks, src_dir)) - # 4. Generate and write setup notebooks (create-scope, create-volume, etc.). - # These are the *executable* provisioning artifacts; SETUP.md (below) - # is the human-readable companion. + # 4. Generate and write setup notebooks (create-scope, create-volume, etc.) — the executable + # provisioning artifacts; SETUP.md (below) is the human-readable companion. setup_notebooks: list[DabNotebook] = generate_setup_tasks( secrets=workflow.secrets, setup_tasks=workflow.setup_tasks, @@ -241,10 +228,8 @@ def write_bundle( if inner_setup: created_files.extend(write_notebooks(inner_setup, src_dir)) - # 5. Build SETUP.md — a root-level, human-readable summary of every - # external step the user must take before ``bundle run``. This is - # additive to the setup/ notebooks above; the setup notebooks are - # the executable path, SETUP.md is the checklist. + # 5. Build SETUP.md — a root-level, human-readable summary of every external step needed before + # ``bundle run``. Additive to the setup/ notebooks above (those are the executable path). all_notebooks = list(workflow.notebooks) for inner in workflow.inner_workflows: all_notebooks.extend(inner.notebooks) @@ -255,54 +240,37 @@ def write_bundle( for inner in workflow.inner_workflows: parameter_approximations.extend(inner.parameter_approximations) known_bundle_jobs = {resource_key} | {normalize_task_key(inner.name) for inner in workflow.inner_workflows} - # ``manual_parameters`` was collected above (before YAML emission) so - # the broken values are also stripped from the on-disk YAML. - # VAREX3-003: manual_variable_rollup SetupTasks emitted by - # workflow_preparer surface in SETUP.md so the user knows where to add - # a roll-up notebook. - rollup_configs = [ - st.config - for st in workflow.setup_tasks - if st.type == "manual_variable_rollup" - ] + # manual_parameters was collected above (before YAML emission) so broken values are stripped on disk too. + # VAREX3-003: manual_variable_rollup SetupTasks from workflow_preparer surface in SETUP.md so the user + # knows where to add a roll-up notebook. + rollup_configs = [task.config for task in workflow.setup_tasks if task.type == "manual_variable_rollup"] for inner in workflow.inner_workflows: - rollup_configs.extend( - st.config for st in inner.setup_tasks if st.type == "manual_variable_rollup" - ) + rollup_configs.extend(task.config for task in inner.setup_tasks if task.type == "manual_variable_rollup") dynamic_dispatch_configs = [ - st.config for st in workflow.setup_tasks if st.type == "dynamic_notebook_dispatch" - ] - unresolved_library_configs = [ - st.config for st in workflow.setup_tasks if st.type == "unresolved_library" - ] - manual_variable_init_configs = [ - st.config for st in workflow.setup_tasks if st.type == "manual_variable_init" + task.config for task in workflow.setup_tasks if task.type == "dynamic_notebook_dispatch" ] + unresolved_library_configs = [task.config for task in workflow.setup_tasks if task.type == "unresolved_library"] + manual_variable_init_configs = [task.config for task in workflow.setup_tasks if task.type == "manual_variable_init"] manual_schedule_time_of_day_configs = [ - st.config for st in workflow.setup_tasks if st.type == "manual_schedule_time_of_day" - ] - manual_credential_configs = [ - st.config for st in workflow.setup_tasks if st.type == "manual_credential" + task.config for task in workflow.setup_tasks if task.type == "manual_schedule_time_of_day" ] + manual_credential_configs = [task.config for task in workflow.setup_tasks if task.type == "manual_credential"] for inner in workflow.inner_workflows: dynamic_dispatch_configs.extend( - st.config for st in inner.setup_tasks if st.type == "dynamic_notebook_dispatch" + task.config for task in inner.setup_tasks if task.type == "dynamic_notebook_dispatch" ) unresolved_library_configs.extend( - st.config for st in inner.setup_tasks if st.type == "unresolved_library" + task.config for task in inner.setup_tasks if task.type == "unresolved_library" ) manual_variable_init_configs.extend( - st.config for st in inner.setup_tasks if st.type == "manual_variable_init" + task.config for task in inner.setup_tasks if task.type == "manual_variable_init" ) manual_schedule_time_of_day_configs.extend( - st.config for st in inner.setup_tasks if st.type == "manual_schedule_time_of_day" + task.config for task in inner.setup_tasks if task.type == "manual_schedule_time_of_day" ) - manual_credential_configs.extend( - st.config for st in inner.setup_tasks if st.type == "manual_credential" - ) - # LSC3-006: union typed SecretInstructions from the workflow (and - # inner workflows) with the notebook-scanned scopes so SETUP.md and - # create_secrets.py reference the same set of (scope, key) pairs. + manual_credential_configs.extend(task.config for task in inner.setup_tasks if task.type == "manual_credential") + # LSC3-006: union typed SecretInstructions (workflow + inner) with notebook-scanned scopes so SETUP.md + # and create_secrets.py reference the same set of (scope, key) pairs. all_secret_instructions = list(workflow.secrets) for inner in workflow.inner_workflows: all_secret_instructions.extend(inner.secrets) @@ -343,22 +311,47 @@ def write_bundle( return created_files -def main() -> None: - """CLI entry point for DAB bundle generation.""" +def _default_report_path(output_dir: Path) -> Path: + """Returns the conventional report path under a migration dir's .work/ folder. + + Prefers the modify-stamped report; falls back to the raw translation report + when modify was not run (no configuration applied). + """ + work = Path(output_dir) / ".work" + stamped = work / "translation_report.stamped.json" + if stamped.exists(): + return stamped + return work / "translation_report.json" + + +def main(argv: list[str] | None = None) -> int: + """Package-phase entry point for DAB bundle generation. + + Returns a process exit code so the adapter can run this phase in-process (instead of spawning a + second interpreter) and still propagate failures. + """ parser = argparse.ArgumentParser( description="Generate a Databricks Declarative Automation Bundle from a translation report.", ) parser.add_argument( "--report", type=Path, - required=True, - help="Path to the translation report or pipeline IR JSON produced by the translate phase.", + default=None, + help=( + "Path to the (stamped) translation report produced by translate/modify. " + "Defaults to /.work/translation_report.stamped.json (falling back to " + "/.work/translation_report.json when modify was not run)." + ), ) parser.add_argument( "--output-dir", type=Path, - default=Path("./orchestra_output/bundle"), - help="Output directory for the DAB bundle (default: ./orchestra_output/bundle).", + default=Path("./flowx_output"), + help=( + "Migration output directory. The bundle (databricks.yml, resources/, src/) is " + "written here alongside the metadata/ folder; the transient .work/ folder is pruned " + "after a successful build." + ), ) parser.add_argument( "--catalog", @@ -385,23 +378,30 @@ def main() -> None: help="Databricks CLI profile to use when downloading workspace artifacts.", ) parser.add_argument( - "--no-vendor-workspace-files", + "--no-download-workspace-files", action="store_true", help=( "Skip downloading workspace-resident notebooks / Python files / JARs. " "Tasks keep their original workspace paths and the bundle is not self-contained." ), ) - args = parser.parse_args() + parser.add_argument( + "--keep-intermediates", + action="store_true", + help="Keep the transient .work/ folder (translation report + IR) instead of pruning it.", + ) + args = parser.parse_args(argv) + if args.report is None: + args.report = _default_report_path(args.output_dir) if not args.report.exists(): print(f"Error: Report file not found: {args.report}", file=sys.stderr) - sys.exit(1) + return 1 if args.profile: set_profile(args.profile) - if not args.no_vendor_workspace_files: + if not args.no_download_workspace_files: workspace_paths = collect_workspace_artifact_paths(args.report) if workspace_paths: if not prompt_for_auth_if_missing(workspace_paths): @@ -409,7 +409,7 @@ def main() -> None: "Aborted. Run `databricks auth login --host ` and retry.", file=sys.stderr, ) - sys.exit(2) + return 2 enable_workspace_downloads(True) print(f"Loading translation report: {args.report}") @@ -417,7 +417,7 @@ def main() -> None: if not workflows: print("No translated pipelines found in the report.", file=sys.stderr) - sys.exit(1) + return 1 all_created: list[Path] = [] for index, workflow in enumerate(workflows): @@ -437,12 +437,21 @@ def main() -> None: all_created.extend(created) print(f" [{index + 1}/{len(workflows)}] {workflow.name}: {len(created)} files") + if not args.keep_intermediates: + work_dir = args.output_dir / ".work" + if work_dir.is_dir(): + import shutil + + shutil.rmtree(work_dir, ignore_errors=True) + print(f"Pruned transient {work_dir}") + print(f"\nBundle generation complete: {len(all_created)} files written to {args.output_dir}") print("\nNext steps:") print(" 1. Review the generated notebooks in src/") print(" 2. Run the setup notebooks to create secrets and volumes") print(" 3. Validate the bundle: databricks bundle validate") print(" 4. Deploy: databricks bundle deploy -t dev") + return 0 def _warn(task_key: str, message: str) -> None: @@ -453,13 +462,9 @@ def _warn(task_key: str, message: str) -> None: _DEFAULT_SPARK_VERSION = "15.4.x-scala2.12" _DEFAULT_NODE_TYPE_ID = "Standard_DS3_v2" -# C-29 (NB-ITER4-002): a real DBR version string matches e.g. -# "15.4.x-scala2.12" / "15.4.x-photon-scala2.12". ADF expressions like -# ``@if(equals(item()?.photon,true),...)`` slip through unfiltered today -# and land in ``databricks.yml`` as the spark_version variable default, -# which bundle deploy rejects. The regex anchors on the canonical -# Databricks Runtime shape so unrecognised strings fall through to the -# safe default. +# C-29 (NB-ITER4-002): anchor on the canonical DBR version shape (e.g. "15.4.x-photon-scala2.12") so +# unresolved ADF expressions like @if(equals(item()?.photon,true),...) fall through to the safe default +# instead of landing in databricks.yml as a spark_version that bundle deploy rejects. _DBR_VERSION_RE = re.compile(r"^\d+\.\d+\.x(-[a-z0-9.]+)*$") @@ -481,7 +486,7 @@ def _is_valid_node_type_id(value: Any) -> bool: return False if value.startswith("@"): return False - if any(ch.isspace() for ch in value): + if any(char.isspace() for char in value): return False return True @@ -497,19 +502,13 @@ def _infer_bundle_cluster_defaults(workflow: PreparedWorkflow) -> tuple[str, str """ from collections import Counter - # C-29 (NB-ITER4-002): filter out unparseable spark_version / - # node_type_id hints before Counter so unresolved ADF expressions - # (e.g. ``@if(equals(item()?.photon,true),...)``) don't land as the - # bundle's default and break ``databricks bundle deploy``. + # C-29 (NB-ITER4-002): filter out unparseable spark_version / node_type_id hints before Counter so + # unresolved ADF expressions don't land as the bundle default and break ``databricks bundle deploy``. spark_versions = [ - hint["spark_version"] - for hint in workflow.cluster_hints - if _is_valid_spark_version(hint.get("spark_version")) + hint["spark_version"] for hint in workflow.cluster_hints if _is_valid_spark_version(hint.get("spark_version")) ] node_types = [ - hint["node_type_id"] - for hint in workflow.cluster_hints - if _is_valid_node_type_id(hint.get("node_type_id")) + hint["node_type_id"] for hint in workflow.cluster_hints if _is_valid_node_type_id(hint.get("node_type_id")) ] spark_version = Counter(spark_versions).most_common(1)[0][0] if spark_versions else _DEFAULT_SPARK_VERSION @@ -571,6 +570,7 @@ def _build_databricks_yml( spark_version: str = _DEFAULT_SPARK_VERSION, node_type_id: str = _DEFAULT_NODE_TYPE_ID, include_cluster_variables: bool = True, + extra_variables: dict[str, Any] | None = None, ) -> dict[str, Any]: """Builds the root ``databricks.yml`` configuration as a dict. @@ -586,6 +586,9 @@ def _build_databricks_yml( False when no task in the bundle uses classic compute (every generated notebook runs on serverless), so the bundle stays free of unused tunables. + extra_variables: Additional variable declarations (name -> DAB + declaration dict) to merge into the ``variables`` block, e.g. + the source-side variables a Lakeflow Connect pipeline references. Returns: Dict ready for YAML serialization. @@ -612,9 +615,10 @@ def _build_databricks_yml( "description": "Databricks Runtime for the default job_cluster.", "default": spark_version, } - # Declare a variable for each cross-bundle ExecutePipeline reference so - # `${var.X_job_id}` resolves and `bundle validate` passes. Users fill in - # the numeric job ID per SETUP.md. + for name, declaration in (extra_variables or {}).items(): + variables.setdefault(name, declaration) + # Declare a variable for each cross-bundle ExecutePipeline reference so `${var.X_job_id}` resolves and + # `bundle validate` passes. Users fill in the numeric job ID per SETUP.md. for variable_name, target_pipeline in sorted(_cross_bundle_variables.items()): variables[variable_name] = { "description": ( @@ -774,6 +778,71 @@ def _wrap_pipeline_resource(resource: dict[str, Any]) -> dict[str, Any]: return {"resources": {"pipelines": {resource["resource_key"]: resource["definition"]}}} +_VAR_REFERENCE_RE = re.compile(r"\$\{var\.([A-Za-z_][A-Za-z0-9_]*)\}") + +_BUILTIN_BUNDLE_VARIABLES = frozenset({"catalog", "schema", "node_type_id", "spark_version"}) + + +def _collect_variable_references(value: Any) -> set[str]: + """Returns every ``${var.NAME}`` variable name referenced anywhere within *value*.""" + refs: set[str] = set() + if isinstance(value, str): + refs.update(_VAR_REFERENCE_RE.findall(value)) + elif isinstance(value, dict): + for item in value.values(): + refs |= _collect_variable_references(item) + elif isinstance(value, list): + for item in value: + refs |= _collect_variable_references(item) + return refs + + +def _build_pipeline_variable_declarations( + pipeline_resources: list[dict[str, Any]], + catalog: str, + schema: str, +) -> dict[str, Any]: + """Returns ``variables:`` declarations for every ``${var.…}`` a pipeline resource references. + + Lakeflow Connect ingestion definitions fall back to ``${var.source_catalog}`` / + ``${var.source_schema}`` (and could reference further variables) when the translator + couldn't resolve a literal. Each such variable must appear in the root ``variables:`` + block or ``databricks bundle validate`` fails on an undefined reference. + + Args: + pipeline_resources: The pipeline-resource dicts collected for the bundle. + catalog: The migration target catalog (used as the source_catalog default). + schema: The migration target schema (used as the source_schema default). + + Returns: + Mapping of variable name to its DAB declaration dict. ``source_catalog`` and + ``source_schema`` get a sensible default so validation passes out of the box; + any other referenced variable is declared without a default (user fills it in). + """ + referenced: set[str] = set() + for resource in pipeline_resources: + referenced |= _collect_variable_references(resource.get("definition")) + referenced -= _BUILTIN_BUNDLE_VARIABLES + + source_defaults = {"source_catalog": catalog, "source_schema": schema} + declarations: dict[str, Any] = {} + for name in sorted(referenced): + if name in source_defaults: + kind = name.removeprefix("source_") + declarations[name] = { + "description": ( + f"Source-side {kind} the Lakeflow Connect ingestion reads from. " + f"Defaults to the migration {kind}; override with the real source {kind}." + ), + "default": source_defaults[name], + } + else: + declarations[name] = { + "description": f"Value for ${{var.{name}}} referenced by a generated pipeline resource.", + } + return declarations + + def _collect_required_cluster_keys(tasks: list[dict[str, Any]]) -> set[str]: """Walks every task and returns the set of job_cluster keys actually bound. @@ -797,10 +866,9 @@ def _strip_compute_mode_markers(tasks: list[dict[str, Any]]) -> None: task.pop("_compute_mode", None) -# Patterns that signal a base_parameter value couldn't be evaluated cleanly. -# When any task references an *existing* notebook (absolute workspace path), -# flowx can't inject the runtime computation, so these end up as manual -# work for the user. +# Patterns that signal a base_parameter value couldn't be evaluated cleanly. When a task references an +# existing notebook (absolute workspace path), flowx can't inject the runtime computation, so these +# end up as manual work for the user. _HYBRID_ADF_FN_RE = re.compile(r"@[a-zA-Z][a-zA-Z0-9]*\(") _PYTHON_CODE_HINTS = ("dbutils.widgets.get(", "datetime.now(", "datetime.fromisoformat(") @@ -893,9 +961,8 @@ def _bind_cluster_to_notebook_tasks(tasks: list[dict[str, Any]]) -> None: continue compute_mode = task.get("_compute_mode") if compute_mode == "serverless": - # Serverless cannot host jar/whl libraries. When the task - # ships libraries we must still bind a classic cluster so the - # Jobs API accepts the libraries block. + # Serverless cannot host jar/whl libraries, so when the task ships libraries we still bind a + # classic cluster so the Jobs API accepts the libraries block. if _task_has_jar_or_whl_libraries(task): task["job_cluster_key"] = DEFAULT_JOB_CLUSTER_KEY continue @@ -904,10 +971,8 @@ def _bind_cluster_to_notebook_tasks(tasks: list[dict[str, Any]]) -> None: task["job_cluster_key"] = cluster_key continue notebook_path = notebook_task.get("notebook_path", "") - # Stub notebooks (../src/...) are normally left unbound for - # serverless compute. But when libraries are attached we must - # bind to a real cluster (NB-2) -- serverless cannot install - # jar / whl libraries. + # Stub notebooks (../src/...) are normally left unbound for serverless compute, but when libraries + # are attached we must bind a real cluster (NB-2) -- serverless cannot install jar/whl libraries. if notebook_path.startswith("../src/"): if _task_has_jar_or_whl_libraries(task): task["job_cluster_key"] = DEFAULT_JOB_CLUSTER_KEY @@ -1100,9 +1165,8 @@ def visit(task: dict[str, Any]) -> None: if _is_dangling(value): job_parameters[param_name] = "" - # C-12: condition_task operands can also carry dangling refs - # when an upstream renamed task disappeared between rewrite - # passes. C-43: record each neutralised operand for SETUP.md. + # C-12: condition_task operands can also carry dangling refs when an upstream renamed task + # disappeared between rewrite passes. C-43: record each neutralised operand for SETUP.md. condition_task = task.get("condition_task") or {} if condition_task: task_key = task.get("task_key", "") @@ -1189,15 +1253,12 @@ def _build_job_resource( Dict ready for YAML serialization. """ _rewrite_post_branch_dependencies(workflow.tasks) - # For inner jobs (invoked via run_job_task), notebooks live in the parent - # workflow's notebooks list — pass them in so widget auto-augment can - # still find the bound notebook and populate base_parameters. + # For inner jobs (run_job_task), notebooks live in the parent workflow's list — pass them in so widget + # auto-augment can still find the bound notebook and populate base_parameters. augment_scope = list(workflow.notebooks) + list(extra_notebooks_for_augment or []) _augment_base_parameters(workflow.tasks, augment_scope) - # Task values don't cross ``run_job_task`` boundaries; any such - # reference in this job resolves to an empty string at runtime. Emit - # the empty string now so SETUP.md §4 flags it. C-43: a blanked - # condition operand silently makes the predicate always-true, so record + # Task values don't cross run_job_task boundaries; such a reference resolves to an empty string at + # runtime, so emit it now for SETUP.md §4. C-43: a blanked condition operand is always-true, so record # each neutralised condition for the SETUP.md re-wiring section. _neutralized_conditions.extend( _strip_dangling_task_value_refs(workflow.tasks, _collect_all_task_keys(workflow.tasks)) @@ -1221,17 +1282,30 @@ def _build_job_resource( _strip_compute_mode_markers(workflow.tasks) if workflow.parameters: - job_def["parameters"] = workflow.parameters + # Emit each job parameter once in the DAB shape ({name, default}); dropping the internal ``type`` + # field keeps both bundle paths byte-identical and matches the Databricks job-parameter schema. + seen_param_names: set[str | None] = set() + normalized_parameters: list[dict[str, Any]] = [] + for parameter in workflow.parameters: + name = parameter.get("name") + if name in seen_param_names: + continue + seen_param_names.add(name) + entry: dict[str, Any] = {"name": name} + default = parameter.get("default") + if default is not None: + # Databricks job-parameter defaults are strings; JSON-encode + # Array / Object defaults so the YAML carries valid JSON. + entry["default"] = json.dumps(default) if isinstance(default, (list, dict)) else default + normalized_parameters.append(entry) + job_def["parameters"] = normalized_parameters # C-10 (SCHED-001): render the workflow schedule / trigger spec. schedule_spec = getattr(workflow, "schedule", None) if schedule_spec: _apply_schedule_to_job(job_def, schedule_spec) - # SCHED3-003: trigger-supplied per-pipeline parameter overrides - # update the matching job.parameter defaults so scheduled runs - # receive the trigger's pinned values instead of the bare pipeline - # default. Overrides only mutate existing declared parameters; - # unknown names are silently ignored to keep job_def well-formed. + # SCHED3-003: trigger-supplied parameter overrides update the matching job.parameter defaults so + # scheduled runs get the trigger's pinned values; unknown names are ignored to keep job_def valid. overrides = schedule_spec.get("parameter_overrides") or {} if overrides and job_def.get("parameters"): for entry in job_def["parameters"]: @@ -1296,12 +1370,9 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]: return workflows if "translations" in report: - # Aggregated translation_report.json format: ``translations`` is a - # flat list of ``{pipeline, ir, status, ...}`` entries. Group by - # pipeline name and route each group through the same - # ``_pipeline_dict_to_workflow`` machinery as the single-pipeline IR - # format, so secret discovery / setup tasks / control-flow handling - # all match. + # Aggregated translation_report.json: ``translations`` is a flat list of {pipeline, ir, status}. + # Group by pipeline and route each group through _pipeline_dict_to_workflow (same machinery as the + # single-pipeline IR format) so secret discovery / setup tasks / control-flow handling all match. pipelines: dict[str, list[dict]] = {} pipeline_params: dict[str, list[dict[str, Any]]] = {} pipeline_schedules: dict[str, dict[str, Any]] = {} @@ -1313,16 +1384,13 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]: if not ir: continue pipelines.setdefault(pipeline_name, []).append(ir) - # Round-trip pipeline-level parameters either supplied per- - # translation (newer report shape) or alongside the ir under - # an ``ir.parameters`` key (older single-pipeline serialisations - # roundtripped through this aggregator). + # Round-trip pipeline-level parameters supplied per-translation (newer shape) or under + # ir.parameters (older single-pipeline serialisations roundtripped through this aggregator). params = translation.get("parameters") or ir.get("parameters") if params and pipeline_name not in pipeline_params: pipeline_params[pipeline_name] = list(params) - # Likewise carry pipeline-level ``schedule`` through to the - # rehydrated pipeline_dict so trigger-derived schedule / trigger - # blocks survive the aggregated report shape. + # Likewise carry pipeline-level schedule through to the rehydrated pipeline_dict so + # trigger-derived schedule/trigger blocks survive the aggregated report shape. schedule = translation.get("schedule") or ir.get("schedule") if schedule and pipeline_name not in pipeline_schedules: pipeline_schedules[pipeline_name] = dict(schedule) @@ -1351,11 +1419,10 @@ def _pipeline_dict_to_workflow(pipeline_dict: dict[str, Any]) -> PreparedWorkflo expression resolution, and motif handling without duplicating the per-activity preparer logic. """ - pipeline, parameters = pipeline_dict_to_ir(pipeline_dict) - workflow = prepare_workflow(pipeline) - if parameters: - workflow.parameters.extend(parameters) - return workflow + pipeline, _parameters = pipeline_dict_to_ir(pipeline_dict) + # prepare_workflow already carries pipeline.parameters onto the workflow, so re-extending here would + # duplicate every job parameter (the same dict twice -> a duplicate ``region`` entry in YAML). + return prepare_workflow(pipeline) def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[dict[str, Any]]]: @@ -1377,14 +1444,16 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d entry: dict[str, Any] = {"name": param["name"]} if "default" in param and param["default"] is not None: default_value = param["default"] - # Bool / int / float defaults must survive the JSON round-trip - # as their declared type so the emitted YAML carries a real - # boolean / number, not a quoted string. String defaults go - # through normalize_value to resolve embedded ADF refs. + # Bool / int / float defaults must survive the JSON round-trip as their declared type so the + # YAML carries a real boolean/number; string defaults go through normalize_value for ADF refs. if isinstance(default_value, bool): entry["default"] = default_value elif isinstance(default_value, (int, float)): entry["default"] = default_value + elif isinstance(default_value, (list, dict)): + # Array / Object defaults are JSON-encoded to a string at emission time; keep the structure + # here so both bundle paths converge (a Python str() here would emit invalid JSON). + entry["default"] = default_value else: entry["default"] = normalize_value(str(default_value)) parameters.append(entry) @@ -1392,31 +1461,30 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d name=pipeline_dict.get("name", "unknown"), tasks=activities, parameters=parameters or None, - translation_preferences=_reconstruct_preferences(pipeline_dict.get("translation_preferences")), + translation_configuration=_reconstruct_configuration(pipeline_dict.get("translation_configuration")), schedule=pipeline_dict.get("schedule"), ) return pipeline, parameters -def _reconstruct_preferences(raw: dict[str, Any] | None) -> Any: - """Rebuilds a :class:`TranslationPreferences` from its serialised form. +def _reconstruct_configuration(raw: dict[str, Any] | None) -> Any: + """Rebuilds a :class:`TranslationConfiguration` from its serialised form. Args: - raw: Dict emitted by ``engine._preferences_to_dict``, or ``None`` - when the report carries no preferences. + raw: Dict emitted by ``engine._configuration_to_dict``, or ``None`` + when the report carries no configuration. Returns: - A :class:`TranslationPreferences` instance, or ``None`` when + A :class:`TranslationConfiguration` instance, or ``None`` when *raw* is falsy. """ if not raw: return None - from flowx.adapter.models import TranslationPreferences + from flowx.adapter.models import TranslationConfiguration - # Reports authored before the databricks_task_compute option was - # removed may still carry that key; drop it silently so old reports - # remain rehydratable. - return TranslationPreferences( + # Reports authored before the databricks_task_compute option was removed may still carry that key; + # drop it silently so old reports remain rehydratable. + return TranslationConfiguration( copy_activity_paradigm=raw.get("copy_activity_paradigm", "notebook"), non_databricks_task_compute=raw.get("non_databricks_task_compute", "serverless"), use_lakeflow_connectors=raw.get("use_lakeflow_connectors", "existing"), @@ -1467,6 +1535,11 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: body=task_ir.get("body"), headers=task_ir.get("headers"), authentication=task_ir.get("authentication"), + body_code=task_ir.get("body_code"), + body_imports=list(task_ir.get("body_imports") or []), + body_required_parameters=dict(task_ir.get("body_required_parameters") or {}), + disable_cert_validation=bool(task_ir.get("disable_cert_validation", False)), + http_request_timeout_seconds=task_ir.get("http_request_timeout_seconds"), ) if task_type == "SetVariableActivity": return SetVariableActivity( @@ -1562,9 +1635,8 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: right=task_ir.get("right", ""), if_true_activities=[_reconstruct_ir(child) for child in task_ir.get("if_true_activities") or []], if_false_activities=[_reconstruct_ir(child) for child in task_ir.get("if_false_activities") or []], - # C-14 (CF3-001 / VAREX3-001): preserve bridge fields so the - # preparer can re-synthesise the hidden _bridge SetVariable task - # after a JSON roundtrip. + # C-14 (CF3-001 / VAREX3-001): preserve bridge fields so the preparer can re-synthesise the + # hidden _bridge SetVariable task after a JSON roundtrip. bridge_notebook_code=task_ir.get("bridge_notebook_code"), bridge_notebook_imports=list(task_ir.get("bridge_notebook_imports") or []), bridge_required_parameters=dict(task_ir.get("bridge_required_parameters") or {}), @@ -1581,9 +1653,8 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: for case in task_ir.get("cases") or [] ], default_activities=[_reconstruct_ir(child) for child in task_ir.get("default_activities") or []], - # C-14 (CF3-001 / VAREX3-001): preserve bridge fields for Switch - # so the preparer can re-synthesise the bridge task after a - # JSON roundtrip. + # C-14 (CF3-001 / VAREX3-001): preserve bridge fields for Switch so the preparer can + # re-synthesise the bridge task after a JSON roundtrip. bridge_notebook_code=task_ir.get("bridge_notebook_code"), bridge_notebook_imports=list(task_ir.get("bridge_notebook_imports") or []), bridge_required_parameters=dict(task_ir.get("bridge_required_parameters") or {}), @@ -1640,6 +1711,7 @@ def _common_activity_kwargs(task_ir: dict[str, Any]) -> dict[str, Any]: "parameter_approximations": list(task_ir.get("parameter_approximations") or []), "required_parameters": dict(task_ir.get("required_parameters") or {}), "compute_mode": task_ir.get("compute_mode"), + "notifications": task_ir.get("notifications"), } @@ -1653,4 +1725,4 @@ def _reconstruct_dependencies(raw: list[dict[str, Any]] | None) -> list[Dependen if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/src/orchestra/bundler/inner_job_params.py b/src/flowx/bundler/inner_job_params.py similarity index 94% rename from src/orchestra/bundler/inner_job_params.py rename to src/flowx/bundler/inner_job_params.py index bbb51b3..a42e1f7 100644 --- a/src/orchestra/bundler/inner_job_params.py +++ b/src/flowx/bundler/inner_job_params.py @@ -1,4 +1,4 @@ -"""Collects and normalize parameters for ForEach inner jobs.""" +"""Collects and normalizes parameters for ForEach inner jobs.""" from __future__ import annotations @@ -74,17 +74,14 @@ def collect_inner_job_params( _scan_tasks(tasks, param_names, item_field_names=item_field_names, variable_names=variable_names) if raw_ir_tasks: - _scan_ir_tasks( - raw_ir_tasks, param_names, item_field_names=item_field_names, variable_names=variable_names - ) + _scan_ir_tasks(raw_ir_tasks, param_names, item_field_names=item_field_names, variable_names=variable_names) var_task_keys = variable_task_keys or {} parameters: list[dict[str, Any]] = [] for name in sorted(param_names): - # C-06: variables with a known setter task on the parent job route - # via {{tasks.X.values.Y}} -- they must NOT show up as inner-job - # parameter declarations. + # C-06: variables with a known setter task on the parent job route via {{tasks.X.values.Y}}, + # so they must NOT show up as inner-job parameter declarations. if name in variable_names and name in var_task_keys: continue param: dict[str, Any] = {"name": name} @@ -92,9 +89,8 @@ def collect_inner_job_params( param["default"] = "" parameters.append(param) - # "item" (bare @item()) maps to {{input}} (the full iteration value); - # item field names (@item().field) map to {{input.}}; - # pipeline params / variables map to {{job.parameters.}}. + # Map bare @item() -> {{input}}, @item().field -> {{input.}}, and pipeline params/variables + # -> {{job.parameters.}}. job_parameters: dict[str, str] = {} for name in sorted(param_names): if name == "item": @@ -154,28 +150,28 @@ def _scan_tasks( variable_names: Optional accumulator for names sourced from ``variables('X')`` references (separate from pipeline params). """ - kw: dict[str, Any] = {"item_field_names": item_field_names, "variable_names": variable_names} + field_name_kwargs: dict[str, Any] = {"item_field_names": item_field_names, "variable_names": variable_names} for task in tasks: notebook_task = task.get("notebook_task", {}) params = notebook_task.get("base_parameters", {}) for value in params.values(): - _extract_refs(value, param_names, **kw) + _extract_refs(value, param_names, **field_name_kwargs) run_job_task = task.get("run_job_task", {}) for value in run_job_task.get("job_parameters", {}).values(): - _extract_refs(value, param_names, **kw) + _extract_refs(value, param_names, **field_name_kwargs) condition_task = task.get("condition_task", {}) if condition_task: - _extract_refs(condition_task.get("left", ""), param_names, **kw) - _extract_refs(condition_task.get("right", ""), param_names, **kw) - _scan_tasks(condition_task.get("if_true", []), param_names, **kw) - _scan_tasks(condition_task.get("if_false", []), param_names, **kw) + _extract_refs(condition_task.get("left", ""), param_names, **field_name_kwargs) + _extract_refs(condition_task.get("right", ""), param_names, **field_name_kwargs) + _scan_tasks(condition_task.get("if_true", []), param_names, **field_name_kwargs) + _scan_tasks(condition_task.get("if_false", []), param_names, **field_name_kwargs) for_each_task = task.get("for_each_task", {}) body = for_each_task.get("task") if body: - _scan_tasks([body], param_names, **kw) + _scan_tasks([body], param_names, **field_name_kwargs) def _scan_ir_tasks( diff --git a/src/orchestra/bundler/notebook_writer.py b/src/flowx/bundler/notebook_writer.py similarity index 100% rename from src/orchestra/bundler/notebook_writer.py rename to src/flowx/bundler/notebook_writer.py diff --git a/src/orchestra/bundler/prereqs_writer.py b/src/flowx/bundler/prereqs_writer.py similarity index 94% rename from src/orchestra/bundler/prereqs_writer.py rename to src/flowx/bundler/prereqs_writer.py index a66f7e5..0149507 100644 --- a/src/orchestra/bundler/prereqs_writer.py +++ b/src/flowx/bundler/prereqs_writer.py @@ -111,32 +111,25 @@ class Prereqs: network_endpoints: list[NetworkEndpoint] = field(default_factory=list) manual_parameters: list[ManualParameter] = field(default_factory=list) parameter_approximations: list[ParameterApproximation] = field(default_factory=list) - # VAREX3-003: variables mutated inside a ForEach inner-job that a - # sibling task reads. Each entry is the SetupTask.config dict shape - # ({variable_name, parent_foreach, message}). + # VAREX3-003: variables mutated inside a ForEach inner-job that a sibling reads; each entry is the + # SetupTask.config dict ({variable_name, parent_foreach, message}). manual_variable_rollups: list[dict[str, Any]] = field(default_factory=list) - # C-28 (NB-ITER4-001): notebook activities whose ADF ``notebookPath`` is - # a runtime expression the translator couldn't resolve. Each entry is - # the SetupTask.config dict ({task_key, activity_name, expression, - # widget_name}). + # C-28 (NB-ITER4-001): notebook activities whose ADF notebookPath is an unresolved runtime expression; + # each entry is the SetupTask.config dict ({task_key, activity_name, expression, widget_name}). dynamic_notebook_dispatches: list[dict[str, Any]] = field(default_factory=list) - # C-30 (NB-ITER4-003): library descriptor jar/whl paths the translator - # couldn't resolve to a literal/dab_ref. Each entry is the SetupTask - # config dict ({task_key, library_type, expression, missing}). + # C-30 (NB-ITER4-003): library jar/whl paths unresolved to a literal/dab_ref; each entry is the + # SetupTask config dict ({task_key, library_type, expression, missing}). unresolved_libraries: list[dict[str, Any]] = field(default_factory=list) - # C-33 (VAREX4-001/CF4-003): SetVariable activities whose ADF - # expression couldn't be lowered. Each entry is the SetupTask config - # dict ({task_key, variable_name, expression}). + # C-33 (VAREX4-001/CF4-003): SetVariable activities whose ADF expression couldn't be lowered; each + # entry is the SetupTask config dict ({task_key, variable_name, expression}). manual_variable_inits: list[dict[str, Any]] = field(default_factory=list) # C-36 (SCHED4-001): scheduled jobs whose recurrence carried # hours/minutes/weekDays the cron emitter could not encode. manual_schedule_time_of_day: list[dict[str, Any]] = field(default_factory=list) # C-39 (LSC4-004): MSI / CredentialReference cluster substitutions. manual_credentials: list[dict[str, Any]] = field(default_factory=list) - # C-43 (CF5-001 / CF5-002): condition_task operands the bundler had to - # blank because they referenced a task in another job. Each entry is - # {task_key, field, original_ref}. A blanked operand makes the - # predicate always-true, so the user must re-wire the condition. + # C-43 (CF5-001 / CF5-002): condition_task operands blanked because they referenced a task in another + # job ({task_key, field, original_ref}); a blanked operand is always-true, so the user must re-wire it. neutralized_conditions: list[dict[str, str]] = field(default_factory=list) def is_empty(self) -> bool: @@ -393,11 +386,9 @@ def build_prereqs( # the tasks (in case upstream still emits them). cross_bundle.extend(collect_cross_bundle_refs(tasks, known_bundle_jobs)) - # LSC3-006: union notebook-scanned secrets with the workflow's typed - # SecretInstruction list so SETUP.md Option A (scope/key checklist) and - # Option B (create_secrets.py from workflow.secrets) reference the same - # set of (scope, key) pairs. De-dupe by hash; later additions don't - # overwrite earlier values. + # LSC3-006: union notebook-scanned secrets with the workflow's typed SecretInstruction list so + # SETUP.md Option A (scope/key checklist) and Option B (create_secrets.py) reference the same + # (scope, key) set. De-dupe by scope; later additions don't overwrite earlier values. secrets = scan_notebooks_for_secrets(notebooks) for instruction in secret_instructions or []: secrets.setdefault(instruction.scope, set()).add(instruction.key) @@ -490,7 +481,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("") lines.append( "The following notebooks are stubs that raise `NotImplementedError`. " - "Flowx could not download the source (either no workspace path was " + "flowx could not download the source (either no workspace path was " "supplied in ADF, or the path did not resolve against the " "authenticated workspace). Replace each stub with the real logic." ) @@ -510,7 +501,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("") lines.append( "If the workspace path exists in a reachable Databricks workspace, you can " - "have Flowx re-ingest it by running `databricks workspace export` and " + "have flowx re-ingest it by running `databricks workspace export` and " "placing the result at the indicated bundle path." ) lines.append("") @@ -520,7 +511,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("") lines.append( "Each row below describes a `run_job_task` that invokes a job **not** " - "defined in this bundle. Flowx emitted a bundle variable for each " + "defined in this bundle. flowx emitted a bundle variable for each " "one (`${var.}`) so `databricks bundle validate` passes. " "Before running, populate the variable with the numeric job ID the " "target pipeline was deployed under — either set a `default:` in " @@ -583,7 +574,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("## Parameter substitutions") lines.append("") lines.append( - "Flowx mapped the ADF expressions below to Databricks dynamic value " + "flowx mapped the ADF expressions below to Databricks dynamic value " "references so they land directly in the bundle YAML. The substitutions are " "semantically *close* but not identical to the originals; review the listed " "caveats and decide whether each replacement is acceptable for your workload." @@ -604,7 +595,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("") lines.append( "The ADF activities below carried a runtime expression for " - "`notebookPath`. Flowx emitted a dispatch-stub notebook for " + "`notebookPath`. flowx emitted a dispatch-stub notebook for " "each one that reads the resolved path from the listed widget and " "calls `dbutils.notebook.run()`. Supply the widget value at job " "runtime (via `--params`, a parent task value, or job parameter " @@ -650,7 +641,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("") lines.append( "The ADF SetVariable activities below carried expressions the " - "translator couldn't lower. Flowx blanked the variable's " + "translator couldn't lower. flowx blanked the variable's " "initial value to keep the bundle YAML valid. Compute the real " "value yourself (e.g. via a parent task value or runtime widget) " "before downstream tasks read the variable." @@ -691,7 +682,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append( "The cluster compute backing the tasks below was authenticated in " "ADF via a managed identity / CredentialReference that has no " - "direct Databricks equivalent. Flowx defaulted the bundle's " + "direct Databricks equivalent. flowx defaulted the bundle's " "default_cluster to `single_user_name: ${workspace.current_user.userName}` " "so deployment works for the deploying user, but production runs " "should swap that for a service principal." @@ -705,8 +696,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: auth = entry.get("authentication", "") note = entry.get( "note", - "Swap `single_user_name` to the SP application ID or set " - "`run_as.service_principal_name` on the job.", + "Swap `single_user_name` to the SP application ID or set `run_as.service_principal_name` on the job.", ) lines.append(f"| `{source}` | `{linked_service}` | `{auth}` | {note} |") lines.append("") @@ -718,7 +708,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: "The IfCondition tasks below referenced a task value that lives only " "in another job (typically a parent-job init task hoisted out of a " "split-out ForEach inner job). Databricks task values cannot cross " - "`run_job_task` boundaries, so Flowx blanked the operand. A blanked " + "`run_job_task` boundaries, so flowx blanked the operand. A blanked " "operand makes the predicate `NOT_EQUAL('', '0')` **always true**, so the " "branch now runs unconditionally. Re-wire each condition below — either " "recompute the operand inside this job or pass it as a job parameter." @@ -750,9 +740,7 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: var_name = rollup.get("variable_name", "") parent_key = rollup.get("parent_foreach", "") message = rollup.get("message", "") - lines.append( - f"| `{var_name}` | `{parent_key}` | {message} |" - ) + lines.append(f"| `{var_name}` | `{parent_key}` | {message} |") lines.append("") if prereqs.network_endpoints: diff --git a/src/orchestra/bundler/setup_generator.py b/src/flowx/bundler/setup_generator.py similarity index 88% rename from src/orchestra/bundler/setup_generator.py rename to src/flowx/bundler/setup_generator.py index 3bce6f2..838f5cc 100644 --- a/src/orchestra/bundler/setup_generator.py +++ b/src/flowx/bundler/setup_generator.py @@ -30,12 +30,12 @@ def generate_setup_tasks( notebook = _generate_secrets_setup_notebook(secrets) notebooks.append(notebook) - volume_tasks = [t for t in setup_tasks if t.type == "volume"] + volume_tasks = [task for task in setup_tasks if task.type == "volume"] if volume_tasks: notebook = _generate_volume_setup_notebook(volume_tasks, catalog, schema) notebooks.append(notebook) - connection_tasks = [t for t in setup_tasks if t.type == "connection"] + connection_tasks = [task for task in setup_tasks if task.type == "connection"] if connection_tasks: notebook = _generate_connection_setup_notebook(connection_tasks, catalog) notebooks.append(notebook) @@ -57,7 +57,7 @@ def _generate_secrets_setup_notebook(secrets: list[SecretInstruction]) -> DabNot # MAGIC %md # MAGIC # Setup: Create Secret Scopes and Secrets # MAGIC - # MAGIC *Auto-generated by Flowx. Run this notebook once before deploying the job.* + # MAGIC *Auto-generated by flowx. Run this notebook once before deploying the job.* # MAGIC # MAGIC This notebook creates the Databricks secret scopes and placeholder secrets # MAGIC required by the translated pipelines. After running, update each secret @@ -71,13 +71,11 @@ def _generate_secrets_setup_notebook(secrets: list[SecretInstruction]) -> DabNot separator = "\n# COMMAND ----------\n\n" scopes: dict[str, list[SecretInstruction]] = {} - for s in secrets: - scopes.setdefault(s.scope, []).append(s) + for secret in secrets: + scopes.setdefault(secret.scope, []).append(secret) - # C-46 (LSC5-002): the ``dbutils.secrets`` submodule is read-only - # (get / getBytes / list / listScopes) — ``createScope`` and ``put`` do - # not exist and raise AttributeError on the first cell. Provision via - # the Databricks SDK ``WorkspaceClient`` instead. + # C-46 (LSC5-002): dbutils.secrets is read-only (get/getBytes/list/listScopes) — createScope/put + # don't exist and raise AttributeError, so provision via the Databricks SDK WorkspaceClient instead. init_cell = textwrap.dedent("""\ from databricks.sdk import WorkspaceClient @@ -99,10 +97,7 @@ def _generate_secrets_setup_notebook(secrets: list[SecretInstruction]) -> DabNot for secret in scope_secrets: lines.append(f"# {secret.value_source}") - lines.append( - f'w.secrets.put_secret(scope="{scope_name}", key="{secret.key}", ' - 'string_value="PLACEHOLDER")' - ) + lines.append(f'w.secrets.put_secret(scope="{scope_name}", key="{secret.key}", string_value="PLACEHOLDER")') lines.append(f'print("Created secret: {scope_name}/{secret.key}")') lines.append("") @@ -153,7 +148,7 @@ def _generate_volume_setup_notebook( # MAGIC %md # MAGIC # Setup: Create Unity Catalog Volumes # MAGIC - # MAGIC *Auto-generated by Flowx. Run this notebook once before deploying the job.* + # MAGIC *Auto-generated by flowx. Run this notebook once before deploying the job.* # MAGIC # MAGIC For each external volume below, this notebook attempts to create the # MAGIC underlying Storage Credential and External Location first. Both require @@ -215,11 +210,11 @@ def _generate_volume_setup_notebook( def _credential_name_for(volume_name: str) -> str: - return f"orchestra_{volume_name}_credential" + return f"flowx_{volume_name}_credential" def _external_location_name_for(volume_name: str) -> str: - return f"orchestra_{volume_name}_location" + return f"flowx_{volume_name}_location" def _render_storage_credential_ddl(credential_name: str, location_type: str) -> str: @@ -233,7 +228,7 @@ def _render_storage_credential_ddl(credential_name: str, location_type: str) -> " WITH AZURE_MANAGED_IDENTITY (\n" " 'PLACEHOLDER_ACCESS_CONNECTOR_RESOURCE_ID'\n" " )\n" - f" COMMENT 'Auto-generated by Flowx for volume {credential_name}'\n" + f" COMMENT 'Auto-generated by flowx for volume {credential_name}'\n" '""")' ) if location_type == "AmazonS3Location": @@ -242,7 +237,7 @@ def _render_storage_credential_ddl(credential_name: str, location_type: str) -> 'spark.sql("""\n' f" CREATE STORAGE CREDENTIAL IF NOT EXISTS {credential_name}\n" " WITH IAM_ROLE 'arn:aws:iam::PLACEHOLDER_ACCOUNT_ID:role/PLACEHOLDER_ROLE'\n" - f" COMMENT 'Auto-generated by Flowx for volume {credential_name}'\n" + f" COMMENT 'Auto-generated by flowx for volume {credential_name}'\n" '""")' ) if location_type == "GoogleCloudStorageLocation": @@ -251,7 +246,7 @@ def _render_storage_credential_ddl(credential_name: str, location_type: str) -> 'spark.sql("""\n' f" CREATE STORAGE CREDENTIAL IF NOT EXISTS {credential_name}\n" " WITH GCP_SERVICE_ACCOUNT 'PLACEHOLDER_SERVICE_ACCOUNT_EMAIL'\n" - f" COMMENT 'Auto-generated by Flowx for volume {credential_name}'\n" + f" COMMENT 'Auto-generated by flowx for volume {credential_name}'\n" '""")' ) return ( @@ -267,7 +262,7 @@ def _render_external_location_ddl(name: str, url: str, credential: str) -> str: f" CREATE EXTERNAL LOCATION IF NOT EXISTS {name}\n" f" URL '{url}'\n" f" WITH (STORAGE CREDENTIAL {credential})\n" - f" COMMENT 'Auto-generated by Flowx'\n" + f" COMMENT 'Auto-generated by flowx'\n" '""")' ) @@ -290,7 +285,7 @@ def _generate_connection_setup_notebook( # MAGIC %md # MAGIC # Setup: Create External Connections # MAGIC - # MAGIC *Auto-generated by Flowx. Run this notebook once before deploying the job.* + # MAGIC *Auto-generated by flowx. Run this notebook once before deploying the job.* # MAGIC # MAGIC Update the placeholder connection details below with real values before running. """) @@ -305,11 +300,16 @@ def _generate_connection_setup_notebook( host = config.get("host", "PLACEHOLDER_HOST") port = config.get("port", "3306") + options = [f"host '{host}'", f"port '{port}'"] + if conn_type == "SQLSERVER": + options.append(f"user '{config.get('user', 'PLACEHOLDER_USER')}'") + options.append(f"password '{config.get('password', 'PLACEHOLDER_PASSWORD')}'") + lines: list[str] = [f"# Create connection: {conn_name}"] lines.append('spark.sql("""') lines.append(f" CREATE CONNECTION IF NOT EXISTS {conn_name}") lines.append(f" TYPE {conn_type}") - lines.append(f" OPTIONS (host '{host}', port '{port}')") + lines.append(f" OPTIONS ({', '.join(options)})") lines.append('""")') lines.append(f'print("Created connection: {conn_name}")') body_parts.append("\n".join(lines)) diff --git a/src/flowx/mcp/__init__.py b/src/flowx/mcp/__init__.py new file mode 100644 index 0000000..924b4f4 --- /dev/null +++ b/src/flowx/mcp/__init__.py @@ -0,0 +1,18 @@ +"""MCP packaging for flowx: exposes the migration phases and adapter operations as MCP tools. + +``build_server`` / ``build_http_app`` require the ``mcp`` extra (``pip install -e .[mcp]``) and are +imported lazily so :mod:`flowx.mcp.runner` stays usable without it. +""" + +from typing import Any + +__all__ = ["build_server", "build_http_app"] + + +def __getattr__(name: str) -> Any: + # Lazy re-export: import server (and the `mcp` extra) only when these names are accessed. + if name in ("build_server", "build_http_app"): + from flowx.mcp import server + + return getattr(server, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/flowx/mcp/__main__.py b/src/flowx/mcp/__main__.py new file mode 100644 index 0000000..e97f770 --- /dev/null +++ b/src/flowx/mcp/__main__.py @@ -0,0 +1,6 @@ +"""``python -m flowx.mcp`` entry point.""" + +from flowx.mcp.server import serve + +if __name__ == "__main__": + serve() diff --git a/src/flowx/mcp/runner.py b/src/flowx/mcp/runner.py new file mode 100644 index 0000000..bc6986e --- /dev/null +++ b/src/flowx/mcp/runner.py @@ -0,0 +1,365 @@ +"""Subprocess bridge between MCP tools and the flowx adapter CLI. + +Every MCP tool shells out to ``python -m flowx.adapter`` — the same unified +entry point the agent skills already use — then reads back the JSON/CSV +artifacts each phase writes. This reuses the tested phase contracts instead of +re-implementing their logic, so the MCP surface stays in lockstep with the CLI. +""" + +from __future__ import annotations + +import csv +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +# Phases can take a while on large factories; allow generous default headroom. +DEFAULT_TIMEOUT = int(os.environ.get("FLOWX_MCP_TIMEOUT", "1800")) + +# Cap on inline `adf_definitions` payloads (they pass through the agent context); above this, stage +# to a UC Volume and pass a path reference instead. Configurable via FLOWX_MAX_INLINE_BYTES. +MAX_INLINE_BYTES = int(os.environ.get("FLOWX_MAX_INLINE_BYTES", str(5_000_000))) + + +@dataclass +class AdapterResult: + """Outcome of a single ``flowx.adapter`` invocation.""" + + command: list[str] + returncode: int + stdout: str + stderr: str + + @property + def ok(self) -> bool: + return self.returncode == 0 + + def as_dict(self) -> dict[str, Any]: + """Serialise the raw process outcome for inclusion in a tool result.""" + return { + "command": " ".join(self.command), + "ok": self.ok, + "returncode": self.returncode, + "stdout": self.stdout.strip(), + "stderr": self.stderr.strip(), + } + + +def run_adapter(args: list[Any], *, cwd: str | Path | None = None, timeout: int = DEFAULT_TIMEOUT) -> AdapterResult: + """Invoke ``python -m flowx.adapter`` with the supplied arguments. + + Args: + args: Adapter subcommand and flags (each item is stringified). + cwd: Working directory for the subprocess. Defaults to the current one. + timeout: Seconds before the subprocess is killed. + + Returns: + The captured :class:`AdapterResult`. + """ + command = [sys.executable, "-m", "flowx.adapter", *[str(arg) for arg in args]] + proc = subprocess.run( + command, + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + timeout=timeout, + ) + return AdapterResult(command=command, returncode=proc.returncode, stdout=proc.stdout, stderr=proc.stderr) + + +def read_json(path: Path) -> Any | None: + """Return parsed JSON at *path*, or ``None`` when the file is absent/invalid.""" + try: + return json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + + +def read_text(path: Path) -> str | None: + """Return text at *path*, or ``None`` when it cannot be read.""" + try: + return path.read_text() + except OSError: + return None + + +def parse_stdout_json(result: AdapterResult) -> Any | None: + """Parse the adapter's stdout as JSON (used by inspect/inputs/workspace-paths).""" + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + return None + + +def list_tree(root: Path, *, max_entries: int = 250) -> list[str]: + """Return repo-relative paths of files under *root* (sorted, capped).""" + if not root.exists(): + return [] + files = sorted(str(p.relative_to(root)) for p in root.rglob("*") if p.is_file()) + return files[:max_entries] + + +def summarize_inventory(output_dir: Path) -> dict[str, Any] | None: + """Summarise ``metadata/inventory.json`` produced by the discover phase. + + The discover phase writes a ready-made ``summary`` block (pipeline/activity + counts by strategy plus coverage); surface it directly when present. + """ + inventory = read_json(output_dir / "metadata" / "inventory.json") + if not isinstance(inventory, dict): + return None + + summary = inventory.get("summary") + if isinstance(summary, dict): + return summary + + pipelines = inventory.get("pipelines") or [] + return {"pipeline_count": len(pipelines)} + + +def summarize_translation(output_dir: Path) -> dict[str, Any] | None: + """Summarise the translation report (transient under ``.work/``).""" + for candidate in (".work/translation_report.json", "translation_report.json"): + report = read_json(output_dir / candidate) + if isinstance(report, dict): + break + else: + return None + + pipelines = report.get("pipelines") or [] + statuses: dict[str, int] = {} + for pipeline in pipelines: + for task in pipeline.get("tasks", []): + status = str(task.get("status", "translated")).lower() + statuses[status] = statuses.get(status, 0) + 1 + return {"pipelines": len(pipelines), "task_status_counts": statuses} + + +def materialize_lookup_rows(source: str) -> list[dict[str, str]]: + """Parse a CSV file path or literal CSV string into a list of row dicts. + + Mirrors the adapter's ``materialize-lookup`` parsing so the MCP tool can + validate input without a second subprocess hop. + """ + text = Path(source).read_text() if Path(source).exists() else source + reader = csv.DictReader(io.StringIO(text)) + return [dict(row) for row in reader] + + +def materialize_adf_definitions(definitions: dict[str, Any]) -> str: + """Write an inline ADF-definitions payload to a temp dir and return a source path. + + A hosted MCP server (Databricks App) cannot read the user's workspace / UC Volume files, so + the caller (which can) passes the ADF JSON inline and the server materializes it locally. + + ``definitions`` maps relative file paths — mirroring the ADF Git-export layout, e.g. + ``"pipeline/Foo.json"``, ``"dataset/Bar.json"``, ``"linkedService/Baz.json"``, + ``"trigger/Qux.json"`` — to JSON content (a dict, or a JSON string). The files are written + under a fresh temp directory whose path is returned (the loader reads it as a tree). + + Special case: a single entry whose content is an ARM template (a dict with a top-level + ``resources`` list) is written as one file and that file path is returned, so the loader + parses it in ARM-template mode. + + Raises: + ValueError: if the payload is empty or a key escapes the temp directory. + """ + if not definitions: + raise ValueError("adf_definitions is empty") + + total_bytes = sum(len(v if isinstance(v, str) else json.dumps(v)) for v in definitions.values()) + if total_bytes > MAX_INLINE_BYTES: + raise ValueError( + f"adf_definitions is ~{total_bytes} bytes (limit {MAX_INLINE_BYTES}); inline payloads pass " + "through the agent's context and do not scale. Stage the ADF export to a UC Volume and pass " + "'adf_volume_path' instead (the server reads it directly via the SDK Files API)." + ) + + base = Path(tempfile.mkdtemp(prefix="flowx-adf-")) + + if len(definitions) == 1: + (only_value,) = definitions.values() + content = json.loads(only_value) if isinstance(only_value, str) else only_value + if isinstance(content, dict) and isinstance(content.get("resources"), list): + file_path = base / "arm_template.json" + file_path.write_text(json.dumps(content), encoding="utf-8") + return str(file_path) + + base_resolved = base.resolve() + for rel_path, content in definitions.items(): + dest = (base / rel_path).resolve() + if base_resolved not in dest.parents and dest != base_resolved: + shutil.rmtree(base, ignore_errors=True) + raise ValueError(f"unsafe path in adf_definitions: {rel_path!r}") + dest.parent.mkdir(parents=True, exist_ok=True) + text = content if isinstance(content, str) else json.dumps(content) + dest.write_text(text, encoding="utf-8") + return str(base) + + +def cleanup_materialized(source: str) -> None: + """Remove a temp tree created by :func:`materialize_adf_definitions`. + + Accepts either the returned directory or the single-file path (whose parent temp dir is + removed). Only paths under the system temp dir are deleted, as a safety guard. + """ + path = Path(source) + target = path if path.is_dir() else path.parent + if str(target.resolve()).startswith(str(Path(tempfile.gettempdir()).resolve())): + shutil.rmtree(target, ignore_errors=True) + + +def read_tree(root: Path, *, max_total_bytes: int = 2_000_000) -> dict[str, Any]: + """Return the text contents of files under *root* so a caller can persist them. + + The hosted app writes the generated bundle to ephemeral local disk that the user cannot + reach, so the bundle contents are returned inline. Binary/unreadable files are skipped, and + once the cumulative size passes *max_total_bytes* further files are listed under ``truncated`` + instead of being included. + + Returns: + ``{"files": {relpath: text, ...}, "truncated": [relpath, ...]}`` ("truncated" omitted when empty). + """ + files: dict[str, str] = {} + truncated: list[str] = [] + total = 0 + if root.exists(): + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + rel = str(path.relative_to(root)) + try: + data = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + if total + len(data) > max_total_bytes: + truncated.append(rel) + continue + files[rel] = data + total += len(data) + result: dict[str, Any] = {"files": files} + if truncated: + result["truncated"] = truncated + return result + + +def download_volume_dir(volume_path: str) -> str: + """Download a Unity Catalog Volume directory tree to a local temp dir via the SDK Files API. + + This is the scalable input path for large factories: the bytes are pulled by the server (which + can read the volume as its service principal) and never pass through the calling agent. Returns + the local temp-dir path (clean up with :func:`cleanup_materialized`). + """ + from databricks.sdk import WorkspaceClient + + client = WorkspaceClient() + base = Path(tempfile.mkdtemp(prefix="flowx-vol-")) + root = volume_path.rstrip("/") + + def _recurse(directory: str) -> None: + for entry in client.files.list_directory_contents(directory): + entry_path = entry.path or "" + if entry.is_directory: + _recurse(entry_path) + continue + rel = entry_path[len(root) :].lstrip("/") + dest = base / rel + dest.parent.mkdir(parents=True, exist_ok=True) + contents = client.files.download(entry_path).contents + dest.write_bytes(contents.read() if contents is not None else b"") + + _recurse(root) + return str(base) + + +def download_workspace_dir(workspace_path: str) -> str: + """Download a ``/Workspace`` directory tree to a local temp dir via the SDK Workspace API. + + Workspace files (e.g. an ADF Git folder cloned under ``/Workspace``) use the Workspace API + (``w.workspace.list`` / ``w.workspace.download``), which is distinct from the Files API used for + UC Volumes. Like the volume path, the bytes are pulled by the server and bypass the agent. + Returns the local temp-dir path (clean up with :func:`cleanup_materialized`). + """ + from databricks.sdk import WorkspaceClient + from databricks.sdk.service.workspace import ObjectType + + client = WorkspaceClient() + base = Path(tempfile.mkdtemp(prefix="flowx-ws-")) + root = workspace_path.rstrip("/") + + def _recurse(directory: str) -> None: + for entry in client.workspace.list(directory): + entry_path = entry.path or "" + if entry.object_type in (ObjectType.DIRECTORY, ObjectType.REPO): + _recurse(entry_path) + elif entry.object_type == ObjectType.FILE: + rel = entry_path[len(root) :].lstrip("/") + dest = base / rel + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(client.workspace.download(entry_path).read()) + + _recurse(root) + return str(base) + + +def upload_tree_to_volume(local_root: Path, volume_path: str) -> dict[str, Any]: + """Upload a local directory tree to a Unity Catalog Volume via the SDK Files API. + + This is the scalable output path: the generated bundle is written to a location the user can + reach without the (potentially large) file contents passing back through the agent. + + Returns: + ``{"output_volume_path": , "files": [relpath, ...], "count": n}``. + """ + from databricks.sdk import WorkspaceClient + + client = WorkspaceClient() + root = volume_path.rstrip("/") + uploaded: list[str] = [] + for path in sorted(local_root.rglob("*")): + if not path.is_file(): + continue + rel = str(path.relative_to(local_root)) + client.files.upload(f"{root}/{rel}", io.BytesIO(path.read_bytes()), overwrite=True) + uploaded.append(rel) + return {"output_volume_path": root, "files": uploaded, "count": len(uploaded)} + + +def upload_tree_to_workspace(local_root: Path, workspace_path: str) -> dict[str, Any]: + """Upload a local directory tree to a ``/Workspace`` directory via the SDK Workspace API. + + The output counterpart to :func:`download_workspace_dir`: the generated DAB lands in a workspace + folder the user can reach without the contents passing back through the agent. Files are imported + with ``ImportFormat.RAW`` so each one (``databricks.yml``, ``*.py``, ``*.yml``, ``SETUP.md``, …) is + stored verbatim as a workspace **file** rather than being interpreted as a notebook (which + ``AUTO`` would do to ``.py`` files, corrupting the bundle source tree). + + Returns: + ``{"output_workspace_path": , "files": [relpath, ...], "count": n}``. + """ + from databricks.sdk import WorkspaceClient + from databricks.sdk.service.workspace import ImportFormat + + client = WorkspaceClient() + root = workspace_path.rstrip("/") + uploaded: list[str] = [] + made_dirs: set[str] = set() + for path in sorted(local_root.rglob("*")): + if not path.is_file(): + continue + rel = str(path.relative_to(local_root)) + dest = f"{root}/{rel}" + parent = dest.rsplit("/", 1)[0] + if parent not in made_dirs: + client.workspace.mkdirs(parent) + made_dirs.add(parent) + client.workspace.upload(dest, path.read_bytes(), format=ImportFormat.RAW, overwrite=True) + uploaded.append(rel) + return {"output_workspace_path": root, "files": uploaded, "count": len(uploaded)} diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py new file mode 100644 index 0000000..f21e8c9 --- /dev/null +++ b/src/flowx/mcp/server.py @@ -0,0 +1,554 @@ +"""MCP server exposing flowx as a single dispatcher tool, ``flowx(command, parameters)``. + +Each command is a thin wrapper over ``python -m flowx.adapter`` (see :mod:`flowx.mcp.runner`); +keeping flowx to one tool stays under host tool-count caps such as Genie Code's 20-tool limit. +""" + +from __future__ import annotations + +import os +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from mcp.server.fastmcp import FastMCP +from mcp.server.transport_security import TransportSecuritySettings + +from flowx.mcp import runner + + +def _allowed_origins() -> list[str]: + """Allowed browser/MCP origins from ``FLOWX_ALLOWED_ORIGINS`` (comma-separated, default ``*``).""" + raw = os.environ.get("FLOWX_ALLOWED_ORIGINS", "*") + return [origin.strip() for origin in raw.split(",") if origin.strip()] + + +def _transport_security() -> TransportSecuritySettings: + """Builds transport-security settings with the SDK's DNS-rebinding check disabled. + + Returns: + Settings that skip the Host/Origin allowlist check. + + Notes: + Behind the Databricks Apps OAuth proxy the check misfires (403/421) and adds nothing on top + of the proxy. Browser CORS is handled separately in :func:`build_http_app`. See the "MCP + server design notes" in AGENTS.md. + """ + return TransportSecuritySettings(enable_dns_rebinding_protection=False) + + +_INSTRUCTIONS = """\ +flowx translates Azure Data Factory (ADF) pipelines into Databricks Lakeflow Jobs packaged as +Declarative Automation Bundles (DABs). Everything is driven through the single `flowx` tool: +`flowx(command="", parameters={...})`. + +Typical flow: + flowx("inputs", {"phase": "discover"}) # learn a phase's inputs + flowx("discover", {"adf_source_path": "...", "output_dir": "..."}) + flowx("convert", {"output_dir": "..."}) + flowx("inspect", {"report_path": "/.work/translation_report.json"}) + flowx("apply_answers", {"report_path": "...", "answers": ["id=value"], "output_dir": "..."}) + flowx("package", {"output_dir": "...", "catalog": "main", "schema": "default"}) +Or run it all at once: + flowx("migrate", {"adf_source_path": "...", "output_dir": "...", "catalog": "...", "schema": "..."}) + +All phases share one output_dir. Provide ADF source paths and output_dir as locations the server can +read/write (a local path, or a Unity Catalog Volume path when the host has volume access). +""" + + +def _phase_result(result: runner.AdapterResult, output_dir: Path, **extra: Any) -> dict[str, Any]: + """Assemble a structured tool result from an adapter run plus artifacts.""" + payload: dict[str, Any] = {"ok": result.ok, "process": result.as_dict(), "output_dir": str(output_dir)} + payload.update({k: v for k, v in extra.items() if v is not None}) + return payload + + +def _resolve_source(p: dict[str, Any], path_key: str = "adf_source_path") -> tuple[str | None, Callable[[], None]]: + """Resolve the ADF source for a command into a local path the adapter can read. + + Input modes, in priority order — a hosted app can't read the user's files directly, so it relies + on the first three: + + 1. ``adf_volume_path`` — a UC Volume directory; the server downloads it via the SDK Files API. + 2. ``adf_workspace_path`` — a ``/Workspace`` directory (e.g. an ADF Git folder); the server + downloads it via the SDK Workspace API. + Both (1) and (2) scale to large factories — the bytes bypass the agent. Each returns a temp + dir + cleanup. + 3. ``adf_definitions`` — an inline ARM-JSON payload (small jobs); materialized to a temp dir. + 4. ``path_key`` (``adf_source_path`` / ``source_dir``) — a path the server itself can read + (local hosting or a mounted volume). + """ + if p.get("adf_volume_path"): + src = runner.download_volume_dir(p["adf_volume_path"]) + return src, lambda: runner.cleanup_materialized(src) + if p.get("adf_workspace_path"): + src = runner.download_workspace_dir(p["adf_workspace_path"]) + return src, lambda: runner.cleanup_materialized(src) + definitions = p.get("adf_definitions") + if definitions: + src = runner.materialize_adf_definitions(definitions) + return src, lambda: runner.cleanup_materialized(src) + return p.get(path_key), (lambda: None) + + +def _bundle_output(p: dict[str, Any], out: Path) -> dict[str, Any]: + """Deliver the generated bundle to a location the user can reach. + + The server's ``output_dir`` is local/ephemeral, so the bundle is written to the target via the + SDK (contents bypass the agent and it scales), in priority order: + + 1. ``output_volume_path`` — upload to a UC Volume via the SDK Files API. + 2. ``output_workspace_path`` — upload to a ``/Workspace`` directory via the SDK Workspace API. + 3. neither — return the contents inline as ``bundle`` for the agent to persist (small bundles). + """ + if p.get("output_volume_path"): + return {"bundle_uploaded": runner.upload_tree_to_volume(out, p["output_volume_path"])} + if p.get("output_workspace_path"): + return {"bundle_uploaded": runner.upload_tree_to_workspace(out, p["output_workspace_path"])} + return {"bundle": runner.read_tree(out)} + + +def _noop() -> None: + """Cleanup placeholder used when there is no materialized source to remove.""" + + +def _pending_options(inspect_result: dict[str, Any]) -> list[dict[str, Any]]: + """Extract the per-pipeline configuration options still awaiting an answer. + + Reads the payload :func:`_cmd_inspect` returns (``{"questions": {"pipelines": [...]}}``) + and keeps only pipelines that still have unanswered ``options`` — empty when nothing needs + input, which is the signal for ``migrate`` to package without pausing. + """ + questions = inspect_result.get("questions") or {} + pipelines = questions.get("pipelines") or [] + return [pipeline for pipeline in pipelines if pipeline.get("options")] + + +# Command handlers: map a `parameters` dict to a structured result; required keys via p[...] so a +# missing one raises KeyError, which the dispatcher converts into a clear error. + + +def _cmd_inputs(p: dict[str, Any]) -> dict[str, Any]: + result = runner.run_adapter(["inputs", p["phase"]]) + return {"ok": result.ok, "inputs": runner.parse_stdout_json(result), "process": result.as_dict()} + + +def _cmd_discover(p: dict[str, Any]) -> dict[str, Any]: + output_dir = p.get("output_dir", "./flowx_output") + source, cleanup = _resolve_source(p) + if not source: + return {"ok": False, "error": "Provide 'adf_definitions' (inline ARM JSON) or 'adf_source_path'."} + try: + args = ["discover", "--adf-source-path", source, "--output-dir", output_dir] + if p.get("pipeline"): + args += ["--pipeline", p["pipeline"]] + result = runner.run_adapter(args) + out = Path(output_dir) + return _phase_result(result, out, inventory=runner.summarize_inventory(out)) + finally: + cleanup() + + +def _cmd_convert(p: dict[str, Any]) -> dict[str, Any]: + output_dir = p.get("output_dir", "./flowx_output") + source, cleanup = _resolve_source(p) + try: + args = ["convert", "--output-dir", output_dir] + if source: + args += ["--adf-source-path", source] + if p.get("pipeline"): + args += ["--pipeline", p["pipeline"]] + result = runner.run_adapter(args) + out = Path(output_dir) + return _phase_result(result, out, translation=runner.summarize_translation(out)) + finally: + cleanup() + + +def _cmd_merge_agentic(p: dict[str, Any]) -> dict[str, Any]: + args = ["convert", "--merge-agentic", "--report", p["report_path"], "--agentic-results", p["agentic_results_dir"]] + if p.get("output_path"): + args += ["--output", p["output_path"]] + result = runner.run_adapter(args) + return {"ok": result.ok, "process": result.as_dict()} + + +def _cmd_inspect(p: dict[str, Any]) -> dict[str, Any]: + args: list[Any] = ["inspect", p["report_path"]] + for answer in p.get("answers") or []: + args += ["--answer", answer] + result = runner.run_adapter(args) + return {"ok": result.ok, "questions": runner.parse_stdout_json(result), "process": result.as_dict()} + + +def _cmd_apply_answers(p: dict[str, Any]) -> dict[str, Any]: + args: list[Any] = ["modify", p["report_path"]] + for answer in p["answers"]: + args += ["--answer", answer] + if p.get("output_dir"): + args += ["--output-dir", p["output_dir"]] + if p.get("lookup_csv"): + args += ["--lookup-csv", p["lookup_csv"]] + result = runner.run_adapter(args) + return {"ok": result.ok, "process": result.as_dict()} + + +def _cmd_materialize_lookup(p: dict[str, Any]) -> dict[str, Any]: + result = runner.run_adapter(["materialize-lookup", p["source"], "--out", p["out"]]) + return {"ok": result.ok, "out": p["out"], "process": result.as_dict()} + + +def _cmd_workspace_paths(p: dict[str, Any]) -> dict[str, Any]: + args: list[Any] = ["workspace-paths", p["report_path"]] + source, cleanup = _resolve_source(p, path_key="source_dir") + try: + if source: + args += ["--source-dir", source] + result = runner.run_adapter(args) + return {"ok": result.ok, "result": runner.parse_stdout_json(result), "process": result.as_dict()} + finally: + cleanup() + + +def _cmd_package(p: dict[str, Any]) -> dict[str, Any]: + output_dir = p.get("output_dir", "./flowx_output") + args: list[Any] = [ + "package", + "--output-dir", + output_dir, + "--catalog", + p.get("catalog", "main"), + "--schema", + p.get("schema", "default"), + ] + if p.get("report_path"): + args += ["--report", p["report_path"]] + if p.get("bundle_name"): + args += ["--bundle-name", p["bundle_name"]] + if p.get("profile"): + args += ["--profile", p["profile"]] + if p.get("download_workspace_files") is False: + args += ["--no-download-workspace-files"] + if p.get("keep_intermediates"): + args += ["--keep-intermediates"] + result = runner.run_adapter(args) + out = Path(output_dir) + setup_md = runner.read_text(out / "SETUP.md") or runner.read_text(out / "setup" / "SETUP.md") + extra = _bundle_output(p, out) if result.ok else {} + return _phase_result(result, out, bundle_files=runner.list_tree(out), setup_md=setup_md, **extra) + + +def _cmd_migrate(p: dict[str, Any]) -> dict[str, Any]: + """Run discover→convert→package, pausing for configuration when options are available. + + Because an MCP call can't prompt mid-flight, ``migrate`` is interactive by *handing the questions + back to the agent*: after ``convert`` it returns the **full option schema** once + (``status="needs_input"`` with ``pending_options`` -- every option annotated with a ``show_when`` + condition). The agent drives the whole chain locally (asking only the options whose ``show_when`` + is satisfied, performing any data lookups), then re-calls ``migrate`` a single time with the + complete ``answers`` (``["option_id=value", ...]``), which applies them and packages + (``status="completed"``). No per-follow-up round trip. Pass ``interactive=False`` to skip the + prompt and package with defaults. + """ + output_dir = p.get("output_dir", "./flowx_output") + catalog = p.get("catalog", "main") + schema = p.get("schema", "default") + pipeline = p.get("pipeline") + answers = p.get("answers") or [] + interactive = p.get("interactive", True) + out = Path(output_dir) + report_path = str(out / ".work" / "translation_report.json") + steps: dict[str, Any] = {} + + # Resume: with answers in hand and a prior report present, skip re-running discover/convert. + resume = bool(answers) and (out / ".work" / "translation_report.json").is_file() + + cleanup = _noop + try: + if not resume: + source, cleanup = _resolve_source(p) + if not source: + return { + "ok": False, + "error": ( + "Provide 'adf_volume_path' / 'adf_workspace_path' / 'adf_definitions' / 'adf_source_path'." + ), + } + discover_args = ["discover", "--adf-source-path", source, "--output-dir", output_dir] + if pipeline: + discover_args += ["--pipeline", pipeline] + discover_res = runner.run_adapter(discover_args) + steps["discover"] = _phase_result(discover_res, out, inventory=runner.summarize_inventory(out)) + if not discover_res.ok: + return {"ok": False, "status": "failed", "failed_phase": "discover", "steps": steps} + + convert_args = ["convert", "--output-dir", output_dir, "--adf-source-path", source] + if pipeline: + convert_args += ["--pipeline", pipeline] + convert_res = runner.run_adapter(convert_args) + steps["convert"] = _phase_result(convert_res, out, translation=runner.summarize_translation(out)) + if not convert_res.ok: + return {"ok": False, "status": "failed", "failed_phase": "convert", "steps": steps} + + # Interactive gate: on the first (answerless) call, hand the full option schema to the agent. + if interactive and not answers: + options_schema = _pending_options(_cmd_inspect({"report_path": report_path})) + if options_schema: + return { + "ok": True, + "status": "needs_input", + "pending_options": options_schema, + "report_path": report_path, + "output_dir": output_dir, + "steps": steps, + "message": ( + "Configuration options are available. Each option carries a `show_when` " + "condition (a list of {option_id, in:[values]} clauses; empty = always). Ask " + "only the options whose `show_when` clauses are all satisfied by the answers " + "collected so far, validating each answer against its `choices`. When the user " + "has answered every applicable option, call migrate again with the full " + "`answers` list (['option_id=value', ...]) to apply and package in one shot. To " + "accept defaults and skip prompting, call migrate with interactive=false." + ), + } + + # No (more) pending options: stamp the collected answers (if any) then package. + if answers: + apply_res = _cmd_apply_answers( + { + "report_path": report_path, + "answers": answers, + "output_dir": output_dir, + "lookup_csv": p.get("lookup_csv"), + } + ) + steps["apply_answers"] = apply_res + if not apply_res.get("ok"): + return {"ok": False, "status": "failed", "failed_phase": "apply_answers", "steps": steps} + + package_res = runner.run_adapter( + ["package", "--output-dir", output_dir, "--catalog", catalog, "--schema", schema] + ) + extra = _bundle_output(p, out) if package_res.ok else {} + steps["package"] = _phase_result(package_res, out, bundle_files=runner.list_tree(out), **extra) + return { + "ok": package_res.ok, + "status": "completed" if package_res.ok else "failed", + "failed_phase": None if package_res.ok else "package", + "steps": steps, + } + finally: + cleanup() + + +def _cmd_record_results(p: dict[str, Any]) -> dict[str, Any]: + args: list[Any] = ["record-results", "--output-dir", p["output_dir"], "--results-table", p["results_table"]] + if p.get("warehouse_id"): + args += ["--warehouse-id", p["warehouse_id"]] + result = runner.run_adapter(args) + return {"ok": result.ok, "process": result.as_dict()} + + +def _cmd_install_dashboard(p: dict[str, Any]) -> dict[str, Any]: + args: list[Any] = ["install-dashboard", "--results-table", p["results_table"]] + if p.get("warehouse_id"): + args += ["--warehouse-id", p["warehouse_id"]] + if p.get("dashboard_name"): + args += ["--dashboard-name", p["dashboard_name"]] + if p.get("parent_path"): + args += ["--parent-path", p["parent_path"]] + result = runner.run_adapter(args) + return {"ok": result.ok, "result": runner.parse_stdout_json(result), "process": result.as_dict()} + + +_COMMANDS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = { + "inputs": _cmd_inputs, + "discover": _cmd_discover, + "convert": _cmd_convert, + "merge_agentic": _cmd_merge_agentic, + "inspect": _cmd_inspect, + "apply_answers": _cmd_apply_answers, + "materialize_lookup": _cmd_materialize_lookup, + "workspace_paths": _cmd_workspace_paths, + "package": _cmd_package, + "migrate": _cmd_migrate, + "record_results": _cmd_record_results, + "install_dashboard": _cmd_install_dashboard, +} + + +def build_server() -> FastMCP: + """Construct and return the flowx :class:`FastMCP` server with the single dispatcher tool. + + ``stateless_http=True`` is required by Databricks Genie Code (no persistent ``Mcp-Session-Id`` + round-trip). ``streamable_http_path="/mcp"`` pins the transport to ``/mcp`` (Genie expects the + server at ``/mcp``). ``transport_security`` disables the SDK's DNS-rebinding Origin/Host + check (see :func:`_transport_security`). + """ + mcp = FastMCP( + "flowx", + instructions=_INSTRUCTIONS, + stateless_http=True, + streamable_http_path="/mcp", + transport_security=_transport_security(), + ) + + # structured_output=False: suppress the auto-derived outputSchema (Genie Code rejects tools that + # declare one); the dict is still returned as JSON text. See "MCP server design notes" in AGENTS.md. + @mcp.tool(structured_output=False) + def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, Any]: + """Run an flowx ADF→Databricks migration command. + + Call as ``flowx(command="", parameters={...})``. Commands and their + ``parameters`` keys (req = required; phases share ``output_dir``, default "./flowx_output"): + + - "inputs": phase(req: "discover"|"convert"|"package") — list a phase's input prompts. + - "discover": one of adf_volume_path | adf_workspace_path | adf_definitions | adf_source_path + (req), output_dir, pipeline — parse ADF JSON, classify activities. + - "convert": output_dir, (adf_volume_path | adf_workspace_path | adf_definitions | + adf_source_path), pipeline. + - "merge_agentic": report_path(req), agentic_results_dir(req), output_path — merge agent results. + - "inspect": report_path(req) — return the full translation-option schema (every option with + a `show_when` condition) for the agent to walk locally. See "Collecting options" below. + - "apply_answers": report_path(req), answers(req, list of "ID=VALUE"), output_dir, lookup_csv. + - "materialize_lookup": source(req: CSV path or literal CSV), out(req: destination JSON path). + - "workspace_paths": report_path(req), (adf_volume_path | adf_workspace_path | adf_definitions + | source_dir). + - "package": output_dir, output_volume_path, output_workspace_path, report_path, + catalog(default "main"), schema(default "default"), bundle_name, profile, + download_workspace_files(bool), keep_intermediates(bool). + - "migrate": one of adf_volume_path | adf_workspace_path | adf_definitions | adf_source_path + (req), output_dir, output_volume_path, output_workspace_path, catalog, schema, pipeline, + answers(list of "ID=VALUE"), interactive(bool, default true), lookup_csv — runs + discover→convert→package, returning the full option schema once (status "needs_input") when + configuration is available; re-call once with the complete answers to apply (see below). + - "record_results": output_dir(req), results_table(req: catalog.schema.table), warehouse_id. + - "install_dashboard": results_table(req), warehouse_id, dashboard_name, parent_path. + + Providing the ADF source (a hosted app can't read the user's workspace/volume files directly): + - ``adf_volume_path``: a UC Volume directory the server reads via the SDK Files API. **Preferred + for large factories** — the bytes never pass through the agent. Requires the app's service + principal to have read on the volume. + - ``adf_workspace_path``: a ``/Workspace`` directory (e.g. an ADF Git folder) the server reads + via the SDK Workspace API. Also scales (bytes bypass the agent); needs SP read on that path. + - ``adf_definitions``: an inline mapping of relative path → JSON content mirroring the ADF + Git-export layout, e.g. {"pipeline/Foo.json": {...}, "linkedService/Bar.json": {...}} (a single + ARM-template object is also accepted). Convenient for small jobs; capped (~5 MB) since it flows + through the agent's context — over the cap, switch to ``adf_volume_path``. + - ``adf_source_path`` / ``source_dir``: a path the server itself can read (local hosting / mounted volume). + + Delivering the generated DAB (the server's output_dir is local/ephemeral, so "package"/"migrate" + write it to the target via the SDK — the contents bypass the agent): + - Set ``output_volume_path`` to upload the bundle to a UC Volume (SDK Files API), or + ``output_workspace_path`` to upload it to a ``/Workspace`` directory (SDK Workspace API, files + written verbatim). Either returns ``bundle_uploaded`` = {"output_volume_path" or + "output_workspace_path", "files":[...], "count"}. **Preferred** — required for large bundles. + - With neither set, they return ``bundle`` = {"files": {relpath: text, ...}, "truncated": [...]} + inline for the agent to persist (small bundles only; capped). + + Collecting options (agent-driven chain): + - The server returns the **full option schema** in one shot — it never runs a multi-step + prompt loop itself. "inspect" (and "migrate" on its first, answerless call via + ``status="needs_input"``) returns ``pending_options`` = + ``[{"pipeline_name", "options":[{option_id, prompt, rationale, choices, free_text, default, + show_when}, ...]}, ...]``, where ``show_when`` is a list of ``{option_id, in:[values]}`` + clauses (empty = always shown). + - The **agent** drives the conversation locally: ask an option only when every ``show_when`` + clause is satisfied by the answers gathered so far (e.g. ``notify_slack_url`` shows once + ``notify_destination=slack``); validate each answer against ``choices`` (``free_text`` options + accept any value); perform any data action (e.g. run the lookup query when + ``metadata_driven_lookup_tool=have``). No round trip per follow-up. + - When every applicable option is answered, submit **once**: "migrate" re-called with the full + ``answers`` (applies + packages), or standalone "apply_answers" → "package". The server still + validates every answer at apply time. ``interactive=false`` on "migrate" skips prompting. + + Returns a dict ``{"ok": bool, ...}`` with per-command summaries (inventory / translation / + bundle_files / questions / result) and a "process" block (stdout/stderr/returncode). An unknown + command, missing required parameter, or an oversized inline payload returns + ``{"ok": false, "error": ...}``. + + Args: + command: The operation to run (see the list above). + parameters: Operation-specific keyword arguments. + """ + handler = _COMMANDS.get(command) + if handler is None: + return {"ok": False, "error": f"Unknown command {command!r}. Valid commands: {', '.join(_COMMANDS)}."} + try: + return handler(parameters or {}) + except KeyError as missing: + return {"ok": False, "error": f"Missing required parameter {missing} for command {command!r}."} + except ValueError as error: + return {"ok": False, "error": str(error)} + + return mcp + + +def build_http_app() -> Any: + """Builds the streamable-HTTP ASGI app for hosting (Databricks Apps / Genie Code). + + Returns: + FastMCP's own streamable-HTTP app, serving ``/mcp`` plus ``/`` and ``/health`` routes, with + CORS attached (origins from ``FLOWX_ALLOWED_ORIGINS``, default ``*``). + + Notes: + Returns FastMCP's *own* app rather than mounting it inside another Starlette app: mounting + drops the sub-app's lifespan, leaving the StreamableHTTP session manager uninitialized so + every ``/mcp`` request 500s. See the "MCP server design notes" in AGENTS.md. + """ + from starlette.middleware.cors import CORSMiddleware + from starlette.responses import JSONResponse + + mcp = build_server() + + @mcp.custom_route("/", methods=["GET"]) + async def health(_request: Any) -> JSONResponse: + return JSONResponse({"status": "ok", "service": "mcp-flowx"}) + + @mcp.custom_route("/health", methods=["GET"]) + async def health_alias(_request: Any) -> JSONResponse: + return JSONResponse({"status": "ok", "service": "mcp-flowx"}) + + # FastMCP's own app — its lifespan starts the StreamableHTTP session manager. + app = mcp.streamable_http_app() + + allow_origins = _allowed_origins() + # Credentialed requests cannot use the "*" wildcard per the CORS spec. + allow_credentials = allow_origins != ["*"] + app.add_middleware( + CORSMiddleware, + allow_origins=allow_origins, + allow_credentials=allow_credentials, + allow_methods=["*"], + allow_headers=["*"], + expose_headers=["Mcp-Session-Id"], + ) + return app + + +def serve() -> None: + """Entry point used by ``python -m flowx.mcp``. + + Defaults to stdio (local agents). With ``--http`` (or FLOWX_MCP_HTTP=1) it serves + the streamable-HTTP app via uvicorn on ``--port`` / ``$DATABRICKS_APP_PORT`` / 8000. + """ + import argparse + + parser = argparse.ArgumentParser(prog="python -m flowx.mcp", description="Run the flowx MCP server.") + parser.add_argument("--http", action="store_true", help="Serve over streamable HTTP instead of stdio.") + parser.add_argument("--host", default="0.0.0.0", help="Bind host for --http mode.") + parser.add_argument( + "--port", + type=int, + default=int(os.environ.get("DATABRICKS_APP_PORT", "8000")), + help="Bind port for --http mode (defaults to $DATABRICKS_APP_PORT or 8000).", + ) + args = parser.parse_args() + + if args.http or os.environ.get("FLOWX_MCP_HTTP") == "1": + import uvicorn + + uvicorn.run(build_http_app(), host=args.host, port=args.port) + else: + build_server().run(transport="stdio") diff --git a/src/orchestra/parser/__init__.py b/src/flowx/models/__init__.py similarity index 100% rename from src/orchestra/parser/__init__.py rename to src/flowx/models/__init__.py diff --git a/src/orchestra/models/adf_ast.py b/src/flowx/models/adf_ast.py similarity index 96% rename from src/orchestra/models/adf_ast.py rename to src/flowx/models/adf_ast.py index 9e2d2f9..3fa8d90 100644 --- a/src/orchestra/models/adf_ast.py +++ b/src/flowx/models/adf_ast.py @@ -149,6 +149,8 @@ class AdfActivity: if_true_activities: list[AdfActivity] | None = None if_false_activities: list[AdfActivity] | None = None activities: list[AdfActivity] | None = None # ForEach, Until + # Original ADF/ARM activity JSON, retained so agentic handlers can translate from the source. + raw: dict[str, Any] | None = None # --------------------------------------------------------------------------- @@ -167,6 +169,9 @@ class AdfPipeline: variables: Pipeline variable declarations, keyed by name. annotations: Free-form annotation strings attached to the pipeline. folder: Organisational folder path within the ADF workspace. + raw: Original ADF/ARM pipeline JSON as loaded from source, retained so + the discover phase can emit a verbatim ``.arm.json`` into the + bundle's metadata folder for provenance. """ name: str @@ -175,6 +180,7 @@ class AdfPipeline: variables: dict[str, AdfVariable] | None = None annotations: list[str] | None = None folder: str | None = None + raw: dict[str, Any] | None = None # --------------------------------------------------------------------------- diff --git a/src/orchestra/models/dab.py b/src/flowx/models/dab.py similarity index 100% rename from src/orchestra/models/dab.py rename to src/flowx/models/dab.py diff --git a/src/orchestra/models/ir.py b/src/flowx/models/ir.py similarity index 93% rename from src/orchestra/models/ir.py rename to src/flowx/models/ir.py index 79457ad..1b272ba 100644 --- a/src/orchestra/models/ir.py +++ b/src/flowx/models/ir.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, TypeAlias if TYPE_CHECKING: - from flowx.adapter.models import TranslationPreferences + from flowx.adapter.models import TranslationConfiguration @dataclass(slots=True, kw_only=True) @@ -86,16 +86,14 @@ class Activity: cluster: dict[str, Any] | None = None existing_cluster_id: str | None = None libraries: list[dict[str, Any]] | None = None - # Approximate parameter substitutions made at translation time (e.g. - # ``utcnow()`` mapped to ``{{job.start_time.iso_datetime}}``). Each - # entry has keys ``widget_name``, ``raw_expression``, ``replacement``, - # and ``note``; the bundler surfaces these in SETUP.md. + # Parameter substitutions approximated at translation time (e.g. utcnow()); the bundler lists each + # entry (widget_name/raw_expression/replacement/note) in SETUP.md. parameter_approximations: list[dict[str, str]] = field(default_factory=list) required_parameters: dict[str, str] = field(default_factory=dict) - # Compute mode stamped by the pipeline modifier in response to user - # preferences. One of "serverless", "classic_single_node", - # "classic_multi_node", "inherit", or None when no preferences were applied. + # Compute mode stamped by the modifier: serverless | classic_single_node | classic_multi_node | inherit | None. compute_mode: str | None = None + # Collapsed activity_and_notify spec set by the adapter: {destination, events, args, destination_name}. + notifications: dict[str, Any] | None = None @dataclass(slots=True, kw_only=True) @@ -158,15 +156,11 @@ class CopyActivity(Activity): sink_format: str | None = None sink_resolved_path: str | None = None column_mapping: list[dict[str, str]] | None = None - # Code paradigm chosen by the pipeline modifier: "notebook" (default - # PySpark output) or "sdp" (Lakeflow Spark Declarative Pipeline). + # Code paradigm chosen by the modifier: "notebook" (PySpark) or "sdp" (Lakeflow SDP). target_format: str | None = None - # True when the modifier selected Lakeflow Connect for an eligible - # database-source Copy → Delta ingestion. + # True when the modifier selected Lakeflow Connect for an eligible database-source Copy -> Delta. use_lakeflow_connector: bool = False - # Lakeflow Connect connector flavour resolved by the modifier when - # use_lakeflow_connector is True: "query_based" or "cdc". None when - # the modifier did not stamp a connector type. + # LFC connector flavour when use_lakeflow_connector is True: "query_based" | "cdc" (None if unstamped). lakeflow_connector_type: str | None = None @@ -294,6 +288,11 @@ class WebActivity(Activity): authentication: dict[str, Any] | None = None disable_cert_validation: bool = False http_request_timeout_seconds: int | None = None + # Request body pre-lowered to a Python expression at translate time when it contained + # @-expressions; the code generator emits it verbatim instead of re-resolving. + body_code: str | None = None + body_imports: list[str] = field(default_factory=list) + body_required_parameters: dict[str, str] = field(default_factory=dict) @dataclass(slots=True, kw_only=True) @@ -483,6 +482,9 @@ class PlaceholderActivity(Activity): original_type: str notebook_path: str = "/UNSUPPORTED_ADF_ACTIVITY" comment: str | None = None + # For an agentic gap (e.g. Until): the recommended skill the agent should translate from. + agentic_skill: str | None = None + raw_definition: dict[str, Any] | None = None @dataclass(slots=True, kw_only=True) @@ -511,22 +513,13 @@ class MotifActivity(Activity): confidence_notes: list[str] = field(default_factory=list) original_activities: list[Activity] = field(default_factory=list) notebook_template: str | None = None - # Set by the pipeline modifier when the user opts into metadata-driven - # consolidation, has access to query the lookup source, and the - # configuration size is S or M. When True the preparer should emit - # a single consolidated pipeline whose objects come from lookup_values. + # Set by the modifier when the user opts into metadata-driven consolidation (access granted, size S/M). consolidate_metadata_driven: bool = False - # Concrete lookup rows materialised at translation time (CLI - # ``materialize-lookup`` subcommand or agent-supplied JSON). Each - # element is a dict mirroring a row from the original ADF Lookup - # query. Empty when consolidation is requested but values have not - # been resolved yet. + # Concrete lookup rows materialised at translation time; empty when consolidation was requested but + # the values have not been resolved yet. lookup_values: list[dict[str, Any]] = field(default_factory=list) - # Small dict of motif-specific settings extracted from the collapsed - # activities — e.g. ``{"lookup_query": ..., "lookup_scope": ...}`` for - # ``for_each_ingestion``. Used by the notebook generator so the motif - # can fetch its input list itself instead of requiring an ``items`` - # widget that has no upstream writer. + # Motif-specific settings from the collapsed activities (e.g. lookup_query/lookup_scope for + # for_each_ingestion) so the notebook generator can fetch its own input list. motif_config: dict[str, Any] = field(default_factory=dict) @@ -549,7 +542,7 @@ class Pipeline: tasks: list[Activity] = field(default_factory=list) tags: dict[str, str] = field(default_factory=dict) not_translatable: list[dict[str, Any]] = field(default_factory=list) - translation_preferences: TranslationPreferences | None = None + translation_configuration: TranslationConfiguration | None = None @dataclass(frozen=True, slots=True) @@ -667,9 +660,7 @@ def with_variable_types( variable_cache=self.variable_cache, variable_value_cache=self.variable_value_cache, variable_types=MappingProxyType({**self.variable_types, **types}), - variable_default_literals=MappingProxyType( - {**self.variable_default_literals, **(default_literals or {})} - ), + variable_default_literals=MappingProxyType({**self.variable_default_literals, **(default_literals or {})}), global_parameters=self.global_parameters, linked_service_parameters=self.linked_service_parameters, ) diff --git a/src/orchestra/models/motifs.py b/src/flowx/models/motifs.py similarity index 93% rename from src/orchestra/models/motifs.py rename to src/flowx/models/motifs.py index 1bc85fa..d962947 100644 --- a/src/orchestra/models/motifs.py +++ b/src/flowx/models/motifs.py @@ -169,17 +169,17 @@ class DetectedMotif: notebook_template="staged_load.py", ) -MOTIF_COPY_AND_NOTIFY = MotifDefinition( - motif_id="copy_and_notify", - display_name="Copy and Notify", +MOTIF_ACTIVITY_AND_NOTIFY = MotifDefinition( + motif_id="activity_and_notify", + display_name="Activity and Notify", description=( - "A Copy activity followed by WebActivity calls for success/failure " - "notifications (Logic Apps, Slack, email). Translates to a notebook " - "task with built-in notification via job email/webhook settings." + "Any activity (Copy, Notebook, Lookup, stored procedure, …) followed by WebActivity " + "calls for success/failure notifications (Logic Apps, Slack, email). Collapses to the " + "upstream task with built-in notification via job email/webhook settings." ), - expected_activity_types=("Copy", "WebActivity"), - databricks_replacement="notebook_with_notification", - notebook_template="copy_and_notify.py", + expected_activity_types=("*", "WebActivity"), + databricks_replacement="task_with_notification", + notebook_template=None, ) MOTIF_LAKEFLOW_CONNECT_DATABASE = MotifDefinition( @@ -207,6 +207,6 @@ class DetectedMotif: MOTIF_FILE_EXISTENCE_VALIDATION, MOTIF_SCD_TYPE_2, MOTIF_STAGED_LOAD_SYNAPSE, - MOTIF_COPY_AND_NOTIFY, + MOTIF_ACTIVITY_AND_NOTIFY, MOTIF_LAKEFLOW_CONNECT_DATABASE, ) diff --git a/src/orchestra/models/source_types.py b/src/flowx/models/source_types.py similarity index 56% rename from src/orchestra/models/source_types.py rename to src/flowx/models/source_types.py index 8d815c1..383743c 100644 --- a/src/orchestra/models/source_types.py +++ b/src/flowx/models/source_types.py @@ -2,10 +2,7 @@ from __future__ import annotations -# Database-style sources reachable via JDBC. Every entry here implies the -# generated notebook will read with ``spark.read.format("jdbc")`` and -# require ``jdbc-url`` / ``jdbc-password`` (and optionally ``jdbc-user``) -# secrets. +# Database sources read via spark.read.format("jdbc") (need jdbc-url/jdbc-password/jdbc-user secrets). JDBC_SOURCE_TYPES: frozenset[str] = frozenset( { "AzureSqlSource", @@ -21,9 +18,7 @@ ) -# File-based sources that resolve to an object store location. These -# trigger UC volume / external-location provisioning and use Auto Loader -# (``cloudFiles``) for ingestion. +# Object-store file sources: trigger UC volume / external-location provisioning and Auto Loader ingestion. FILE_SOURCE_TYPES: frozenset[str] = frozenset( { "BlobSource", @@ -43,8 +38,6 @@ ) -# Paginated REST API sources -- handled by a generic ``requests``-based -# pagination loop in the generated copy notebook. ADF ``HttpSource`` -# is *not* in this set: it downloads a single file (CSV / JSON / -# Parquet) over HTTP and is handled as a FILE source via Auto Loader. +# Paginated REST API sources -- a generic requests-based pagination loop in the copy notebook. ADF +# HttpSource is NOT here: it downloads a single file over HTTP and is treated as a FILE source. REST_SOURCE_TYPES: frozenset[str] = frozenset({"RestSource"}) diff --git a/src/orchestra/motifs/__init__.py b/src/flowx/motifs/__init__.py similarity index 100% rename from src/orchestra/motifs/__init__.py rename to src/flowx/motifs/__init__.py diff --git a/src/orchestra/motifs/collapser.py b/src/flowx/motifs/collapser.py similarity index 91% rename from src/orchestra/motifs/collapser.py rename to src/flowx/motifs/collapser.py index e532e01..8dede0a 100644 --- a/src/orchestra/motifs/collapser.py +++ b/src/flowx/motifs/collapser.py @@ -34,11 +34,8 @@ def collapse_motifs( tasks_by_name: dict[str, Activity] = {task.name: task for task in pipeline.tasks} new_tasks: list[Activity] = [] - # Maps a *sanitised* task_key of a collapsed activity to the - # MotifActivity's task_key so ``_rewire_dependencies`` can match - # against ``Dependency.task_key`` (which is also sanitised). Keying - # by raw activity name here would silently fail to rewire any edge - # whose source had spaces or other characters in its name. + # Maps a collapsed activity's sanitised task_key to the MotifActivity's task_key so + # _rewire_dependencies can match Dependency.task_key (also sanitised); raw names would miss edges. motif_task_keys: dict[str, str] = {} inserted_motifs: set[str] = set() @@ -99,10 +96,7 @@ def _build_motif_activity( original_activities = [tasks_by_name[name] for name in motif.matched_activities if name in tasks_by_name] - # Use the sanitised task_keys (not raw activity names) for the - # internal-dependency check; ``Dependency.task_key`` is sanitised by - # the translator, so comparing against raw names would mis-classify - # any internal dep whose source name contained spaces / hyphens. + # Compare sanitised task_keys (Dependency.task_key is sanitised); raw names would mis-classify deps. matched_task_keys = {activity.task_key for activity in original_activities} external_deps = _collect_external_dependencies(original_activities, matched_task_keys) diff --git a/src/orchestra/motifs/detector.py b/src/flowx/motifs/detector.py similarity index 95% rename from src/orchestra/motifs/detector.py rename to src/flowx/motifs/detector.py index 2579501..76180a5 100644 --- a/src/orchestra/motifs/detector.py +++ b/src/flowx/motifs/detector.py @@ -11,8 +11,8 @@ AdfPipeline, ) from flowx.models.motifs import ( + MOTIF_ACTIVITY_AND_NOTIFY, MOTIF_CDC_CHANGE_TRACKING, - MOTIF_COPY_AND_NOTIFY, MOTIF_FILE_EXISTENCE_VALIDATION, MOTIF_FILE_LANDING_ZONE_PROCESSING, MOTIF_INCREMENTAL_LOAD_WATERMARK, @@ -105,7 +105,7 @@ def detect_motifs( (MOTIF_FILE_EXISTENCE_VALIDATION, _detect_file_existence_validation), (MOTIF_SCD_TYPE_2, _detect_scd_type_2), (MOTIF_STAGED_LOAD_SYNAPSE, _detect_staged_load_synapse), - (MOTIF_COPY_AND_NOTIFY, _detect_copy_and_notify), + (MOTIF_ACTIVITY_AND_NOTIFY, _detect_activity_and_notify), ] for motif_def, detector_fn in _detectors: @@ -358,13 +358,8 @@ def _detect_metadata_driven_bulk_copy( for_each_activities = _activities_of_type(activities, "ForEach", claimed) for for_each_activity in for_each_activities: - # Bulk-copy motif requires the inner body to *be* the Copy: a single - # Copy child, with no other transform / orchestration activity in the - # loop body. Patterns like Notebook -> Copy or BuildReport -> Export - # are not bulk-copy motifs even when an upstream Lookup is present; - # they are generic "build then archive" pipelines and the user almost - # never wants the Copy collapsed into a metadata-driven ingestion - # template that ignores the upstream notebook work. + # Bulk-copy requires the ForEach body to *be* a single Copy with no other activity; a + # Notebook->Copy or build-then-archive loop is not this motif even with an upstream Lookup. inner_activities = list(for_each_activity.activities or []) if len(inner_activities) != 1 or inner_activities[0].type != "Copy": continue @@ -476,22 +471,27 @@ def _detect_file_landing_zone( return results -def _detect_copy_and_notify( +def _detect_activity_and_notify( activities: list[AdfActivity], by_name: dict[str, AdfActivity], definitions: AdfDefinitions, claimed: set[str], ) -> list[DetectedMotif]: - """Detects copy-and-notify pattern.""" + """Detects the activity-and-notify pattern: any activity followed by notification Web calls. + + Generalised beyond Copy -- any upstream activity (Copy, Notebook, Lookup, stored procedure, …) + that is directly followed by a WebActivity which looks like a notification (Logic Apps / email / + Slack / Teams / webhook keywords, or a success/failure-conditioned dependency) is reported. + """ results: list[DetectedMotif] = [] - copies = _activities_of_type(activities, "Copy", claimed) + upstream_acts = [a for a in activities if a.type != "WebActivity" and a.name not in claimed] - for copy_act in copies: + for upstream_act in upstream_acts: downstream_webs: list[AdfActivity] = [] for activity in activities: if activity.name in claimed: continue - if activity.type == "WebActivity" and _depends_on(activity, copy_act.name): + if activity.type == "WebActivity" and _depends_on(activity, upstream_act.name): downstream_webs.append(activity) if not downstream_webs: @@ -509,27 +509,28 @@ def _detect_copy_and_notify( if not notification_found: # If there is no notification hint, we still accept if the Web - # activity depends on Copy with success/failure conditions + # activity depends on the upstream with success/failure conditions for web in downstream_webs: if web.depends_on: for dep in web.depends_on: - if dep.activity == copy_act.name and dep.dependency_conditions: + if dep.activity == upstream_act.name and dep.dependency_conditions: conds = [cond.lower() for cond in dep.dependency_conditions] if "failed" in conds or "completed" in conds: notification_found = True notes.append( - f"WebActivity '{web.name}' triggers on {dep.dependency_conditions} of Copy" + f"WebActivity '{web.name}' triggers on " + f"{dep.dependency_conditions} of '{upstream_act.name}'" ) if not notification_found: continue - matched = [copy_act.name] + [web.name for web in downstream_webs] - source_hint = _infer_source_type(copy_act, definitions) + matched = [upstream_act.name] + [web.name for web in downstream_webs] + source_hint = _infer_source_type(upstream_act, definitions) _record_motif( results, - definition=MOTIF_COPY_AND_NOTIFY, + definition=MOTIF_ACTIVITY_AND_NOTIFY, matched_activities=matched, source_type_hint=source_hint, confidence_notes=notes, diff --git a/src/orchestra/translator/__init__.py b/src/flowx/parser/__init__.py similarity index 100% rename from src/orchestra/translator/__init__.py rename to src/flowx/parser/__init__.py diff --git a/src/orchestra/parser/adf_loader.py b/src/flowx/parser/adf_loader.py similarity index 66% rename from src/orchestra/parser/adf_loader.py rename to src/flowx/parser/adf_loader.py index acc3ed9..30c7c6e 100644 --- a/src/orchestra/parser/adf_loader.py +++ b/src/flowx/parser/adf_loader.py @@ -1,10 +1,13 @@ -"""Loads ADF JSON files from a directory structure and produce typed AST objects.""" +"""Loads ADF JSON exports from a directory tree and produces typed AST objects (``AdfDefinitions``).""" from __future__ import annotations import argparse +import csv import json import logging +import re +import shutil from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -67,6 +70,20 @@ "Script": "adf-to-databricks:adf-pipeline-converter", } +# Activity complexity weights (easiest first): Databricks-native ~1:1 tasks, then control-flow, then +# everything else (Copy/Web/Lookup/data movement/agentic) -- summed into each pipeline's complexity score. +_DATABRICKS_NATIVE_TYPES: frozenset[str] = frozenset( + {"DatabricksNotebook", "DatabricksSparkJar", "DatabricksSparkPython", "DatabricksJob"} +) +_CONTROL_FLOW_TYPES: frozenset[str] = frozenset( + {"ForEach", "IfCondition", "Switch", "SetVariable", "AppendVariable", "Filter", "Wait", "Until"} +) +_ACTIVITY_WEIGHT: dict[str, int] = {"databricks": 1, "control": 2, "other": 3} + +# Complexity-score -> T-shirt size cutoffs (inclusive upper bounds). Score is +# sum(activity weights) + #datasets + #linked_services + #collapsible_patterns. +_TSHIRT_CUTOFFS: tuple[tuple[int, str], ...] = ((5, "S"), (15, "M"), (30, "L")) + # --------------------------------------------------------------------------- # Public API @@ -249,7 +266,7 @@ def _find_json_dir(source_dir: Path, *candidate_names: str) -> Path | None: if candidate.is_dir(): return candidate # Case-insensitive fallback - lower_candidates = {n.lower() for n in candidate_names} + lower_candidates = {candidate_name.lower() for candidate_name in candidate_names} for child in source_dir.iterdir(): if child.is_dir() and child.name.lower() in lower_candidates: return child @@ -266,12 +283,13 @@ def _parse_pipeline_json(data: dict[str, Any], *, fallback_name: str = "unknown" Returns: Parsed :class:`AdfPipeline`. """ + raw_source = data data = _normalize_arm(data) props = data.get("properties", data) name = data.get("name") or props.get("name") or fallback_name activities_raw: list[dict[str, Any]] = props.get("activities", []) - activities = [parse_activity(a) for a in activities_raw] + activities = [parse_activity(raw_activity) for raw_activity in activities_raw] parameters: dict[str, AdfParameter] | None = None raw_params = props.get("parameters") @@ -310,6 +328,7 @@ def _parse_pipeline_json(data: dict[str, Any], *, fallback_name: str = "unknown" variables=variables, annotations=annotations, folder=folder, + raw=raw_source, ) @@ -372,13 +391,13 @@ def parse_activity(data: dict[str, Any]) -> AdfActivity: if type_properties: raw_if_true = type_properties.get("ifTrueActivities") if raw_if_true: - if_true_activities = [parse_activity(a) for a in raw_if_true] + if_true_activities = [parse_activity(raw_activity) for raw_activity in raw_if_true] raw_if_false = type_properties.get("ifFalseActivities") if raw_if_false: - if_false_activities = [parse_activity(a) for a in raw_if_false] + if_false_activities = [parse_activity(raw_activity) for raw_activity in raw_if_false] raw_children = type_properties.get("activities") if raw_children: - child_activities = [parse_activity(a) for a in raw_children] + child_activities = [parse_activity(raw_activity) for raw_activity in raw_children] return AdfActivity( name=name, @@ -392,6 +411,7 @@ def parse_activity(data: dict[str, Any]) -> AdfActivity: if_true_activities=if_true_activities, if_false_activities=if_false_activities, activities=child_activities, + raw=data, ) @@ -635,7 +655,7 @@ def _classify_activities( """ for activity in activities: strategy, skill = classify_activity(activity.type) - dep_names = [d.activity for d in activity.depends_on] if activity.depends_on else None + dep_names = [dependency.activity for dependency in activity.depends_on] if activity.depends_on else None items.append( InventoryItem( @@ -702,39 +722,286 @@ def _inventory_to_dict(inventory: Inventory, source_dir: str) -> dict[str, Any]: } +# --------------------------------------------------------------------------- +# Profile complexity report (CSV) +# --------------------------------------------------------------------------- + + +def _walk_activities(activities: list[AdfActivity]): + """Yields every activity in *activities*, descending into container children.""" + for activity in activities: + yield activity + for child in (activity.if_true_activities, activity.if_false_activities, activity.activities): + if child: + yield from _walk_activities(child) + + +def _activity_category(activity_type: str) -> str: + """Returns the complexity category of *activity_type*. + + One of ``"databricks"`` (native, simplest), ``"control"`` (control-flow / + parameter-setting), or ``"other"`` (data movement, web, agentic, ...). + """ + if activity_type in _DATABRICKS_NATIVE_TYPES: + return "databricks" + if activity_type in _CONTROL_FLOW_TYPES: + return "control" + return "other" + + +def _dataset_refs_for_activity(activity: AdfActivity) -> set[str]: + """Collects dataset names referenced by a single activity. + + Looks at the activity's ``inputs``/``outputs`` plus the ``dataset`` / + ``datasets`` references some activity types (Lookup, Delete, GetMetadata) + carry inside ``typeProperties``. + """ + names: set[str] = set() + for ref in (activity.inputs or []) + (activity.outputs or []): + if ref.reference_name: + names.add(ref.reference_name) + props = activity.type_properties or {} + for key in ("dataset", "source", "sink"): + candidate = props.get(key) + if isinstance(candidate, dict) and candidate.get("referenceName"): + names.add(candidate["referenceName"]) + return names + + +def _pipeline_reference_counts( + pipeline: AdfPipeline, definitions: AdfDefinitions +) -> tuple[int, set[str], set[str], dict[str, int]]: + """Returns ``(activity_count, dataset_names, linked_service_names, category_counts)``. + + Linked services are attributed both from activity-level references (e.g. + DatabricksNotebook compute) and transitively via the datasets a pipeline + touches (each dataset names its backing linked service). + """ + dataset_names: set[str] = set() + linked_service_names: set[str] = set() + category_counts = {"databricks": 0, "control": 0, "other": 0} + activity_count = 0 + for activity in _walk_activities(pipeline.activities): + activity_count += 1 + category_counts[_activity_category(activity.type)] += 1 + dataset_names |= _dataset_refs_for_activity(activity) + if activity.linked_service_name and activity.linked_service_name.reference_name: + linked_service_names.add(activity.linked_service_name.reference_name) + for dataset_name in dataset_names: + dataset = definitions.datasets.get(dataset_name) + if dataset and dataset.linked_service_name: + linked_service_names.add(dataset.linked_service_name) + return activity_count, dataset_names, linked_service_names, category_counts + + +def _complexity_score(category_counts: dict[str, int], n_datasets: int, n_linked: int, n_patterns: int) -> int: + """Weighted complexity score: activity weights + datasets + linked services + patterns.""" + weighted = sum(category_counts[cat] * _ACTIVITY_WEIGHT[cat] for cat in category_counts) + return weighted + n_datasets + n_linked + n_patterns + + +def _tshirt_size(score: int) -> str: + """Maps a complexity score to a T-shirt size (S / M / L / XL).""" + for cutoff, size in _TSHIRT_CUTOFFS: + if score <= cutoff: + return size + return "XL" + + +def build_profile_rows(definitions: AdfDefinitions) -> list[dict[str, Any]]: + """Builds one profile-report row per pipeline. + + Each row carries the source activity / dataset / linked-service counts, the + number of collapsible motif patterns detected, and a weighted complexity + score plus its T-shirt size. + + Args: + definitions: Parsed ADF definitions. + + Returns: + List of row dicts ordered by pipeline name. + """ + from flowx.motifs.detector import detect_motifs + + rows: list[dict[str, Any]] = [] + for pipeline in sorted(definitions.pipelines, key=lambda p: p.name): + activity_count, datasets, linked_services, category_counts = _pipeline_reference_counts(pipeline, definitions) + try: + n_patterns = len(detect_motifs(pipeline, definitions)) + except Exception as exc: # noqa: BLE001 - profiling must never hard-fail on motif detection + logger.warning("Motif detection failed for pipeline %r: %s", pipeline.name, exc) + n_patterns = 0 + score = _complexity_score(category_counts, len(datasets), len(linked_services), n_patterns) + rows.append( + { + "pipeline": pipeline.name, + "activities": activity_count, + "datasets": len(datasets), + "linked_services": len(linked_services), + "collapsible_patterns": n_patterns, + "databricks_native_activities": category_counts["databricks"], + "control_flow_activities": category_counts["control"], + "other_activities": category_counts["other"], + "complexity_score": score, + "complexity_size": _tshirt_size(score), + } + ) + return rows + + +_PROFILE_CSV_COLUMNS: tuple[str, ...] = ( + "pipeline", + "activities", + "datasets", + "linked_services", + "collapsible_patterns", + "databricks_native_activities", + "control_flow_activities", + "other_activities", + "complexity_score", + "complexity_size", +) + + +def write_profile_csv(rows: list[dict[str, Any]], path: Path) -> None: + """Writes the per-pipeline profile rows to *path* as CSV.""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=list(_PROFILE_CSV_COLUMNS)) + writer.writeheader() + for row in rows: + writer.writerow(row) + + +def _sanitize_filename(name: str) -> str: + """Slugifies a pipeline name into a safe filename stem.""" + slug = re.sub(r"[^0-9A-Za-z._-]+", "_", name).strip("_") + return slug or "pipeline" + + +def write_pipeline_arm(definitions: AdfDefinitions, metadata_dir: Path) -> list[Path]: + """Writes each pipeline's original ARM JSON to ``metadata_dir/.arm.json``. + + Returns the list of written paths. Pipelines whose source JSON was not + retained are skipped (should not happen for parsed sources). + """ + metadata_dir.mkdir(parents=True, exist_ok=True) + written: list[Path] = [] + for pipeline in definitions.pipelines: + if pipeline.raw is None: + logger.warning("No source ARM JSON retained for pipeline %r; skipping arm export.", pipeline.name) + continue + arm_path = metadata_dir / f"{_sanitize_filename(pipeline.name)}.arm.json" + arm_path.write_text(json.dumps(pipeline.raw, indent=2), encoding="utf-8") + written.append(arm_path) + return written + + # --------------------------------------------------------------------------- # CLI entry point # --------------------------------------------------------------------------- +# Flowx-managed entries under the shared output_dir, cleared at the start of each fresh run. +_MANAGED_OUTPUT_DIRS: tuple[str, ...] = ("metadata", ".work", "resources", "src", "setup") +_MANAGED_OUTPUT_FILES: tuple[str, ...] = ("databricks.yml", "SETUP.md", "WARNINGS.md") -if __name__ == "__main__": + +def clear_stale_outputs(output_dir: Path) -> None: + """Removes a prior run's artifacts from a reused ``output_dir``. + + Discover begins a fresh migration, so a previous run's per-pipeline metadata, transient + intermediates, and generated bundle must not survive into this run. Without this, a + single-pipeline migration into a reused output directory ships the earlier run's other + pipelines' source ARM and generated notebooks in the packaged bundle. + + Args: + output_dir: Migration output directory shared by all three phases. + + Notes: + Only flowx-managed entries are removed (never the directory itself or unrelated + files), so pointing ``output_dir`` at a populated directory stays safe. + """ + for directory in _MANAGED_OUTPUT_DIRS: + shutil.rmtree(output_dir / directory, ignore_errors=True) + for filename in _MANAGED_OUTPUT_FILES: + (output_dir / filename).unlink(missing_ok=True) + + +def main(argv: list[str] | None = None) -> int: + """Discover-phase entry point: load ADF, build the inventory + profile report. + + Exposed as a callable (not just an ``if __name__`` block) so the adapter can run the phase + in-process instead of spawning a second interpreter. Clears any prior run's artifacts from the + shared output directory first, so a reused ``output_dir`` never leaks stale pipelines into the + bundle this run packages. + """ parser = argparse.ArgumentParser(description="Load ADF definitions and build a translation inventory.") parser.add_argument("--source-dir", required=True, type=Path, help="Root directory containing ADF JSON exports.") parser.add_argument( "--output-dir", type=Path, - default=Path("./orchestra_output/ingest"), - help="Directory to write inventory.json into.", + default=Path("./flowx_output"), + help=( + "Migration output directory. Profile artifacts are written into its " + "metadata/ subfolder (inventory.json, profile_report.csv, .arm.json)." + ), + ) + parser.add_argument( + "--pipeline", + type=str, + default=None, + help="Filter to a single pipeline by name. When omitted, all pipelines are included.", ) - args = parser.parse_args() + args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") definitions = load_adf_definitions(args.source_dir) logger.info("Loaded %d pipeline(s) from %s", len(definitions.pipelines), args.source_dir) + # Filter to a single pipeline when --pipeline is specified + if args.pipeline: + matched = [pipeline for pipeline in definitions.pipelines if pipeline.name == args.pipeline] + if not matched: + available = [pipeline.name for pipeline in definitions.pipelines] + logger.error( + "Pipeline %r not found. Available pipelines: %s", + args.pipeline, + ", ".join(available) or "(none)", + ) + return 1 + definitions = AdfDefinitions( + pipelines=matched, + datasets=definitions.datasets, + linked_services=definitions.linked_services, + triggers=definitions.triggers, + global_parameters=definitions.global_parameters, + ) + logger.info("Filtered to pipeline: %s", args.pipeline) + inventory = build_inventory(definitions) output_dir: Path = args.output_dir.resolve() - output_dir.mkdir(parents=True, exist_ok=True) - inventory_path = output_dir / "inventory.json" + clear_stale_outputs(output_dir) + metadata_dir = output_dir / "metadata" + metadata_dir.mkdir(parents=True, exist_ok=True) + + inventory_path = metadata_dir / "inventory.json" inventory_dict = _inventory_to_dict(inventory, str(args.source_dir)) inventory_path.write_text(json.dumps(inventory_dict, indent=2), encoding="utf-8") logger.info("Wrote inventory to %s", inventory_path) + profile_rows = build_profile_rows(definitions) + csv_path = metadata_dir / "profile_report.csv" + write_profile_csv(profile_rows, csv_path) + logger.info("Wrote profile report to %s", csv_path) + + arm_paths = write_pipeline_arm(definitions, metadata_dir) + logger.info("Wrote %d pipeline ARM JSON file(s) to %s", len(arm_paths), metadata_dir) + summary = inventory_dict["summary"] - print("\nADF Ingestion Summary") - print("=====================") + print("\nADF Profile Summary") + print("===================") print(f"Pipelines parsed: {summary['pipeline_count']}") print(f"Total activities: {summary['activity_count']}") print("\nStrategy Breakdown:") @@ -742,3 +1009,16 @@ def _inventory_to_dict(inventory: Inventory, source_dir: str) -> dict[str, Any]: print(f" Agentic: {summary['agentic_count']}") print(f" Unsupported: {summary['unsupported_count']}") print(f"\nCoverage: {summary['coverage_pct']}%") + print("\nComplexity by pipeline (metadata/profile_report.csv):") + print(f" {'pipeline':<32} {'acts':>4} {'ds':>3} {'ls':>3} {'patt':>4} {'score':>5} size") + for row in profile_rows: + print( + f" {row['pipeline'][:32]:<32} {row['activities']:>4} {row['datasets']:>3} " + f"{row['linked_services']:>3} {row['collapsible_patterns']:>4} " + f"{row['complexity_score']:>5} {row['complexity_size']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/orchestra/parser/expression_parser.py b/src/flowx/parser/expression_parser.py similarity index 92% rename from src/orchestra/parser/expression_parser.py rename to src/flowx/parser/expression_parser.py index af09506..2bec906 100644 --- a/src/orchestra/parser/expression_parser.py +++ b/src/flowx/parser/expression_parser.py @@ -10,9 +10,8 @@ _ITEM_RE = re.compile(r"item\(\s*\)$", re.IGNORECASE) -# C-35 (CF4-004): anchor the end-of-string so multi-segment chains like -# ``item().condition.name`` don't match here and silently drop the trailing -# ``.name`` (the previous behaviour mapped to ``{{input.condition}}``). +# C-35 (CF4-004): anchored to end-of-string so multi-segment chains like item().condition.name don't +# match here and silently drop the trailing .name. _ITEM_FIELD_RE = re.compile(r"item\(\s*\)\.(\w+)\s*$", re.IGNORECASE) _ACTIVITY_OUTPUT_RE = re.compile( @@ -89,11 +88,8 @@ re.IGNORECASE | re.DOTALL, ) -# Function names that are no-op wrappers when they appear at the outermost -# position around a single deterministic parameter / variable reference. -# Stripping these lets resolve_expression reach the underlying ref instead -# of falling through to notebook_code for trivial @json(pipeline().parameters.X) -# style wrappers commonly used in ADF for type coercion. +# No-op wrapper function names: stripping them at the outermost position lets resolve_expression reach +# the underlying parameter/variable ref instead of falling through to notebook_code (e.g. @json(...) coercion). _NOOP_WRAPPER_NAMES: frozenset[str] = frozenset({"json", "string", "array"}) _DATETIME_IMPORTS = ["from datetime import datetime, timezone, timedelta"] @@ -134,9 +130,7 @@ def resolve_expression( return None if isinstance(value, bool): - # VAREX3-002: render Python bool as lowercase 'true'/'false' so - # downstream ADF comparisons like @equals(variables('X'), true) - # match ADF's lowercase boolean tokens. + # VAREX3-002: render Python bool as lowercase true/false so @equals(variables('X'), true) matches ADF. return ExpressionResult(kind="literal", value="true" if value else "false") if isinstance(value, (int, float)): return ExpressionResult(kind="literal", value=str(value)) @@ -149,10 +143,8 @@ def resolve_expression( expr = value[1:].rstrip() # strip leading @ and trailing whitespace/newlines - # Strip no-op wrappers like @json(pipeline().parameters.X) so the inner - # ref resolves to its DAB dynamic value. We only unwrap when the inner - # expression itself resolves cleanly (literal / dab_ref) so we don't - # eat the wrapper's semantics where it actually matters. + # Strip no-op wrappers like @json(pipeline().parameters.X) only when the inner ref resolves cleanly + # (literal/dab_ref), so we don't eat the wrapper's semantics where it matters. unwrapped = _unwrap_noop_call(expr, context, variable_task_keys=variable_task_keys) if unwrapped is not None: return unwrapped @@ -205,22 +197,14 @@ def resolve_expression( if result is not None: return result - # CF3-004 / fix-attribute-access-on-function-results: handle - # ``....`` chains like - # ``json(pipeline().parameters.items).type`` by resolving the function - # call first then chaining `.get('attr')` onto the resulting code. - result = _resolve_function_call_with_attribute( - expr, context, variable_task_keys=variable_task_keys - ) + # CF3-004: handle . chains (e.g. json(pipeline().parameters.items).type) + # by resolving the call first, then chaining .get('attr') onto the result. + result = _resolve_function_call_with_attribute(expr, context, variable_task_keys=variable_task_keys) if result is not None: return result - # C-33 (VAREX4-001): handle ``[N]`` chains so e.g. - # ``@split(pipeline().parameters.referenceDate,'/')[0]`` lowers to - # notebook_code. - result = _resolve_function_call_with_index( - expr, context, variable_task_keys=variable_task_keys - ) + # C-33 (VAREX4-001): handle [N] chains, e.g. @split(...,'/')[0], lowering to notebook_code. + result = _resolve_function_call_with_index(expr, context, variable_task_keys=variable_task_keys) if result is not None: return result @@ -417,12 +401,8 @@ def _resolve_item_safe_nav(expr: str) -> ExpressionResult | None: segments: list[tuple[str, str]] = re.findall(r"(\??\.)(\w+)", chain) if not segments: return None - # If the chain has no safe-nav operator at all (purely ``item().a.b``) - # AND only one segment, defer to _ITEM_FIELD_RE's dab_ref path. - # C-35 (CF4-004): multi-segment pure-dotted chains like - # ``item().condition.name`` must lower to notebook_code so the - # downstream consumers can walk both segments instead of mapping to - # ``{{input.condition}}`` and silently dropping ``.name``. + # Single-segment pure-dotted item() chains defer to _ITEM_FIELD_RE's dab_ref path; multi-segment + # ones (C-35: item().condition.name) must lower to notebook_code so both segments survive. has_safe_nav = any(op == "?." for op, _ in segments) if not has_safe_nav and len(segments) < 2: return None @@ -541,9 +521,8 @@ def _resolve_variable( "between job start and the moment the activity actually runs." ) -# ADF .NET-style format strings that map cleanly onto a Databricks dynamic -# value reference. Anything not in this table falls back to a notebook_code -# strftime call. +# ADF .NET-style format strings that map cleanly onto a DAB dynamic value; anything else falls back to +# a notebook_code strftime call. _UTCNOW_FORMAT_TO_DAB_REF: dict[str, str] = { "yyyy-MM-dd": "{{job.start_time.iso_date}}", "yyyy-MM-ddTHH:mm:ss": "{{job.start_time.iso_datetime}}", @@ -652,9 +631,8 @@ def _resolve_concat( if not code_parts: return None - # If every part collapsed to a literal value, fold the whole concat into - # a single literal so downstream consumers (notebook library install, - # cluster fields, etc.) get a plain string instead of Python source. + # If every part collapsed to a literal, fold the concat into one literal so downstream consumers + # get a plain string instead of Python source. if all_literal: return ExpressionResult(kind="literal", value="".join(literal_parts)) @@ -731,23 +709,15 @@ def _split_args(inner: str) -> list[str]: _FUNCTION_CALL_WITH_ATTRIBUTE_RE = re.compile( - # Captures `funcName(args).attr.attr...` -- the trailing attribute chain - # must end with a word character so we don't accidentally swallow other - # closing parens / spaces. Used to lower - # ``json(pipeline().parameters.items).type`` to a notebook_code expression - # since the bare function dispatcher requires the function call to be the - # outermost token. + # Captures funcName(args).attr.attr... (trailing chain ends with a word char) to lower e.g. + # json(pipeline().parameters.items).type to notebook_code, since the bare dispatcher needs the call outermost. r"^([a-zA-Z_]\w*)\((.*)\)((?:\.\w+)+)\s*$", re.IGNORECASE | re.DOTALL, ) _FUNCTION_CALL_WITH_INDEX_RE = re.compile( - # C-33 (VAREX4-001): ``funcName(args)[N]`` — captures a trailing - # integer subscript so ``split(...)[0]`` and similar ADF expressions - # lower to notebook_code (the bare dispatcher only matched when the - # function call was the outermost token). We support a single - # numeric subscript for now; nested chains (``...[0][1]``) fall - # through to the legacy unsupported path. + # C-33 (VAREX4-001): captures funcName(args)[N] so split(...)[0] lowers to notebook_code (single + # numeric subscript only; nested ...[0][1] falls through to the unsupported path). r"^([a-zA-Z_]\w*)\((.*)\)\[\s*(-?\d+)\s*\]\s*$", re.IGNORECASE | re.DOTALL, ) @@ -815,9 +785,7 @@ def _resolve_function_call_with_attribute( inner = match.group(2) attr_chain = match.group(3) func_expr = f"@{func_name}({inner})" - base_result = resolve_expression( - func_expr, context, variable_task_keys=variable_task_keys - ) + base_result = resolve_expression(func_expr, context, variable_task_keys=variable_task_keys) if base_result is None: return None if base_result.kind == "literal": @@ -871,22 +839,14 @@ def _resolve_function_call( continue if (raw_arg.startswith("'") and raw_arg.endswith("'")) or (raw_arg.startswith('"') and raw_arg.endswith('"')): - # C-34 (VAREX4-002): preserve the quotedness so the codegen - # downstream emits ``repr(value)`` rather than a bare token — - # otherwise quoted ``'09'`` / ``'12'`` collapse to a bare - # numeric and either raise a SyntaxError (leading zero) or - # silently compare against the wrong value. - resolved_args.append( - ExpressionResult(kind="literal", value=raw_arg[1:-1], was_string_literal=True) - ) + # C-34 (VAREX4-002): preserve quotedness so codegen emits repr(value); else quoted '09'/'12' + # collapse to a bare numeric and raise (leading zero) or compare against the wrong value. + resolved_args.append(ExpressionResult(kind="literal", value=raw_arg[1:-1], was_string_literal=True)) elif _is_numeric(raw_arg): resolved_args.append(ExpressionResult(kind="literal", value=raw_arg)) elif raw_arg.lower() in ("true", "false"): - # C-34 (VAREX4-003): ADF Booleans (``true`` / ``false``) match - # lowercase strings on the SetVariable consumer side (C-21). - # Mark the literal so ``_arg_to_code`` emits ``'true'`` / - # ``'false'`` strings rather than the bare Python ``True`` / - # ``False`` (whose ``str()`` is title-case and never matches). + # C-34 (VAREX4-003): mark ADF booleans so _arg_to_code emits 'true'/'false' strings (matching + # the SetVariable consumer, C-21) rather than Python True/False whose str() is title-case. resolved_args.append( ExpressionResult( kind="literal", @@ -904,9 +864,8 @@ def _resolve_function_call( resolved_args.append(sub_result) handler_result = handler(resolved_args) - # Auto-propagate required_parameters from args onto notebook_code results - # so preparers can thread DAB refs into base_parameters even for handlers - # that pre-date the required_parameters contract. + # Auto-propagate required_parameters from args onto notebook_code results so preparers can thread + # DAB refs into base_parameters even for handlers predating the required_parameters contract. if handler_result is not None and handler_result.kind == "notebook_code": extra_parameters = _collect_required_parameters(*resolved_args) if extra_parameters: @@ -1030,9 +989,9 @@ def _handle_concat(args: list[ExpressionResult]) -> ExpressionResult | None: """ if not args: return None - if all(a.kind == "literal" for a in args): - return ExpressionResult(kind="literal", value="".join(a.value for a in args)) - parts = [f"str({_arg_to_code(a)})" for a in args] + if all(arg.kind == "literal" for arg in args): + return ExpressionResult(kind="literal", value="".join(arg.value for arg in args)) + parts = [f"str({_arg_to_code(arg)})" for arg in args] return _result_from_args(" + ".join(parts), args) @@ -1166,7 +1125,7 @@ def _handle_intersection(args: list[ExpressionResult]) -> ExpressionResult | Non """intersection(c1, c2, ...) -> list(set(c1) & set(c2) & ...)""" if len(args) < 2: return None - parts = " & ".join(f"set({_arg_to_code(a)})" for a in args) + parts = " & ".join(f"set({_arg_to_code(arg)})" for arg in args) return _result_from_args(f"list({parts})", args) @@ -1209,7 +1168,7 @@ def _handle_union(args: list[ExpressionResult]) -> ExpressionResult | None: """union(c1, c2, ...) -> list(set(c1) | set(c2) | ...)""" if len(args) < 2: return None - parts = " | ".join(f"set({_arg_to_code(a)})" for a in args) + parts = " | ".join(f"set({_arg_to_code(arg)})" for arg in args) return _result_from_args(f"list({parts})", args) @@ -1322,13 +1281,13 @@ def _handle_coalesce(args: list[ExpressionResult]) -> ExpressionResult | None: """coalesce(a, b, ...) -> next((x for x in [a, b, ...] if x is not None), None)""" if not args: return None - items = ", ".join(_arg_to_code(a) for a in args) + items = ", ".join(_arg_to_code(arg) for arg in args) return _result_from_args(f"next((x for x in [{items}] if x is not None), None)", args) def _handle_create_array(args: list[ExpressionResult]) -> ExpressionResult | None: """createArray(a, b, ...) -> [a, b, ...]""" - items = ", ".join(_arg_to_code(a) for a in args) + items = ", ".join(_arg_to_code(arg) for arg in args) return _result_from_args(f"[{items}]", args) @@ -1402,7 +1361,7 @@ def _handle_max(args: list[ExpressionResult]) -> ExpressionResult | None: """max(a, b, ...) -> max(a, b, ...)""" if not args: return None - items = ", ".join(_arg_to_code(a) for a in args) + items = ", ".join(_arg_to_code(arg) for arg in args) return _result_from_args(f"max({items})", args) @@ -1410,7 +1369,7 @@ def _handle_min(args: list[ExpressionResult]) -> ExpressionResult | None: """min(a, b, ...) -> min(a, b, ...)""" if not args: return None - items = ", ".join(_arg_to_code(a) for a in args) + items = ", ".join(_arg_to_code(arg) for arg in args) return _result_from_args(f"min({items})", args) @@ -1791,5 +1750,5 @@ def _handle_ticks(args: list[ExpressionResult]) -> ExpressionResult | None: } _FUNCTION_HANDLERS_CI: dict[str, Callable[[list[ExpressionResult]], ExpressionResult | None]] = { - k.lower(): v for k, v in _FUNCTION_HANDLERS.items() if v is not None + name.lower(): handler for name, handler in _FUNCTION_HANDLERS.items() if handler is not None } diff --git a/src/orchestra/parser/ir_rewriter.py b/src/flowx/parser/ir_rewriter.py similarity index 96% rename from src/orchestra/parser/ir_rewriter.py rename to src/flowx/parser/ir_rewriter.py index b3cc0a5..0fc536f 100644 --- a/src/orchestra/parser/ir_rewriter.py +++ b/src/flowx/parser/ir_rewriter.py @@ -166,18 +166,18 @@ def _rewrite_activity(activity: Activity, context: TranslationContext, warnings: flow activities have their inner branches recursed into. """ field_overrides: dict[str, Any] = {} - for f in dataclasses.fields(activity): - if f.name in _FIELDS_TO_SKIP: + for field_info in dataclasses.fields(activity): + if field_info.name in _FIELDS_TO_SKIP: continue - original = getattr(activity, f.name) + original = getattr(activity, field_info.name) rewritten = _rewrite_value( original, context, warnings, - field_path=f"{type(activity).__name__}.{activity.task_key}.{f.name}", + field_path=f"{type(activity).__name__}.{activity.task_key}.{field_info.name}", ) if rewritten is not original: - field_overrides[f.name] = rewritten + field_overrides[field_info.name] = rewritten if not field_overrides: return activity @@ -206,7 +206,7 @@ def _rewrite_value(value: Any, context: TranslationContext, warnings: list[str], return _rewrite_activity(value, context, warnings) if isinstance(value, SwitchCase): new_value = _rewrite_value(value.value, context, warnings, field_path=f"{field_path}.value") - new_activities = [_rewrite_activity(a, context, warnings) for a in value.activities] + new_activities = [_rewrite_activity(inner_activity, context, warnings) for inner_activity in value.activities] if new_value is value.value and all(n is o for n, o in zip(new_activities, value.activities)): return value return SwitchCase(value=new_value, activities=new_activities) diff --git a/src/orchestra/preparer/__init__.py b/src/flowx/preparer/__init__.py similarity index 100% rename from src/orchestra/preparer/__init__.py rename to src/flowx/preparer/__init__.py diff --git a/src/orchestra/preparer/activity_preparers/__init__.py b/src/flowx/preparer/activity_preparers/__init__.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/__init__.py rename to src/flowx/preparer/activity_preparers/__init__.py diff --git a/src/orchestra/preparer/activity_preparers/append_variable.py b/src/flowx/preparer/activity_preparers/append_variable.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/append_variable.py rename to src/flowx/preparer/activity_preparers/append_variable.py diff --git a/src/orchestra/preparer/activity_preparers/copy.py b/src/flowx/preparer/activity_preparers/copy.py similarity index 97% rename from src/orchestra/preparer/activity_preparers/copy.py rename to src/flowx/preparer/activity_preparers/copy.py index c21cf58..4165f23 100644 --- a/src/orchestra/preparer/activity_preparers/copy.py +++ b/src/flowx/preparer/activity_preparers/copy.py @@ -121,6 +121,9 @@ def prepare(activity: CopyActivity, *, scope: str = "") -> PreparedActivity: secrets = _build_secrets(activity, source_type, scope_name) setup_tasks = _build_setup_tasks(activity, source_type, volume_binding) + # Note: a collapsed activity_and_notify notification spec (activity.notifications) is wired onto + # the task generically in workflow_preparer.prepare_activity, for every task type -- not here. + return PreparedActivity(task=task, notebooks=notebooks, secrets=secrets, setup_tasks=setup_tasks) @@ -224,7 +227,7 @@ def _lakeflow_connection_name_for_activity(activity: CopyActivity) -> str: activity: Source Copy activity carrying source-side metadata. Returns: - A connection name namespaced under ``orchestra_`` and derived + A connection name namespaced under ``flowx_`` and derived from the source linked service name when available, so multiple Copies that share a source linked service emit one connection. Falls back to the activity task key when the IR does not record @@ -232,7 +235,7 @@ def _lakeflow_connection_name_for_activity(activity: CopyActivity) -> str: """ source_properties = activity.source_properties or {} linked_service_name = source_properties.get("linked_service_name") or activity.task_key - return f"orchestra_{_sanitize_identifier(linked_service_name)}_connection" + return f"flowx_{_sanitize_identifier(linked_service_name)}_connection" def _sanitize_identifier(value: str) -> str: @@ -464,9 +467,8 @@ def _build_sink_volume_setup_task(activity: CopyActivity) -> SetupTask | None: "volume_name": volume_name, "volume_type": "EXTERNAL", "location": external_location, - # ``location_type`` drives the storage-credential DDL the setup - # notebook emits (Azure managed identity vs S3 IAM vs GCS service - # account); omitting it leaves the user a manual TODO. + # location_type drives the storage-credential DDL the setup notebook emits (Azure MI / + # S3 IAM / GCS service account); omitting it leaves the user a manual TODO. "location_type": sink_properties.get("volume_location_type", ""), "storage_account": sink_properties.get("volume_storage_account", ""), }, diff --git a/src/orchestra/preparer/activity_preparers/databricks_job.py b/src/flowx/preparer/activity_preparers/databricks_job.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/databricks_job.py rename to src/flowx/preparer/activity_preparers/databricks_job.py diff --git a/src/orchestra/preparer/activity_preparers/delete.py b/src/flowx/preparer/activity_preparers/delete.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/delete.py rename to src/flowx/preparer/activity_preparers/delete.py diff --git a/src/orchestra/preparer/activity_preparers/execute_pipeline.py b/src/flowx/preparer/activity_preparers/execute_pipeline.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/execute_pipeline.py rename to src/flowx/preparer/activity_preparers/execute_pipeline.py diff --git a/src/orchestra/preparer/activity_preparers/filter.py b/src/flowx/preparer/activity_preparers/filter.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/filter.py rename to src/flowx/preparer/activity_preparers/filter.py diff --git a/src/orchestra/preparer/activity_preparers/for_each.py b/src/flowx/preparer/activity_preparers/for_each.py similarity index 93% rename from src/orchestra/preparer/activity_preparers/for_each.py rename to src/flowx/preparer/activity_preparers/for_each.py index 95acb6d..fdbe6a8 100644 --- a/src/orchestra/preparer/activity_preparers/for_each.py +++ b/src/flowx/preparer/activity_preparers/for_each.py @@ -203,10 +203,8 @@ def prepare( all_setup_tasks.extend(inner_prepared.setup_tasks) inner_workflows.extend(inner_prepared.inner_workflows) - # If the single child contributed extra_tasks (e.g. IfCondition or - # Switch branch bodies) we cannot inline as for_each_task.task — - # for_each only accepts a single task. Escalate to the sub-job - # path so the entire branch body survives (CF-001). + # CF-001: a single child with extra_tasks (IfCondition/Switch branch bodies) can't inline as + # for_each_task.task (one task only); escalate to the sub-job path so the whole body survives. if inner_prepared.extra_tasks: inner_job_name = f"{activity.task_key}_inner_tasks" inner_tasks: list[dict[str, Any]] = [ @@ -214,13 +212,10 @@ def prepare( *inner_prepared.extra_tasks, ] normalize_inner_task_params(inner_tasks) - parameters, job_parameters = collect_inner_job_params( - inner_tasks, variable_task_keys=variable_task_keys - ) + parameters, job_parameters = collect_inner_job_params(inner_tasks, variable_task_keys=variable_task_keys) - # LSC3-001: gather cluster hints from inner activities so the - # inner-job default cluster lifts spark_env_vars / custom_tags / - # driver_node_type_id etc. from the LS-derived cluster spec. + # LSC3-001: gather cluster hints from inner activities so the inner-job default cluster lifts + # spark_env_vars / custom_tags / driver_node_type_id from the LS-derived cluster spec. inner_cluster_hints: list[dict[str, Any]] = [] for nested_activity in _iter_activity_with_descendants(inner_activities[0]): if nested_activity.cluster: @@ -276,13 +271,10 @@ def prepare( normalize_inner_task_params(inner_tasks) - parameters, job_parameters = collect_inner_job_params( - inner_tasks, variable_task_keys=variable_task_keys - ) + parameters, job_parameters = collect_inner_job_params(inner_tasks, variable_task_keys=variable_task_keys) - # LSC3-001: gather cluster hints from every nested inner activity - # so the inner-job default cluster picks up LS-derived - # spark_env_vars / custom_tags / driver_node_type_id. + # LSC3-001: gather cluster hints from every nested inner activity so the inner-job default + # cluster picks up LS-derived spark_env_vars / custom_tags / driver_node_type_id. inner_cluster_hints = [] for child in inner_activities: for nested_activity in _iter_activity_with_descendants(child): diff --git a/src/orchestra/preparer/activity_preparers/helpers.py b/src/flowx/preparer/activity_preparers/helpers.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/helpers.py rename to src/flowx/preparer/activity_preparers/helpers.py diff --git a/src/orchestra/preparer/activity_preparers/if_condition.py b/src/flowx/preparer/activity_preparers/if_condition.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/if_condition.py rename to src/flowx/preparer/activity_preparers/if_condition.py diff --git a/src/orchestra/preparer/activity_preparers/lookup.py b/src/flowx/preparer/activity_preparers/lookup.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/lookup.py rename to src/flowx/preparer/activity_preparers/lookup.py diff --git a/src/orchestra/preparer/activity_preparers/motif.py b/src/flowx/preparer/activity_preparers/motif.py similarity index 56% rename from src/orchestra/preparer/activity_preparers/motif.py rename to src/flowx/preparer/activity_preparers/motif.py index cf25589..dcf2963 100644 --- a/src/orchestra/preparer/activity_preparers/motif.py +++ b/src/flowx/preparer/activity_preparers/motif.py @@ -1,17 +1,27 @@ -"""Preparer for MotifActivity -> notebook_task or consolidated pipeline_task.""" +"""Preparer for MotifActivity -> notebook_task, for_each_task, or consolidated pipeline_task.""" from __future__ import annotations +import json from typing import TYPE_CHECKING, Any -from flowx.models.dab import SetupTask +from flowx.models.dab import DabNotebook, SetupTask from flowx.preparer.activity_preparers.helpers import build_notebook_activity_task -from flowx.preparer.code_generator import generate_motif_notebook +from flowx.preparer.code_generator import ( + generate_metadata_driven_control_lookup_notebook, + generate_metadata_driven_item_notebook, + generate_motif_notebook, +) from flowx.preparer.workflow_preparer import PreparedActivity, build_common_task_fields if TYPE_CHECKING: from flowx.models.ir import MotifActivity +# Default destination-table pattern when the collapsed Copy did not name a concrete sink. +_DEFAULT_SINK_TABLE_PATTERN = "raw.{schema_name}_{table_name}" +# Mirrors the ForEach preparer's default fan-out. +_FOR_EACH_CONCURRENCY = 20 + def prepare(activity: MotifActivity, *, scope: str = "") -> PreparedActivity: """Converts a MotifActivity into a DAB task. @@ -22,13 +32,18 @@ def prepare(activity: MotifActivity, *, scope: str = "") -> PreparedActivity: Returns: A :class:`PreparedActivity` whose shape depends on the motif: - metadata-driven motifs marked for consolidation emit a - consolidated Lakeflow Connect pipeline resource and a - ``pipeline_task``; every other motif keeps the legacy scaffold - notebook task. + + * Metadata-driven bulk copy the user **consolidated into a managed pipeline** (Lakeflow + Connect) becomes a single ``pipeline_task``. + * Metadata-driven bulk copy on the **default** path (not consolidated; + ``databricks_replacement == "for_each_ingestion"``) becomes a ``for_each_task`` that runs + one Spark JDBC read per source table -- instead of a single notebook looping internally. + * Every other motif keeps the scaffold notebook task. """ if activity.consolidate_metadata_driven and activity.lookup_values: return _prepare_consolidated_metadata_driven(activity) + if activity.databricks_replacement == "for_each_ingestion": + return _prepare_metadata_driven_for_each(activity) task, notebooks = build_notebook_activity_task( activity, notebook_relative_path=f"notebooks/{activity.task_key}.py", @@ -37,6 +52,69 @@ def prepare(activity: MotifActivity, *, scope: str = "") -> PreparedActivity: return PreparedActivity(task=task, notebooks=notebooks) +def _prepare_metadata_driven_for_each(activity: MotifActivity) -> PreparedActivity: + """Returns a ``for_each_task`` that ingests each source table via its own Spark JDBC read. + + This is the default (non-Lakeflow-Connect) translation of the metadata-driven bulk-copy motif. + Each iteration runs :func:`generate_metadata_driven_item_notebook` for one control-table row + (passed as ``{{input}}``), replacing the former single-notebook Python ``for`` loop with a + native Databricks for-each fan-out. + + The iteration ``inputs`` come from one of two sources: + + * **Static** -- when the control rows were materialised (``activity.lookup_values``), they are + inlined as a literal JSON array. + * **Runtime** -- otherwise a control-table lookup notebook task is emitted (queries the metadata + table and publishes the rows as the ``items`` task value); ``inputs`` references + ``{{tasks..values.items}}`` and the for-each task depends on it. + """ + config = activity.motif_config or {} + task_key = activity.task_key + copy_scope = config.get("copy_scope") or task_key + lookup_scope = config.get("lookup_scope") or task_key + sink_table_pattern = config.get("sink_table") or _DEFAULT_SINK_TABLE_PATTERN + + item_notebook_path = f"notebooks/{task_key}_ingest.py" + notebooks = [ + DabNotebook( + relative_path=item_notebook_path, + content=generate_metadata_driven_item_notebook(scope=copy_scope, sink_table_pattern=sink_table_pattern), + ) + ] + inner_task: dict[str, Any] = { + "task_key": f"{task_key}_ingest", + "notebook_task": { + "notebook_path": f"../src/{item_notebook_path}", + "base_parameters": {"item": "{{input}}"}, + }, + } + + task = build_common_task_fields(activity) + extra_tasks: list[dict[str, Any]] = [] + + if activity.lookup_values: + inputs = json.dumps(activity.lookup_values) + else: + lookup_key = f"{task_key}_control_lookup" + lookup_notebook_path = f"notebooks/{lookup_key}.py" + notebooks.append( + DabNotebook( + relative_path=lookup_notebook_path, + content=generate_metadata_driven_control_lookup_notebook( + scope=lookup_scope, lookup_query=config.get("lookup_query", "") + ), + ) + ) + extra_tasks.append( + {"task_key": lookup_key, "notebook_task": {"notebook_path": f"../src/{lookup_notebook_path}"}} + ) + task["depends_on"] = [*(task.get("depends_on") or []), {"task_key": lookup_key}] + inputs = f"{{{{tasks.{lookup_key}.values.items}}}}" + + task["for_each_task"] = {"inputs": inputs, "task": inner_task, "concurrency": _FOR_EACH_CONCURRENCY} + return PreparedActivity(task=task, notebooks=notebooks, extra_tasks=extra_tasks) + + def _prepare_consolidated_metadata_driven(activity: MotifActivity) -> PreparedActivity: """Returns a PreparedActivity that materialises a consolidated ingestion pipeline. @@ -92,10 +170,10 @@ def _consolidated_connection_name(task_key: str) -> str: task_key: Sanitised task key of the source motif activity. Returns: - A connection name namespaced under ``orchestra_`` so the setup + A connection name namespaced under ``flowx_`` so the setup notebook can recreate it idempotently. """ - return f"orchestra_{task_key}_connection" + return f"flowx_{task_key}_connection" def _build_consolidated_pipeline_definition( diff --git a/src/orchestra/preparer/activity_preparers/naming.py b/src/flowx/preparer/activity_preparers/naming.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/naming.py rename to src/flowx/preparer/activity_preparers/naming.py diff --git a/src/orchestra/preparer/activity_preparers/notebook.py b/src/flowx/preparer/activity_preparers/notebook.py similarity index 99% rename from src/orchestra/preparer/activity_preparers/notebook.py rename to src/flowx/preparer/activity_preparers/notebook.py index 1bc8839..429a2c9 100644 --- a/src/orchestra/preparer/activity_preparers/notebook.py +++ b/src/flowx/preparer/activity_preparers/notebook.py @@ -126,7 +126,7 @@ def _dispatch_stub_notebook(activity: NotebookActivity, filename: str) -> str: f"target_notebook = dbutils.widgets.get('{_DISPATCH_STUB_WIDGET}')\n" "if not target_notebook:\n" " raise ValueError(\n" - f" \"Dispatch stub for activity {activity.name!r} requires a runtime \"\n" + f' "Dispatch stub for activity {activity.name!r} requires a runtime "\n' f" \"value for the '{_DISPATCH_STUB_WIDGET}' widget. See SETUP.md.\"\n" " )\n" "\n" diff --git a/src/orchestra/preparer/activity_preparers/set_variable.py b/src/flowx/preparer/activity_preparers/set_variable.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/set_variable.py rename to src/flowx/preparer/activity_preparers/set_variable.py diff --git a/src/orchestra/preparer/activity_preparers/spark_jar.py b/src/flowx/preparer/activity_preparers/spark_jar.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/spark_jar.py rename to src/flowx/preparer/activity_preparers/spark_jar.py diff --git a/src/orchestra/preparer/activity_preparers/spark_python.py b/src/flowx/preparer/activity_preparers/spark_python.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/spark_python.py rename to src/flowx/preparer/activity_preparers/spark_python.py diff --git a/src/orchestra/preparer/activity_preparers/switch.py b/src/flowx/preparer/activity_preparers/switch.py similarity index 96% rename from src/orchestra/preparer/activity_preparers/switch.py rename to src/flowx/preparer/activity_preparers/switch.py index bfec286..a9eb353 100644 --- a/src/orchestra/preparer/activity_preparers/switch.py +++ b/src/flowx/preparer/activity_preparers/switch.py @@ -128,12 +128,9 @@ def prepare(activity: SwitchActivity, *, scope: str = "") -> PreparedActivity: inner_workflows=list(artifacts.inner_workflows), ) - # Build one condition task per case, chained via outcome="false" deps. - # Every case (including the first) is named ``_case_`` - # for clarity in the rendered job graph. The first case carries the - # original Switch's depends_on edges; ``prepare_workflow`` rewrites any - # downstream task that referenced the bare ```` key to point - # at the renamed first case. + # One condition task per case, chained via outcome="false" deps. Every case is named + # _case_; the first carries the Switch's depends_on edges and prepare_workflow + # rewrites downstream refs from the bare key to the renamed first case. case_keys: list[str] = [] for index, case in enumerate(activity.cases): is_first = index == 0 diff --git a/src/orchestra/preparer/activity_preparers/wait.py b/src/flowx/preparer/activity_preparers/wait.py similarity index 100% rename from src/orchestra/preparer/activity_preparers/wait.py rename to src/flowx/preparer/activity_preparers/wait.py diff --git a/src/orchestra/preparer/activity_preparers/web_activity.py b/src/flowx/preparer/activity_preparers/web_activity.py similarity index 91% rename from src/orchestra/preparer/activity_preparers/web_activity.py rename to src/flowx/preparer/activity_preparers/web_activity.py index 08bd1f2..257d30b 100644 --- a/src/orchestra/preparer/activity_preparers/web_activity.py +++ b/src/flowx/preparer/activity_preparers/web_activity.py @@ -21,11 +21,8 @@ def prepare(activity: WebActivity, *, scope: str = "") -> PreparedActivity: """Converts a WebActivity into a notebook_task with a generated HTTP notebook.""" secrets, setup_tasks = _extract_secrets_and_setup(activity, scope=scope) - # C-38 (LSC4-002): when the preparer resolved an AzureKeyVaultSecret - # payload to a real (scope, key) pair, thread it into the notebook - # generator so the rendered ``dbutils.secrets.get`` references the - # real values rather than the hard-coded ``scope=task_key, - # key='auth-credential'`` fallback. + # C-38 (LSC4-002): thread any resolved AzureKeyVaultSecret (scope, key) into the generator so the + # rendered dbutils.secrets.get uses real values, not the scope=task_key, key='auth-credential' fallback. credential_scope: str | None = None credential_key: str | None = None if secrets: @@ -45,6 +42,9 @@ def prepare(activity: WebActivity, *, scope: str = "") -> PreparedActivity: base_parameters={ "url": resolve_param_value(activity.url), "method": resolve_param_value(activity.method), + # Bind variable/task-value widgets referenced by the resolved body + # (e.g. {{tasks._init_batchId.values.batchId}}). + **dict(activity.body_required_parameters or {}), }, ) @@ -119,9 +119,7 @@ def _extract_secrets_and_setup( return secrets, setup_tasks -def _materialise_secret( - value: Any, *, default_scope: str, role: str -) -> SecretInstruction | None: +def _materialise_secret(value: Any, *, default_scope: str, role: str) -> SecretInstruction | None: """Builds a :class:`SecretInstruction` from an ADF secret payload. Handles the two common shapes: diff --git a/src/orchestra/preparer/code_generator.py b/src/flowx/preparer/code_generator.py similarity index 90% rename from src/orchestra/preparer/code_generator.py rename to src/flowx/preparer/code_generator.py index 2f2e43a..6fa73e8 100644 --- a/src/orchestra/preparer/code_generator.py +++ b/src/flowx/preparer/code_generator.py @@ -203,11 +203,9 @@ def _assemble_file_lookup_source_path(props: dict[str, Any]) -> str: folder = _coerce_to_str(props.get("folder_path")) filename = _coerce_to_str(props.get("file_name")) container = _coerce_to_str(props.get("container")) - # C-47 (LSC5-001): never join a raw ``dataset()`` reference into the - # baked default path. lookup.translate substitutes these from the - # dataset reference's parameter bindings; if one still leaks through - # (e.g. an unbound dataset parameter) drop it so spark.read does not get - # a literal broken ``abfss://.../@dataset().fileName`` path. + # C-47 (LSC5-001): never bake a raw dataset() reference into the default path. lookup.translate + # substitutes these; drop any that still leak through so spark.read doesn't get a broken + # abfss://.../@dataset().fileName path. if "dataset(" in folder: folder = "" if "dataset(" in filename: @@ -223,7 +221,7 @@ def _assemble_file_lookup_source_path(props: dict[str, Any]) -> str: parts.append(folder.strip("/")) if filename: parts.append(filename.strip("/")) - return "/".join(p for p in parts if p) + return "/".join(part for part in parts if part) def _file_lookup_body(activity: LookupActivity) -> str: @@ -316,19 +314,14 @@ def generate_web_activity_notebook( if auth: scope = scope or activity.task_key auth_type = auth.get("type", "") - # C-38 (LSC4-002): prefer the resolved (scope, key) tuple from the - # preparer when supplied. Fall back to the legacy - # ``(task_key, 'auth-credential')`` shape only when the preparer - # didn't (or couldn't) compute one. + # C-38 (LSC4-002): prefer the preparer's resolved (scope, key) tuple; fall back to the legacy + # (task_key, 'auth-credential') shape only when the preparer didn't compute one. resolved_scope = credential_scope or scope resolved_key = credential_key or "auth-credential" if auth_type in ("MSI", "ManagedServiceIdentity"): - # LSC3-002: MSI / Managed Identity auth carries no static secret, - # so reading ``auth-credential`` from a secret scope is a fake - # placeholder that fails at runtime. Surface a NotImplementedError - # so the user can implement the credential exchange manually -- - # the manual_credential SetupTask emitted by web_activity preparer - # already flags this in SETUP.md. + # LSC3-002: MSI/Managed Identity has no static secret, so reading auth-credential would be a + # placeholder that fails at runtime; raise NotImplementedError (web_activity's manual_credential + # SetupTask already flags this in SETUP.md) so the user wires the exchange manually. auth_block = textwrap.dedent(f"""\ # Authentication ({auth_type}) - manual implementation required raise NotImplementedError( @@ -361,19 +354,22 @@ def generate_web_activity_notebook( """) body_block = "" + extra_imports: list[str] = [] request_call = "" if activity.method in ("POST", "PUT", "PATCH"): raw_body = activity.body - # If the body was pre-resolved to Python code by the translator - # (contains function calls like __import__ or json.loads), embed directly. - if isinstance(raw_body, str) and ("__import__" in raw_body or "json.loads" in raw_body): + # Prefer the body the translator pre-resolved to Python code (the real TranslationContext lowered + # @concat / @variables / @{...} that the empty context here could not). + if activity.body_code is not None: + extra_imports = list(activity.body_imports) + body_block = f"body = {activity.body_code}\n" + # Legacy: top-level Expression bodies stored directly on ``body``. + elif isinstance(raw_body, str) and ("__import__" in raw_body or "json.loads" in raw_body): body_block = f"body = {raw_body}\n" else: body_str = _resolve_body(raw_body) - # ``_resolve_body`` may return either a JSON literal, a Python - # dict literal, or a ``repr()``'d string containing Python-like - # concat syntax. Parse strings as JSON when possible so the - # downstream ``requests.request(json=...)`` gets a real object. + # _resolve_body may return a JSON literal, a Python dict literal, or a repr()'d concat string; + # parse strings as JSON when possible so requests.request(json=...) gets a real object. body_block = textwrap.dedent(f"""\ body_raw = dbutils.widgets.get("body") or {body_str} if isinstance(body_raw, str): @@ -409,9 +405,13 @@ def generate_web_activity_notebook( else: method_line = f'method = dbutils.widgets.get("method") or "{activity.method}"' - body = textwrap.dedent(f"""\ + body = textwrap.dedent("""\ import json import requests + """) + for imp in dict.fromkeys(extra_imports): + body += f"{imp}\n" + body += textwrap.dedent(f"""\ # Parameters {url_line} @@ -485,9 +485,8 @@ def generate_set_variable_notebook(activity: SetVariableActivity) -> str: else: import_block = "" - # Build body lines list to avoid textwrap.dedent issues when - # import_block starts at column 0 (which would prevent dedent - # from stripping the common leading whitespace). + # Build the body as a lines list to avoid textwrap.dedent issues when import_block starts at + # column 0 (which would stop dedent stripping the common leading whitespace). lines = ["import json"] if import_block: lines.append(import_block.rstrip("\n")) @@ -581,10 +580,8 @@ def generate_copy_notebook(activity: CopyActivity, *, scope: str = "") -> str: else: body = _generate_generic_copy_body(activity) - # Hoist any imports the body needs into a single cell at the top of - # the notebook. ``_render_sink_write`` and a few other helpers used - # to inline ``from datetime import ...`` next to the call site, which - # produced an awkward block sandwiched between two comment groups. + # Hoist imports the body needs into a single cell at the top of the notebook, since some helpers + # inline ``from datetime import ...`` at the call site (an awkward block between comment groups). imports = _detect_imports(body) if imports: body = _strip_inline_imports(body, imports) @@ -681,7 +678,7 @@ def _safe_identifier(value: str) -> str: The input with non-identifier characters replaced by underscores, falling back to ``ingest`` when the result would be empty. """ - cleaned = "".join(ch if ch.isalnum() or ch == "_" else "_" for ch in value) + cleaned = "".join(char if char.isalnum() or char == "_" else "_" for char in value) cleaned = cleaned.strip("_") if not cleaned or cleaned[0].isdigit(): cleaned = f"ingest_{cleaned}" if cleaned else "ingest" @@ -896,7 +893,7 @@ def _notebook_header(title: str) -> str: # MAGIC %md # MAGIC # {title} # MAGIC - # MAGIC *Auto-generated by Flowx. Do not edit manually unless necessary.* + # MAGIC *Auto-generated by flowx. Do not edit manually unless necessary.* """) @@ -1103,9 +1100,8 @@ def _render_sink_write( fmt = activity.sink_format sink_props = activity.sink_properties or {} - # File-format sink — write the actual format declared by the ADF - # output dataset. Delta files written with ``.save(path)`` skip the - # metastore, which matches the ADF semantic of a path-based dataset. + # File-format sink: write the format declared by the ADF output dataset. .save(path) skips the + # metastore, matching the ADF semantic of a path-based dataset. if fmt and fmt != "delta": opts: list[str] = [] format_settings = sink_props.get("formatSettings") or {} @@ -1119,17 +1115,13 @@ def _render_sink_write( volume_relative = sink_props.get("volume_relative_path") if volume_relative is not None: - # Volume-rooted sink: ``output_path_root`` is set by the bundler - # as a base_parameter (with DAB-substituted ``${var.catalog}`` - # / ``${var.schema}``). Any ``@{...}`` expressions in the - # ADF dataset's folderPath / fileName have already been - # rewritten to Python f-string fragments, so we wrap the - # relative path in an f-string and join. + # Volume-rooted sink: the bundler sets output_path_root as a base_parameter (DAB-substituted + # ${var.catalog}/${var.schema}). The dataset's folderPath/fileName @{...} expressions are + # already rewritten to f-string fragments, so wrap the relative path in an f-string and join. rel_literal = volume_relative.replace('"', '\\"') preamble = "" - # Pull in any modules the rewritten f-string fragments reference - # so the notebook is runnable as-is. Today the only one is - # ``datetime`` (from ``@{formatDateTime(...)}`` rewrites). + # Pull in any modules the rewritten f-string fragments reference so the notebook runs as-is + # (today only datetime, from @{formatDateTime(...)} rewrites). if "datetime." in rel_literal: preamble = f"{indent}from datetime import datetime\n" return ( @@ -1141,9 +1133,8 @@ def _render_sink_write( f'{indent}{df_var}.write.format("{fmt}"){opts_str}.mode("{mode}").save(output_path)\n' ) - # No structured sink volume — fall back to a single ``output_path`` - # widget the user fills in. Common when the linked service uses - # a masked connection string and we can't reconstruct any path. + # No structured sink volume: fall back to a single output_path widget the user fills in. Common + # when the linked service uses a masked connection string and no path can be reconstructed. return ( f"{indent}# The ADF output dataset path could not be resolved at translation time.\n" f"{indent}# Set ``output_path`` on this task to the destination URI.\n" @@ -1215,9 +1206,8 @@ def _generate_autoloader_body(activity: CopyActivity) -> str: source_properties = activity.source_properties or {} sink_properties = activity.sink_properties or {} - # Prefer the UC volume path when available (set by the copy preparer - # when an external volume setup task is created); otherwise fall back - # to the resolved abfss:// path or raw dataset path. + # Prefer the UC volume path when available (set by the copy preparer for external-volume setup); + # otherwise fall back to the resolved abfss:// path or raw dataset path. source_path = source_properties.get( "volume_path", source_properties.get( @@ -1228,9 +1218,8 @@ def _generate_autoloader_body(activity: CopyActivity) -> str: sink_table = sink_properties.get("table", sink_properties.get("tableName", f"{activity.task_key}_raw")) file_format = _infer_file_format(activity.source_type, source_properties) - # Use the volume for checkpoints and schema evolution storage instead of - # /tmp. This ensures state persists across cluster restarts and is - # visible in Unity Catalog. + # Use the volume (not /tmp) for checkpoints and schema-evolution storage so state persists across + # cluster restarts and is visible in Unity Catalog. volume_base = source_properties.get("volume_base", "") if volume_base: checkpoint = f"{volume_base}/_checkpoints/{activity.task_key}" @@ -1313,10 +1302,8 @@ def _generate_jdbc_body(activity: CopyActivity, *, scope: str = "") -> str: is_expression = True if is_expression: - # The query is an ADF expression (e.g. @concat('SELECT * FROM ', item().schema_name, ...)). - # Generate a notebook that reads the current ForEach item from the - # "item" widget (set to {{input}} by the for_each_task) and builds - # the SQL query dynamically. + # The query is an ADF expression (e.g. @concat('SELECT * FROM ', item().schema_name, ...)); generate + # a notebook that reads the current ForEach item from the "item" widget and builds the SQL dynamically. return ( textwrap.dedent(f"""\ import json @@ -1478,6 +1465,78 @@ def _generate_generic_copy_body(activity: CopyActivity) -> str: ) +def generate_metadata_driven_item_notebook(*, scope: str, sink_table_pattern: str) -> str: + """Generates the per-iteration notebook for a metadata-driven for_each_task. + + Each ``for_each_task`` iteration runs this notebook with the current control-table row passed as + the ``item`` widget (set to ``{{input}}``). It reads that one source table over Spark JDBC and + writes it to Delta -- the per-row body of the former in-notebook loop, now one task per table. + + Args: + scope: Secret scope holding ``jdbc-url`` / ``jdbc-user`` / ``jdbc-password`` for the source. + sink_table_pattern: ``str.format`` pattern for the destination table, e.g. + ``"raw.{schema_name}_{table_name}"``. + """ + return "# Databricks notebook source\n" + textwrap.dedent(f"""\ + import json + + # The for_each_task passes the current control-table row as the "item" widget via {{{{input}}}}. + item_raw = dbutils.widgets.get("item") + item = json.loads(item_raw) if item_raw else {{}} + schema_name = item.get("schema_name", "dbo") + table_name = item.get("table_name") or item.get("name") or "UNKNOWN_TABLE" + target = {sink_table_pattern!r}.format(schema_name=schema_name, table_name=table_name) + query = f"SELECT * FROM {{schema_name}}.{{table_name}}" + + jdbc_url = dbutils.secrets.get(scope="{scope}", key="jdbc-url") + jdbc_user = dbutils.secrets.get(scope="{scope}", key="jdbc-user") + jdbc_password = dbutils.secrets.get(scope="{scope}", key="jdbc-password") + + ( + spark.read.format("jdbc") + .option("url", jdbc_url) + .option("user", jdbc_user) + .option("password", jdbc_password) + .option("query", query) + .load() + .write.format("delta") + .mode("overwrite") + .option("overwriteSchema", "true") + .saveAsTable(target) + ) + """) + + +def generate_metadata_driven_control_lookup_notebook(*, scope: str, lookup_query: str) -> str: + """Generates the control-table lookup notebook that seeds a metadata-driven for_each_task. + + Used when the control rows were not materialised at translation time: it queries the metadata + table over Spark JDBC and publishes the rows as the ``items`` task value, which the downstream + ``for_each_task`` consumes via ``{{tasks..values.items}}``. + + Args: + scope: Secret scope holding the control DB's ``jdbc-*`` credentials. + lookup_query: SQL that returns one row per source table to ingest. + """ + return "# Databricks notebook source\n" + textwrap.dedent(f"""\ + jdbc_url = dbutils.secrets.get(scope="{scope}", key="jdbc-url") + jdbc_user = dbutils.secrets.get(scope="{scope}", key="jdbc-user") + jdbc_password = dbutils.secrets.get(scope="{scope}", key="jdbc-password") + + control_query = {lookup_query!r} + control_df = ( + spark.read.format("jdbc") + .option("url", jdbc_url) + .option("user", jdbc_user) + .option("password", jdbc_password) + .option("query", control_query) + .load() + ) + items = [row.asDict() for row in control_df.collect()] + dbutils.jobs.taskValues.set(key="items", value=items) + """) + + def generate_motif_notebook(activity: MotifActivity) -> str: """Generates a notebook scaffold for a collapsed motif activity.""" return _build_motif_notebook( @@ -1530,7 +1589,7 @@ def _build_motif_notebook( "# MAGIC", notes_list, "# MAGIC", - "# MAGIC *Auto-generated by Flowx motif collapser.*", + "# MAGIC *Auto-generated by flowx motif collapser.*", "", "# COMMAND ----------", "", diff --git a/src/flowx/preparer/notifications.py b/src/flowx/preparer/notifications.py new file mode 100644 index 0000000..18e8401 --- /dev/null +++ b/src/flowx/preparer/notifications.py @@ -0,0 +1,158 @@ +"""Resolves collapsed ``activity_and_notify`` specs into DAB task notifications. + +Email specs become ``email_notifications`` from raw addresses; Slack/Teams/PagerDuty/Generic Webhook +specs reuse the destination id provisioned at modify time (``provision_destination``), creating one +via the SDK only as a fallback when the spec carries no pre-resolved id. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from flowx.models.dab import SetupTask + +logger = logging.getLogger(__name__) + +_WEBHOOK_DESTINATIONS = frozenset({"slack", "teams", "pagerduty", "webhook"}) + +# Optional SDK config kwargs per destination (beyond the primary field); only passed when the +# user supplied a value, so the SDK defaults anything left blank. +_DESTINATION_CONFIG_FIELDS: dict[str, tuple[str, ...]] = { + "slack": ("url", "channel_id", "oauth_token"), + "teams": ("url",), + "webhook": ("url", "username", "password"), + "pagerduty": ("integration_key",), +} + + +def resolve_task_notifications(spec: dict[str, Any]) -> tuple[dict[str, Any], list[SetupTask]]: + """Return ``(task_notification_keys, setup_tasks)`` for a notification spec. + + ``task_notification_keys`` is merged into the DAB task dict (it carries an + ``email_notifications`` or ``webhook_notifications`` entry). ``setup_tasks`` + is non-empty only when a webhook-style destination could not be created + (e.g. no workspace auth at prepare time), in which case a documentation + SetupTask is emitted instead and the task ships without notifications. + """ + destination = spec.get("destination", "") + events: list[str] = spec.get("events") or ["on_failure"] + args: dict[str, Any] = spec.get("args") or {} + + if destination == "email": + recipients = [recipient for recipient in (args.get("addresses") or []) if recipient] + if not recipients: + logger.warning("activity_and_notify email destination has no recipients; skipping notification wiring.") + return {}, [] + return {"email_notifications": {event: list(recipients) for event in events}}, [] + + if destination not in _WEBHOOK_DESTINATIONS: + return {}, [] + + display_name = spec.get("destination_name") or f"flowx-{destination}" + # Prefer a destination id provisioned at modify time; create one here only when the report + # lacks a pre-resolved id (e.g. no workspace auth was available then). + destination_id = spec.get("destination_id") or _ensure_destination(destination, display_name, args) + if destination_id is None: + setup = SetupTask( + type="notification_destination", + config={ + "destination": destination, + "display_name": display_name, + **{key: value for key, value in args.items() if value}, + "note": ( + "Could not create this notification destination during prepare. Create it " + "(Settings > Notifications, or w.notification_destinations.create), then add " + '{"id": ""} to the task\'s webhook_notifications.' + ), + }, + ) + return {}, [setup] + + return {"webhook_notifications": {event: [{"id": destination_id}] for event in events}}, [] + + +def provision_destination(spec: dict[str, Any]) -> tuple[dict[str, Any], str]: + """Create (or reuse) the SDK notification destination for *spec* at prompt time. + + Called from the adapter ``modify`` phase right after the user answers the + notification follow-ups. For non-email destinations it creates (or reuses by + display name) the Databricks notification destination via the SDK and returns a + copy of *spec* augmented with the resolved ``destination_id`` so prepare can wire + it without another SDK call. + + Email specs (and anything that is not a webhook-style destination) pass through + unchanged -- email uses raw ``email_notifications`` and needs no destination. + + On failure the spec is returned unchanged (its ``args`` retained), so prepare can + retry the create or fall back to a ``notification_destination`` setup task. + + Returns: + ``(spec, status_message)`` where ``status_message`` is a human-readable line + describing the outcome (empty for email / no-op). + """ + destination = spec.get("destination", "") + if destination == "email" or destination not in _WEBHOOK_DESTINATIONS: + return spec, "" + display_name = spec.get("destination_name") or f"flowx-{destination}" + if spec.get("destination_id"): + return ( + spec, + f"Notification destination '{display_name}' ({destination}) already resolved -> {spec['destination_id']}.", + ) + destination_id = _ensure_destination(destination, display_name, spec.get("args") or {}) + if destination_id is None: + return spec, ( + f"WARNING: could not create notification destination '{display_name}' ({destination}) now; " + "prepare will retry or emit a setup task." + ) + return {**spec, "destination_id": destination_id}, ( + f"Created/reused notification destination '{display_name}' ({destination}) -> {destination_id}." + ) + + +def _build_destination_config(sdk_settings: Any, destination: str, args: dict[str, Any]) -> Any | None: + """Build the SDK ``settings.Config`` for *destination* from the resolved *args*. + + Only kwargs the user supplied are passed; blank optional fields are omitted so + the SDK applies its own defaults. + """ + fields = _DESTINATION_CONFIG_FIELDS.get(destination) + if fields is None: + return None + kwargs = {field: args[field] for field in fields if args.get(field)} + if destination == "slack": + return sdk_settings.Config(slack=sdk_settings.SlackConfig(**kwargs)) + if destination == "teams": + return sdk_settings.Config(microsoft_teams=sdk_settings.MicrosoftTeamsConfig(**kwargs)) + if destination == "webhook": + return sdk_settings.Config(generic_webhook=sdk_settings.GenericWebhookConfig(**kwargs)) + if destination == "pagerduty": + return sdk_settings.Config(pagerduty=sdk_settings.PagerdutyConfig(**kwargs)) + return None + + +def _ensure_destination(destination: str, display_name: str, args: dict[str, Any]) -> str | None: + """Create (or reuse) a notification destination via the SDK; return its id or None.""" + try: + from databricks.sdk.service import settings as sdk_settings + + from flowx.preparer.workspace_downloader import _get_workspace_client + + client = _get_workspace_client() + # Reuse an existing destination with the same display name so prepare is idempotent. + for existing in client.notification_destinations.list(): + if getattr(existing, "display_name", None) == display_name and getattr(existing, "id", None): + logger.info("Reusing existing notification destination '%s' (%s).", display_name, existing.id) + return existing.id + + config = _build_destination_config(sdk_settings, destination, args) + if config is None: + return None + + created = client.notification_destinations.create(display_name=display_name, config=config) + logger.info("Created notification destination '%s' (%s).", display_name, getattr(created, "id", None)) + return getattr(created, "id", None) + except Exception as exc: # noqa: BLE001 - degrade gracefully to a setup task + logger.warning("Notification destination create failed for '%s' (%s): %s", display_name, destination, exc) + return None diff --git a/src/orchestra/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py similarity index 86% rename from src/orchestra/preparer/workflow_preparer.py rename to src/flowx/preparer/workflow_preparer.py index 6c99dea..8b460b3 100644 --- a/src/orchestra/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -42,14 +42,9 @@ class PreparedActivity: secrets: list[SecretInstruction] = field(default_factory=list) setup_tasks: list[SetupTask] = field(default_factory=list) inner_workflows: list[PreparedWorkflow] = field(default_factory=list) - # Switch renames its first case from ```` to - # ``_case_``; ``prepare_workflow`` reads this map to - # rewrite ``depends_on`` edges that referenced the original key. + # Switch renames its first case key; prepare_workflow reads this map to rewrite depends_on edges. task_key_remap: dict[str, str] = field(default_factory=dict) - # Lakeflow pipeline resources (e.g. Lakeflow Connect managed - # ingestion pipelines) the bundle writer emits under - # ``resources/pipelines/.yml``. Each entry is a dict - # with ``resource_key`` and ``definition`` keys. + # Lakeflow pipeline resources emitted under resources/.yml ({resource_key, definition}). pipeline_resources: list[dict[str, Any]] = field(default_factory=list) parameter_approximations: list[ParameterApproximation] = field(default_factory=list) @@ -178,15 +173,22 @@ def prepare_activity( elif type(activity) is AppendVariableActivity: prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) elif type(activity) is ForEachActivity: - # C-06 (VAREX-004): inner-job parameter collector needs the parent's - # variable -> setter mapping so @variables('X') references inside the - # ForEach body route through {{tasks.X.values.Y}} rather than an - # undeclared {{job.parameters.X}}. + # C-06 (VAREX-004): pass the parent's variable->setter map so @variables('X') inside the ForEach + # body routes through {{tasks.X.values.Y}} instead of an undeclared {{job.parameters.X}}. prepared = preparer_fn(activity, scope=scope, variable_task_keys=variable_task_keys) else: prepared = preparer_fn(activity, scope=scope) prepared.task = _stamp_compute_mode(prepared.task, activity.compute_mode) + + # Wire any notification spec the adapter stamped onto this task (generic across task types, not just Copy). + if activity.notifications: + from flowx.preparer.notifications import resolve_task_notifications + + notification_keys, notification_setup = resolve_task_notifications(activity.notifications) + prepared.task = {**prepared.task, **notification_keys} + prepared.setup_tasks = prepared.setup_tasks + notification_setup + return prepared @@ -212,9 +214,13 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: """Returns a PreparedActivity with a stub notebook for an unsupported activity.""" task = build_common_task_fields(activity) + agentic_skill: str | None = None + raw_definition: dict[str, Any] | None = None if isinstance(activity, PlaceholderActivity): comment = activity.comment or "This activity requires manual implementation." original_type = activity.original_type + agentic_skill = activity.agentic_skill + raw_definition = activity.raw_definition elif isinstance(activity, UnsupportedActivity): comment = activity.reason or "This activity type is not supported." original_type = activity.original_type @@ -225,6 +231,22 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: notebook_name = f"{activity.task_key}.py" notebook_path = f"notebooks/{notebook_name}" + # When the activity is an agentic gap (e.g. Until), embed its full ADF/ARM + # JSON so the agentic handler can translate it directly from source. + arm_block = "" + if raw_definition is not None: + import json as _json + + skill_hint = f" using `{agentic_skill}`" if agentic_skill else "" + arm_lines = _json.dumps(raw_definition, indent=2).splitlines() + arm_block = ( + "# MAGIC\n" + f"# MAGIC An agent should translate this activity{skill_hint} from the ADF/ARM JSON below,\n" + "# MAGIC then replace the `raise NotImplementedError` cell with the generated code.\n" + "# MAGIC\n" + "# MAGIC ```json\n" + "".join(f"# MAGIC {line}\n" for line in arm_lines) + "# MAGIC ```\n" + ) + content = ( "# Databricks notebook source\n" "# MAGIC %md\n" @@ -232,9 +254,8 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: "# MAGIC\n" f"# MAGIC Original ADF activity type: **{original_type}**\n" "# MAGIC\n" - f"# MAGIC {comment}\n" - "\n# COMMAND ----------\n\n" - f"raise NotImplementedError(\"Activity '{activity.name}' ({original_type}) requires manual implementation.\")\n" + f"# MAGIC {comment}\n" + arm_block + "\n# COMMAND ----------\n\n" + f"raise NotImplementedError(\"Activity '{activity.name}' ({original_type}) needs agentic translation.\")\n" ) task["notebook_task"] = { @@ -294,11 +315,8 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: all_tasks.extend(prepared.extra_tasks) artifacts = merge_prepared_artifacts(artifacts, prepared) task_key_remap.update(prepared.task_key_remap) - # C-04 (NB-ITER2-4 / LSC2-001): walk into IfCondition / Switch / - # ForEach branches so the workflow's cluster_hints aggregation - # picks up cluster config on activities nested inside compound - # activities. Without this the default Standard_DS3_v2 / 15.4.x - # fallback ships even when the inner notebook has an explicit LS. + # C-04 (NB-ITER2-4 / LSC2-001): walk into IfCondition/Switch/ForEach branches so cluster_hints + # picks up nested cluster config; else the default cluster ships even when an inner notebook has an LS. for nested_activity in _iter_activity_with_descendants(activity): if nested_activity.cluster: cluster_hints.append(dict(nested_activity.cluster)) @@ -321,19 +339,13 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: seen_secrets.add(secret_id) unique_secrets.append(secret) - # VAREX3-003: emit a manual_variable_rollup SetupTask whenever a sibling - # IfCondition / Switch / SetVariable reads a variable that is only - # mutated inside a ForEach inner job. ADF semantics treat the post- - # ForEach read as "latest committed value" but that value is unreachable - # across the run_job_task boundary in DAB. Surfacing the warning lets - # the user add a roll-up notebook before the dependent activity runs. + # VAREX3-003: flag a sibling read of a variable mutated only inside a ForEach inner job -- the post-ForEach + # "latest value" is unreachable across the run_job_task boundary in DAB, so emit a manual_variable_rollup. cross_scope_rollups = _detect_cross_foreach_variable_reads(pipeline.tasks) setup_tasks_out = _dedupe_setup_tasks(artifacts.setup_tasks) setup_tasks_out.extend(cross_scope_rollups) - # C-36 (SCHED4-001): emit a manual_schedule_time_of_day SetupTask - # whenever the trigger.periodic schedule carries hours/minutes/weekDays - # that the periodic primitive can't encode. SETUP.md picks it up so - # the user can manually add the time-of-day to the cron expression. + # C-36 (SCHED4-001): the periodic primitive can't encode hours/minutes/weekDays, so emit a + # manual_schedule_time_of_day SetupTask for the user to add the time-of-day to the cron expression. if pipeline.schedule and pipeline.schedule.get("time_of_day_note"): setup_tasks_out.append( SetupTask( @@ -347,11 +359,8 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: ) ) - # C-39 (LSC4-004): when any cluster hint references an ADF - # authentication mode that has no direct Databricks equivalent (MSI, - # CredentialReference) the bundle's default_cluster silently uses - # ``single_user_name: ${workspace.current_user.userName}``. Surface - # a manual_credential SetupTask so SETUP.md flags the substitution. + # C-39 (LSC4-004): ADF auth modes with no Databricks equivalent (MSI, CredentialReference) make the + # default_cluster fall back to single_user_name: ${workspace.current_user.userName}; flag it via SetupTask. seen_auth: set[tuple[str, str]] = set() for hint in cluster_hints: auth = hint.get("_adf_authentication") or "" @@ -382,6 +391,7 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: return PreparedWorkflow( name=pipeline.name, tasks=all_tasks, + parameters=list(pipeline.parameters or []), notebooks=list(artifacts.notebooks), secrets=unique_secrets, setup_tasks=setup_tasks_out, @@ -414,25 +424,18 @@ def _detect_cross_foreach_variable_reads(activities: list[Activity]) -> list[Set if isinstance(activity, ForEachActivity): for inner in activity.inner_activities: if isinstance(inner, SetVariableActivity): - var_set_inside_foreach.setdefault(inner.variable_name, []).append( - activity.task_key - ) + var_set_inside_foreach.setdefault(inner.variable_name, []).append(activity.task_key) if not var_set_inside_foreach: return [] - # Identify variables that are also set OUTSIDE any ForEach -- those are - # not cross-scope dangers because the parent always has a fresh setter - # to point at. + # Variables also set OUTSIDE any ForEach are safe -- the parent always has a fresh setter to point at. set_outside: set[str] = set() for activity in activities: if isinstance(activity, SetVariableActivity): set_outside.add(activity.variable_name) - dangerous_vars = { - name: parents for name, parents in var_set_inside_foreach.items() - if name not in set_outside - } + dangerous_vars = {name: parents for name, parents in var_set_inside_foreach.items() if name not in set_outside} if not dangerous_vars: return [] @@ -443,7 +446,7 @@ def _detect_cross_foreach_variable_reads(activities: list[Activity]) -> list[Set def _read_refs(text: str) -> set[str]: if not isinstance(text, str): return set() - return {m.group(1) for m in var_ref_pattern.finditer(text)} + return {match.group(1) for match in var_ref_pattern.finditer(text)} def _walk_activity_strings(activity: Activity) -> Iterable[str]: # Yield every string-like field the variable might appear in. diff --git a/src/orchestra/preparer/workspace_downloader.py b/src/flowx/preparer/workspace_downloader.py similarity index 55% rename from src/orchestra/preparer/workspace_downloader.py rename to src/flowx/preparer/workspace_downloader.py index e5167a1..3d2e42d 100644 --- a/src/orchestra/preparer/workspace_downloader.py +++ b/src/flowx/preparer/workspace_downloader.py @@ -12,6 +12,105 @@ logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Databricks runtime detection and auto-configuration +# --------------------------------------------------------------------------- + +_WORKSPACE_ROOT = Path("/Workspace") + +# Common notebook file extensions on the Databricks workspace filesystem. +_NOTEBOOK_EXTENSIONS = (".py", ".sql", ".scala", ".r", ".R", ".ipynb") + + +def _is_databricks_runtime() -> bool: + """Returns True if running inside a Databricks cluster or serverless compute.""" + return os.environ.get("DATABRICKS_RUNTIME_VERSION") is not None + + +def _local_workspace_accessible() -> bool: + """Returns True if the /Workspace filesystem is mounted and readable.""" + return _WORKSPACE_ROOT.is_dir() + + +def _try_local_workspace_read(workspace_path: str) -> str | None: + """Attempts to read a notebook directly from the local /Workspace filesystem. + + On Databricks compute (classic or serverless), workspace files are mounted + at ``/Workspace/``. Notebooks are stored with a language extension + (e.g. ``.py``, ``.sql``). This function probes the path with each known + extension and returns the source if found — no SDK auth required. + + Args: + workspace_path: Logical workspace path (e.g. ``"/Shared/ETL/transform"``). + + Returns: + Notebook source as a string, or ``None`` if not found locally. + """ + if not _local_workspace_accessible(): + return None + + base = _WORKSPACE_ROOT / workspace_path.lstrip("/") + + # Try the exact path first (already has an extension or is a plain file) + if base.is_file(): + try: + return base.read_text(encoding="utf-8") + except OSError as exc: + logger.debug("Local read failed for %s: %s", base, exc) + + # Probe with common notebook extensions + for ext in _NOTEBOOK_EXTENSIONS: + candidate = base.with_suffix(ext) + if candidate.is_file(): + try: + content = candidate.read_text(encoding="utf-8") + logger.info("Read notebook from local filesystem: %s", candidate) + return content + except OSError as exc: + logger.debug("Local read failed for %s: %s", candidate, exc) + + return None + + +def _ensure_databricks_runtime_auth() -> bool: + """Auto-configures ~/.databrickscfg from the notebook runtime context. + + On Databricks serverless (or classic cluster) compute, no CLI auth is + pre-configured but the runtime provides host + token via the REPL context. + This function detects that situation and writes a DEFAULT profile so the + Databricks SDK can authenticate transparently. + """ + cfg_path = _get_databrickscfg_path() + if cfg_path.exists() and cfg_path.stat().st_size > 0: + return True + + if os.environ.get("DATABRICKS_HOST") and os.environ.get("DATABRICKS_TOKEN"): + return True + + try: + from dbruntime.databricks_repl_context import get_context # type: ignore[import-not-found] + + context = get_context() + host = f"https://{context.browserHostName}" + token = context.apiToken + if not host or not token: + logger.warning("Databricks runtime detected but host/token unavailable from context") + return False + + cfg_path.parent.mkdir(parents=True, exist_ok=True) + with open(cfg_path, "w") as f: + f.write(f"[DEFAULT]\nhost = {host}\ntoken = {token}\n") + logger.info("Auto-configured Databricks auth from runtime context -> %s", cfg_path) + return True + except ImportError: + logger.debug("dbruntime not available; cannot auto-configure auth via REPL context") + return False + except Exception as exc: + logger.warning("Failed to auto-configure Databricks runtime auth: %s", exc) + return False + + # --------------------------------------------------------------------------- # Module-level state — resolved once per process, reused across calls # --------------------------------------------------------------------------- @@ -19,14 +118,13 @@ _resolved_profile: str | None = None _profile_resolved: bool = False -# When False (default), preparers preserve workspace artifact paths in-place -# instead of attempting a network download. The CLI flips this on so that -# `databricks bundle deploy` can ship the source files across environments. +# When False (default), preparers keep workspace paths in-place; the CLI flips it on so +# `databricks bundle deploy` ships the source files across environments. _downloads_enabled: bool = False def _get_databrickscfg_path() -> Path: - """Return the path to the Databricks CLI config file.""" + """Returns the path to the Databricks CLI config file.""" override = os.environ.get("DATABRICKS_CONFIG_FILE") if override: return Path(override) @@ -34,7 +132,7 @@ def _get_databrickscfg_path() -> Path: def _list_profiles() -> list[str]: - """Parses ``~/.databrickscfg`` and return available profile names. + """Parses ``~/.databrickscfg`` and returns available profile names. Returns: Sorted list of profile section names. Empty list if the file @@ -95,7 +193,7 @@ def _resolve_profile() -> str | None: def _prompt_for_profile(profiles: list[str]) -> str: - """Interactively prompt the user to select a profile. + """Interactively prompts the user to select a profile. Args: profiles: Available profile names. @@ -139,7 +237,7 @@ def _prompt_for_profile(profiles: list[str]) -> str: def set_profile(profile: str | None) -> None: - """Explicitly set the profile to use, bypassing auto-resolution. + """Explicitly sets the profile to use, bypassing auto-resolution. Args: profile: Profile name, or ``None`` to reset to auto-resolution. @@ -150,16 +248,17 @@ def set_profile(profile: str | None) -> None: def _get_workspace_client(): - """Return a ``WorkspaceClient`` configured with the resolved profile. - - Returns: - A ``WorkspaceClient`` instance. + """Returns a ``WorkspaceClient`` configured with the resolved profile. - Raises: - ImportError: If ``databricks-sdk`` is not installed. + On Databricks runtime, auto-configures auth from the notebook context + before constructing the client. """ from databricks.sdk import WorkspaceClient # type: ignore[import-not-found] + # Ensure auth is available when running on Databricks compute + if _is_databricks_runtime(): + _ensure_databricks_runtime_auth() + profile = _resolve_profile() if profile: return WorkspaceClient(profile=profile) @@ -172,7 +271,11 @@ def _get_workspace_client(): def download_notebook(workspace_path: str) -> str | None: - """Download a notebook from Databricks workspace. + """Downloads a notebook from Databricks workspace. + + Attempts local filesystem access first (zero-auth, works on any + Databricks compute where /Workspace is mounted). Falls back to the + Databricks SDK export API when local access is unavailable. Args: workspace_path: Workspace path (e.g., ``"/Shared/flowx/transform"``). @@ -180,6 +283,12 @@ def download_notebook(workspace_path: str) -> str | None: Returns: Notebook source code as a string, or ``None`` if download failed. """ + # Fast path: read directly from /Workspace mount (no auth needed) + local_content = _try_local_workspace_read(workspace_path) + if local_content is not None: + return local_content + + # Slow path: SDK-based export (requires auth) try: from databricks.sdk.service.workspace import ExportFormat # type: ignore[import-not-found] @@ -195,7 +304,7 @@ def download_notebook(workspace_path: str) -> str | None: def download_dbfs_file(dbfs_path: str) -> bytes | None: - """Download a file from DBFS. + """Downloads a file from DBFS. Args: dbfs_path: DBFS path (e.g., ``"dbfs:/scripts/process.py"`` or @@ -223,29 +332,53 @@ def download_dbfs_file(dbfs_path: str) -> bytes | None: def enable_workspace_downloads(enabled: bool = True) -> None: - """Globally enable or disable workspace artifact downloads.""" + """Globally enables or disables workspace artifact downloads.""" global _downloads_enabled # noqa: PLW0603 _downloads_enabled = bool(enabled) def workspace_downloads_enabled() -> bool: - """Return True iff preparers should attempt to download workspace artifacts.""" + """Returns True iff preparers should attempt to download workspace artifacts.""" return _downloads_enabled def auth_available() -> bool: - """Return True iff there is any usable Databricks authentication on this host. - - A resolvable ``.databrickscfg`` profile, ``DATABRICKS_CONFIG_PROFILE``, or - the standard ``DATABRICKS_HOST`` + ``DATABRICKS_TOKEN`` env-var pair will - all satisfy this check. This is a pre-flight signal — it does not validate - that the credentials actually authorize against any specific workspace. + """Returns True iff there is any usable Databricks authentication on this host. + + A resolvable ``.databrickscfg`` profile, ``DATABRICKS_CONFIG_PROFILE``, the + standard ``DATABRICKS_HOST`` + ``DATABRICKS_TOKEN`` pair, or OAuth + machine-to-machine creds (``DATABRICKS_HOST`` + ``DATABRICKS_CLIENT_ID`` + + ``DATABRICKS_CLIENT_SECRET`` — what a Databricks App injects for its service + principal) all satisfy this check. Local /Workspace filesystem access + (available on any Databricks compute) also satisfies it since notebooks can + be read directly without API auth. + + This is a pre-flight signal — it does not validate that the credentials + actually authorize against any specific workspace (the SDK call does that, + falling back to a placeholder on failure). It deliberately avoids + constructing an SDK ``Config``/client, since OAuth resolution can trigger a + network round-trip. """ + if _local_workspace_accessible(): + return True if os.environ.get("DATABRICKS_CONFIG_PROFILE"): return True if os.environ.get("DATABRICKS_HOST") and os.environ.get("DATABRICKS_TOKEN"): return True - return bool(_list_profiles()) + # OAuth M2M (e.g. the MCP path: flowx as a Databricks App injects the SP client id/secret, + # which WorkspaceClient() picks up automatically). + if ( + os.environ.get("DATABRICKS_HOST") + and os.environ.get("DATABRICKS_CLIENT_ID") + and os.environ.get("DATABRICKS_CLIENT_SECRET") + ): + return True + if _list_profiles(): + return True + # Last resort: bootstrap auth from the Databricks runtime context + if _is_databricks_runtime(): + return _ensure_databricks_runtime_auth() + return False def prompt_for_auth_if_missing( @@ -253,7 +386,7 @@ def prompt_for_auth_if_missing( *, interactive: bool | None = None, ) -> bool: - """Warn the user when auth is missing and confirm how to proceed. + """Warns the user when auth is missing and confirms how to proceed. Args: sample_paths: Workspace paths the preparer is about to try to download. @@ -269,7 +402,7 @@ def prompt_for_auth_if_missing( if auth_available(): return True - paths = [p for p in sample_paths if p] + paths = [path for path in sample_paths if path] cfg_path = _get_databrickscfg_path() print( @@ -280,7 +413,7 @@ def prompt_for_auth_if_missing( if paths: preview = ", ".join(paths[:3]) suffix = ", …" if len(paths) > 3 else "" - print(f" Artifacts to vendor: {preview}{suffix}", file=sys.stderr) + print(f" Artifacts to download: {preview}{suffix}", file=sys.stderr) print( "\nTo authenticate, run one of:\n" " databricks auth login --host https://.cloud.databricks.com\n" diff --git a/src/flowx/reporting/__init__.py b/src/flowx/reporting/__init__.py new file mode 100644 index 0000000..443eee4 --- /dev/null +++ b/src/flowx/reporting/__init__.py @@ -0,0 +1 @@ +"""Reporting: persist migration coverage results to a UC table and install a dashboard.""" diff --git a/src/flowx/reporting/coverage.py b/src/flowx/reporting/coverage.py new file mode 100644 index 0000000..356c562 --- /dev/null +++ b/src/flowx/reporting/coverage.py @@ -0,0 +1,117 @@ +"""Build per-pipeline migration-coverage rows from the discover-phase metadata. + +Joins the two artifacts the discover phase writes into ``/metadata/``: + +* ``profile_report.csv`` -- per-pipeline complexity (activity/dataset/linked-service + counts, collapsible patterns, activity-category counts, complexity score + size). +* ``inventory.json`` -- per-activity translation strategy, from which the + deterministic / agentic / unsupported counts and coverage % are derived. + +The result is one metric row per pipeline (no run metadata -- ``run_id`` / +``run_date`` / ``run_by`` are stamped on at write time by :mod:`reporting.results`). +""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +# Metric columns (order matters: it drives the results-table column order). +COVERAGE_METRIC_COLUMNS: tuple[str, ...] = ( + "pipeline", + "activities", + "datasets", + "linked_services", + "collapsible_patterns", + "databricks_native_activities", + "control_flow_activities", + "other_activities", + "deterministic_activities", + "agentic_activities", + "unsupported_activities", + "coverage_pct", + "complexity_score", + "complexity_size", +) + +_CSV_INT_COLUMNS: tuple[str, ...] = ( + "datasets", + "linked_services", + "collapsible_patterns", + "databricks_native_activities", + "control_flow_activities", + "other_activities", + "complexity_score", +) + + +def _coverage_pct(deterministic: int, agentic: int, total: int) -> float: + """Coverage % = (deterministic + agentic) / total activities, rounded to 1dp.""" + if total <= 0: + return 0.0 + return round((deterministic + agentic) / total * 100, 1) + + +def build_coverage_rows(metadata_dir: Path) -> list[dict[str, Any]]: + """Builds per-pipeline coverage rows from a migration ``metadata/`` directory. + + Args: + metadata_dir: The bundle's ``metadata/`` folder containing ``inventory.json`` + and ``profile_report.csv``. + + Returns: + One dict per pipeline keyed by :data:`COVERAGE_METRIC_COLUMNS`, ordered by + pipeline name. The inventory's pipeline set is authoritative; complexity + columns are looked up from the CSV (defaulting to 0 / "" when absent). + + Raises: + FileNotFoundError: When ``inventory.json`` is missing. + """ + inventory_path = metadata_dir / "inventory.json" + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + + csv_by_pipeline: dict[str, dict[str, str]] = {} + csv_path = metadata_dir / "profile_report.csv" + if csv_path.exists(): + with csv_path.open(encoding="utf-8") as handle: + for row in csv.DictReader(handle): + csv_by_pipeline[row["pipeline"]] = row + + rows: list[dict[str, Any]] = [] + for pipeline in inventory.get("pipelines", []): + name = pipeline.get("name", "") + strategies = [activity.get("strategy") for activity in pipeline.get("activities", [])] + deterministic = strategies.count("deterministic") + agentic = strategies.count("agentic") + unsupported = strategies.count("unsupported") + total = len(strategies) + csv_row = csv_by_pipeline.get(name, {}) + + def _csv_int(col: str, _csv_row: dict[str, str] = csv_row) -> int: + try: + return int(_csv_row.get(col, 0) or 0) + except (TypeError, ValueError): + return 0 + + rows.append( + { + "pipeline": name, + "activities": total, + "datasets": _csv_int("datasets"), + "linked_services": _csv_int("linked_services"), + "collapsible_patterns": _csv_int("collapsible_patterns"), + "databricks_native_activities": _csv_int("databricks_native_activities"), + "control_flow_activities": _csv_int("control_flow_activities"), + "other_activities": _csv_int("other_activities"), + "deterministic_activities": deterministic, + "agentic_activities": agentic, + "unsupported_activities": unsupported, + "coverage_pct": _coverage_pct(deterministic, agentic, total), + "complexity_score": _csv_int("complexity_score"), + "complexity_size": csv_row.get("complexity_size", "") or "", + } + ) + rows.sort(key=lambda row: row["pipeline"]) + return rows diff --git a/src/flowx/reporting/dashboard.py b/src/flowx/reporting/dashboard.py new file mode 100644 index 0000000..a950445 --- /dev/null +++ b/src/flowx/reporting/dashboard.py @@ -0,0 +1,97 @@ +"""Install a published AI/BI (Lakeview) dashboard that visualizes migration coverage. + +Builds a dashboard from :data:`dashboard_template.json` (datasets + widgets over the +results table written by :mod:`reporting.results`), creates it via the Databricks SDK +Lakeview API, and publishes it so it is immediately viewable. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +from flowx.reporting.results import resolve_warehouse_id + +logger = logging.getLogger(__name__) + +_TEMPLATE_PATH = Path(__file__).with_name("dashboard_template.json") +_TABLE_PLACEHOLDER = "{{RESULTS_TABLE}}" + + +def build_serialized_dashboard(table_fqn: str) -> str: + """Returns the serialized Lakeview dashboard JSON for *table_fqn*. + + Substitutes the ``{{RESULTS_TABLE}}`` placeholder in every dataset query with the + fully-qualified results table, and returns a compact JSON string suitable for the + Lakeview ``serialized_dashboard`` field. + + Raises: + ValueError: When *table_fqn* is empty. + """ + if not table_fqn: + raise ValueError("table_fqn is required to build the coverage dashboard") + spec = json.loads(_TEMPLATE_PATH.read_text(encoding="utf-8")) + for dataset in spec.get("datasets", []): + dataset["queryLines"] = [line.replace(_TABLE_PLACEHOLDER, table_fqn) for line in dataset.get("queryLines", [])] + return json.dumps(spec) + + +def _default_parent_path(client: Any) -> str: + """Returns ``/Workspace/Users/`` for the dashboard's parent folder.""" + try: + user = client.current_user.me().user_name + if user: + return f"/Workspace/Users/{user}" + except Exception as exc: # noqa: BLE001 - fall back to /Workspace + logger.debug("Could not resolve current user for parent path: %s", exc) + return "/Workspace" + + +def install_dashboard( + table_fqn: str, + warehouse_id: str | None = None, + display_name: str | None = None, + parent_path: str | None = None, + client: Any | None = None, +) -> tuple[str, str]: + """Creates and publishes the migration-coverage dashboard. + + Args: + table_fqn: Results table the dashboard reads (``catalog.schema.table``). + warehouse_id: SQL warehouse backing the dashboard; auto-detected when omitted. + display_name: Dashboard name; defaults to ``Migration Coverage —
``. + parent_path: Workspace folder; defaults to the current user's home. + client: Optional ``WorkspaceClient`` (injected in tests). + + Returns: + ``(dashboard_id, url)`` -- ``url`` is best-effort (empty when host is unknown). + """ + from databricks.sdk.service.dashboards import Dashboard + + if client is None: + from flowx.preparer.workspace_downloader import _get_workspace_client + + client = _get_workspace_client() + + resolved_wh = resolve_warehouse_id(client, warehouse_id) + serialized = build_serialized_dashboard(table_fqn) + name = display_name or f"Migration Coverage — {table_fqn}" + parent = parent_path or _default_parent_path(client) + + created = client.lakeview.create( + dashboard=Dashboard( + display_name=name, + serialized_dashboard=serialized, + warehouse_id=resolved_wh, + parent_path=parent, + ) + ) + dashboard_id = created.dashboard_id + client.lakeview.publish(dashboard_id=dashboard_id, warehouse_id=resolved_wh) + + host = str(getattr(getattr(client, "config", None), "host", "") or "").rstrip("/") + url = f"{host}/sql/dashboardsv3/{dashboard_id}" if host else "" + logger.info("Installed coverage dashboard '%s' (%s).", name, dashboard_id) + return dashboard_id, url diff --git a/src/flowx/reporting/dashboard_template.json b/src/flowx/reporting/dashboard_template.json new file mode 100644 index 0000000..25ed878 --- /dev/null +++ b/src/flowx/reporting/dashboard_template.json @@ -0,0 +1,538 @@ +{ + "datasets": [ + { + "name": "latest_summary", + "displayName": "Latest run summary", + "queryLines": [ + "SELECT COUNT(*) AS pipelines, ", + "SUM(activities) AS activities, ", + "SUM(deterministic_activities) AS deterministic_activities, ", + "SUM(agentic_activities) AS agentic_activities, ", + "SUM(unsupported_activities) AS unsupported_activities, ", + "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(activities),0),1) AS coverage_pct ", + "FROM {{RESULTS_TABLE}} ", + "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1)" + ] + }, + { + "name": "latest_by_size", + "displayName": "Pipelines by complexity (latest run)", + "queryLines": [ + "SELECT complexity_size, COUNT(*) AS pipelines ", + "FROM {{RESULTS_TABLE}} ", + "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1) ", + "GROUP BY complexity_size" + ] + }, + { + "name": "latest_pipelines", + "displayName": "Pipeline coverage (latest run)", + "queryLines": [ + "SELECT pipeline, activities, deterministic_activities, agentic_activities, ", + "unsupported_activities, coverage_pct, collapsible_patterns, complexity_size ", + "FROM {{RESULTS_TABLE}} ", + "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1) ", + "ORDER BY coverage_pct ASC, activities DESC" + ] + }, + { + "name": "runs_over_time", + "displayName": "Coverage over runs", + "queryLines": [ + "SELECT DATE_TRUNC('SECOND', run_date) AS run_ts, ", + "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(activities),0),1) AS coverage_pct, ", + "SUM(activities) AS activities ", + "FROM {{RESULTS_TABLE}} ", + "GROUP BY DATE_TRUNC('SECOND', run_date) ", + "ORDER BY run_ts" + ] + } + ], + "pages": [ + { + "name": "coverage", + "displayName": "Migration Coverage", + "pageType": "PAGE_TYPE_CANVAS", + "layout": [ + { + "widget": { + "name": "title", + "multilineTextboxSpec": { + "lines": [ + "## ADF \u2192 Databricks Migration Coverage" + ] + } + }, + "position": { + "x": 0, + "y": 0, + "width": 6, + "height": 1 + } + }, + { + "widget": { + "name": "subtitle", + "multilineTextboxSpec": { + "lines": [ + "Per-pipeline translation coverage from the latest flowx run. Coverage % = (deterministic + agentic) / total activities." + ] + } + }, + "position": { + "x": 0, + "y": 1, + "width": 6, + "height": 1 + } + }, + { + "widget": { + "name": "kpi-pipelines", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "pipelines", + "expression": "`pipelines`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "pipelines", + "displayName": "Pipelines" + } + }, + "frame": { + "title": "Pipelines", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 2, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "kpi-coverage", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "coverage_pct", + "expression": "`coverage_pct`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "coverage_pct", + "displayName": "Coverage %" + } + }, + "frame": { + "title": "Coverage %", + "showTitle": true + } + } + }, + "position": { + "x": 2, + "y": 2, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "kpi-activities", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "activities", + "expression": "`activities`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "activities", + "displayName": "Activities" + } + }, + "frame": { + "title": "Activities", + "showTitle": true + } + } + }, + "position": { + "x": 4, + "y": 2, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "kpi-det", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "deterministic_activities", + "expression": "`deterministic_activities`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "deterministic_activities", + "displayName": "Deterministic" + } + }, + "frame": { + "title": "Deterministic", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 5, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "kpi-agentic", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "agentic_activities", + "expression": "`agentic_activities`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "agentic_activities", + "displayName": "Agentic" + } + }, + "frame": { + "title": "Agentic", + "showTitle": true + } + } + }, + "position": { + "x": 2, + "y": 5, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "kpi-unsup", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_summary", + "fields": [ + { + "name": "unsupported_activities", + "expression": "`unsupported_activities`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "unsupported_activities", + "displayName": "Unsupported" + } + }, + "frame": { + "title": "Unsupported", + "showTitle": true + } + } + }, + "position": { + "x": 4, + "y": 5, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "by-size", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_by_size", + "fields": [ + { + "name": "complexity_size", + "expression": "`complexity_size`" + }, + { + "name": "pipelines", + "expression": "`pipelines`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 3, + "widgetType": "bar", + "encodings": { + "x": { + "fieldName": "complexity_size", + "scale": { + "type": "categorical" + }, + "displayName": "Complexity" + }, + "y": { + "fieldName": "pipelines", + "scale": { + "type": "quantitative" + }, + "displayName": "Pipelines" + } + }, + "frame": { + "title": "Pipelines by complexity size", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 8, + "width": 3, + "height": 6 + } + }, + { + "widget": { + "name": "coverage-trend", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "runs_over_time", + "fields": [ + { + "name": "run_ts", + "expression": "`run_ts`" + }, + { + "name": "coverage_pct", + "expression": "`coverage_pct`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 3, + "widgetType": "line", + "encodings": { + "x": { + "fieldName": "run_ts", + "scale": { + "type": "temporal" + }, + "displayName": "Run" + }, + "y": { + "fieldName": "coverage_pct", + "scale": { + "type": "quantitative" + }, + "displayName": "Coverage %" + } + }, + "frame": { + "title": "Coverage over runs", + "showTitle": true + } + } + }, + "position": { + "x": 3, + "y": 8, + "width": 3, + "height": 6 + } + }, + { + "widget": { + "name": "pipeline-table", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "latest_pipelines", + "fields": [ + { + "name": "pipeline", + "expression": "`pipeline`" + }, + { + "name": "activities", + "expression": "`activities`" + }, + { + "name": "deterministic_activities", + "expression": "`deterministic_activities`" + }, + { + "name": "agentic_activities", + "expression": "`agentic_activities`" + }, + { + "name": "unsupported_activities", + "expression": "`unsupported_activities`" + }, + { + "name": "coverage_pct", + "expression": "`coverage_pct`" + }, + { + "name": "collapsible_patterns", + "expression": "`collapsible_patterns`" + }, + { + "name": "complexity_size", + "expression": "`complexity_size`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "table", + "encodings": { + "columns": [ + { + "fieldName": "pipeline", + "displayName": "Pipeline" + }, + { + "fieldName": "activities", + "displayName": "Activities" + }, + { + "fieldName": "deterministic_activities", + "displayName": "Deterministic" + }, + { + "fieldName": "agentic_activities", + "displayName": "Agentic" + }, + { + "fieldName": "unsupported_activities", + "displayName": "Unsupported" + }, + { + "fieldName": "coverage_pct", + "displayName": "Coverage %" + }, + { + "fieldName": "collapsible_patterns", + "displayName": "Collapsible patterns" + }, + { + "fieldName": "complexity_size", + "displayName": "Complexity" + } + ] + }, + "frame": { + "title": "Pipeline coverage detail", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 14, + "width": 6, + "height": 7 + } + } + ] + } + ] +} \ No newline at end of file diff --git a/src/flowx/reporting/results.py b/src/flowx/reporting/results.py new file mode 100644 index 0000000..b5d3af7 --- /dev/null +++ b/src/flowx/reporting/results.py @@ -0,0 +1,171 @@ +"""Persist per-pipeline migration coverage to a Unity Catalog table. + +Each migration run is stamped with a single UUID ``run_id`` (shared by every row of +the run), ``run_date`` (``CURRENT_TIMESTAMP()``), and ``run_by`` (``CURRENT_USER()``), +so coverage can be tracked over time and per user. Rows are written via the +Databricks SDK Statement Execution API against a SQL warehouse (auto-detected when one +is not supplied). +""" + +from __future__ import annotations + +import logging +import uuid +from pathlib import Path +from typing import Any + +from flowx.reporting.coverage import COVERAGE_METRIC_COLUMNS, build_coverage_rows + +logger = logging.getLogger(__name__) + +# Full results-table schema: run metadata first, then the per-pipeline metrics. +# ``run_date`` / ``run_by`` are populated by SQL functions (not Python literals). +_METRIC_SQL_TYPES: dict[str, str] = { + "pipeline": "STRING", + "activities": "INT", + "datasets": "INT", + "linked_services": "INT", + "collapsible_patterns": "INT", + "databricks_native_activities": "INT", + "control_flow_activities": "INT", + "other_activities": "INT", + "deterministic_activities": "INT", + "agentic_activities": "INT", + "unsupported_activities": "INT", + "coverage_pct": "DOUBLE", + "complexity_score": "INT", + "complexity_size": "STRING", +} + +RESULTS_COLUMNS: tuple[tuple[str, str], ...] = ( + ("run_id", "STRING"), + ("run_date", "TIMESTAMP"), + ("run_by", "STRING"), + *((col, _METRIC_SQL_TYPES[col]) for col in COVERAGE_METRIC_COLUMNS), +) + +_STRING_METRICS: frozenset[str] = frozenset({"pipeline", "complexity_size"}) + + +def _sql_str(value: Any) -> str: + """Renders a value as a single-quoted SQL string literal (quotes doubled).""" + return "'" + str(value).replace("'", "''") + "'" + + +def _metric_value_sql(column: str, value: Any) -> str: + """Renders one metric column value as a SQL literal.""" + if column in _STRING_METRICS: + return _sql_str(value) + if column == "coverage_pct": + return repr(float(value or 0)) + return str(int(value or 0)) + + +def build_create_table_sql(table_fqn: str) -> str: + """Returns ``CREATE TABLE IF NOT EXISTS`` for the results table.""" + cols = ",\n ".join(f"{name} {sql_type}" for name, sql_type in RESULTS_COLUMNS) + return f"CREATE TABLE IF NOT EXISTS {table_fqn} (\n {cols}\n)" + + +def build_insert_sql(table_fqn: str, rows: list[dict[str, Any]], run_id: str) -> str: + """Returns a single multi-row ``INSERT`` stamping run metadata onto every row. + + ``run_id`` is a literal (same for the whole run); ``run_date`` and ``run_by`` use + the ``CURRENT_TIMESTAMP()`` / ``CURRENT_USER()`` SQL functions so the workspace + records the actual write time and identity. + """ + column_names = ", ".join(name for name, _ in RESULTS_COLUMNS) + tuples: list[str] = [] + for row in rows: + metrics = ", ".join(_metric_value_sql(col, row.get(col)) for col in COVERAGE_METRIC_COLUMNS) + tuples.append(f"({_sql_str(run_id)}, CURRENT_TIMESTAMP(), CURRENT_USER(), {metrics})") + values = ",\n ".join(tuples) + return f"INSERT INTO {table_fqn} ({column_names}) VALUES\n {values}" + + +def resolve_warehouse_id(client: Any, warehouse_id: str | None = None) -> str: + """Resolves a SQL warehouse id, preferring RUNNING then serverless when auto-detecting. + + Args: + client: A ``WorkspaceClient``. + warehouse_id: An explicit id; returned as-is when provided. + + Returns: + The resolved warehouse id. + + Raises: + RuntimeError: When no warehouse is available to auto-detect. + """ + if warehouse_id: + return warehouse_id + warehouses = list(client.warehouses.list()) + if not warehouses: + raise RuntimeError( + "No SQL warehouse found to write results. Pass --warehouse-id with a warehouse " + "that can write to the target table." + ) + + def _rank(warehouse: Any) -> tuple[int, int]: + state = str(getattr(getattr(warehouse, "state", None), "value", getattr(warehouse, "state", "")) or "") + is_running = 1 if state.upper() == "RUNNING" else 0 + type_value = str( + getattr(getattr(warehouse, "warehouse_type", None), "value", getattr(warehouse, "warehouse_type", "")) or "" + ) + is_serverless = ( + 1 if getattr(warehouse, "enable_serverless_compute", False) or "SERVERLESS" in type_value.upper() else 0 + ) + return (is_running, is_serverless) + + best = max(warehouses, key=_rank) + logger.info("Auto-selected SQL warehouse '%s' (%s).", getattr(best, "name", "?"), best.id) + return best.id + + +def _execute(client: Any, statement: str, warehouse_id: str) -> None: + """Runs a SQL statement via the Statement Execution API; raises on failure.""" + resp = client.statement_execution.execute_statement( + statement=statement, warehouse_id=warehouse_id, wait_timeout="50s" + ) + state = getattr(getattr(resp, "status", None), "state", None) + state_str = str(getattr(state, "value", state) or "") + if state_str.upper() not in ("", "SUCCEEDED"): + err = getattr(getattr(resp, "status", None), "error", None) + raise RuntimeError(f"Statement failed ({state_str}): {getattr(err, 'message', err)}") + + +def write_results( + metadata_dir: Path, + table_fqn: str, + warehouse_id: str | None = None, + client: Any | None = None, +) -> tuple[str, int]: + """Writes per-pipeline coverage rows for one run to *table_fqn*. + + Creates the table if needed, then inserts one row per pipeline stamped with a + fresh ``run_id`` (and SQL ``run_date`` / ``run_by``). + + Args: + metadata_dir: The migration ``metadata/`` directory. + table_fqn: Target table as ``catalog.schema.table``. + warehouse_id: Optional SQL warehouse id; auto-detected when omitted. + client: Optional ``WorkspaceClient`` (injected in tests). + + Returns: + ``(run_id, row_count)``. + """ + rows = build_coverage_rows(metadata_dir) + if not rows: + logger.warning("No pipelines found in %s; nothing to record.", metadata_dir) + return "", 0 + + if client is None: + from flowx.preparer.workspace_downloader import _get_workspace_client + + client = _get_workspace_client() + + resolved_wh = resolve_warehouse_id(client, warehouse_id) + run_id = str(uuid.uuid4()) + _execute(client, build_create_table_sql(table_fqn), resolved_wh) + _execute(client, build_insert_sql(table_fqn, rows, run_id), resolved_wh) + logger.info("Recorded %d pipeline rows to %s (run_id=%s).", len(rows), table_fqn, run_id) + return run_id, len(rows) diff --git a/src/orchestra/translator/activity_translators/__init__.py b/src/flowx/translator/__init__.py similarity index 100% rename from src/orchestra/translator/activity_translators/__init__.py rename to src/flowx/translator/__init__.py diff --git a/src/flowx/translator/activity_translators/__init__.py b/src/flowx/translator/activity_translators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/orchestra/translator/activity_translators/append_variable.py b/src/flowx/translator/activity_translators/append_variable.py similarity index 100% rename from src/orchestra/translator/activity_translators/append_variable.py rename to src/flowx/translator/activity_translators/append_variable.py diff --git a/src/orchestra/translator/activity_translators/copy.py b/src/flowx/translator/activity_translators/copy.py similarity index 96% rename from src/orchestra/translator/activity_translators/copy.py rename to src/flowx/translator/activity_translators/copy.py index 4fdf14d..af595a4 100644 --- a/src/orchestra/translator/activity_translators/copy.py +++ b/src/flowx/translator/activity_translators/copy.py @@ -25,11 +25,8 @@ "DeltaLakeDataset": "delta", } -# Map ADF dataset location types to (uri-scheme, host-template) pairs used -# when constructing the external-volume URL. ``{account}`` is replaced with -# the storage account name (or a ``${var.storage_account}`` placeholder when -# the linked service does not expose one) and ``{bucket}`` with the bucket -# name for AWS / GCS sinks. +# Maps ADF dataset location types to (uri-scheme, host-template) pairs for the external-volume URL; +# {account} -> storage account (or ${var.storage_account}) and {bucket} -> bucket name for AWS/GCS. _LOCATION_URL_TEMPLATE: dict[str, str] = { "AzureBlobFSLocation": "abfss://{container}@{account}.dfs.core.windows.net/", "AzureBlobStorageLocation": "abfss://{container}@{account}.dfs.core.windows.net/", @@ -42,11 +39,8 @@ _ACCOUNT_NAME_RE = re.compile(r"AccountName=([A-Za-z0-9]+)", re.IGNORECASE) _DATASET_PARAM_RE = re.compile(r"^@dataset\(\)\.([A-Za-z_][A-Za-z0-9_]*)$") -# Database connection-string parsers: each picks up the canonical -# host/port/database fields from an ADF linked service's -# ``connectionString``. The patterns are intentionally tolerant of -# casing and surrounding whitespace because ADF accepts both -# ``Server=`` and ``server=`` etc. +# Database connection-string parsers: pull host/port/database from an ADF linked service's +# connectionString. Tolerant of casing/whitespace (ADF accepts Server= and server=). _AZURE_SQL_SERVER_RE = re.compile(r"\bServer=(?:tcp:)?([^,;]+?)(?:,(\d+))?(?:;|$)", re.IGNORECASE) _AZURE_SQL_DATABASE_RE = re.compile(r"\b(?:Initial Catalog|Database)=([^;]+)", re.IGNORECASE) _MYSQL_SERVER_RE = re.compile(r"\b(?:Server|Host)=([^;]+)", re.IGNORECASE) @@ -231,9 +225,8 @@ def _resolve_path_info( if dataset_ref is not None and getattr(dataset_ref, "parameters", None): effective.update(dict(dataset_ref.parameters)) - # Container name is used in the volume URL (no expressions allowed); the - # other path components flow into the notebook write call as f-string - # fragments so date/time expressions evaluate at runtime. + # Container name goes in the volume URL (no expressions allowed); other path components flow into + # the notebook write as f-string fragments so date/time expressions evaluate at runtime. container = _resolve_param_value( location.get("container") or location.get("fileSystem") or location.get("bucketName"), effective, @@ -623,10 +616,8 @@ def translate( sink_dataset_type = sink_dataset_props.get("type") sink_format = _DATASET_TYPE_TO_SPARK_FORMAT.get(sink_dataset_type or "") - # File-on-cloud-storage sinks: compose a UC external volume - # path so the notebook writes through Unity Catalog and the - # bundler can emit the matching SetupTask (storage credential - # + external location + external volume). + # File-on-cloud-storage sinks: compose a UC external-volume path so the notebook writes + # through Unity Catalog and the bundler emits the matching SetupTask. sink_path_info = _resolve_path_info(sink_dataset_ref, sink_dataset_props, definitions, context) if sink_path_info is not None: sink_resolved_path = sink_path_info.uc_volume_path diff --git a/src/orchestra/translator/activity_translators/databricks_job.py b/src/flowx/translator/activity_translators/databricks_job.py similarity index 100% rename from src/orchestra/translator/activity_translators/databricks_job.py rename to src/flowx/translator/activity_translators/databricks_job.py diff --git a/src/orchestra/translator/activity_translators/delete.py b/src/flowx/translator/activity_translators/delete.py similarity index 100% rename from src/orchestra/translator/activity_translators/delete.py rename to src/flowx/translator/activity_translators/delete.py diff --git a/src/orchestra/translator/activity_translators/execute_pipeline.py b/src/flowx/translator/activity_translators/execute_pipeline.py similarity index 100% rename from src/orchestra/translator/activity_translators/execute_pipeline.py rename to src/flowx/translator/activity_translators/execute_pipeline.py diff --git a/src/orchestra/translator/activity_translators/filter.py b/src/flowx/translator/activity_translators/filter.py similarity index 82% rename from src/orchestra/translator/activity_translators/filter.py rename to src/flowx/translator/activity_translators/filter.py index 2f3a104..439cf21 100644 --- a/src/orchestra/translator/activity_translators/filter.py +++ b/src/flowx/translator/activity_translators/filter.py @@ -9,11 +9,8 @@ from flowx.models.ir import Activity, FilterActivity, TranslationContext from flowx.parser.expression_parser import resolve_expression -# The expression resolver translates ``item().X`` into -# ``dbutils.widgets.get('X')`` because ``{{input.X}}`` is the DAB ref it -# emits for ForEach-iteration item access. Inside a Filter notebook the -# items array is iterated locally with a Python ``item`` dict per -# iteration, so we rewrite each widget read to a dict lookup. +# The resolver maps item().X to dbutils.widgets.get('X') (the DAB ForEach item ref). Inside a Filter +# notebook the array is iterated locally, so each widget read is rewritten to a dict lookup. _WIDGET_ITEM_ACCESS_RE = re.compile(r"""dbutils\.widgets\.get\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\)""") @@ -34,9 +31,8 @@ def translate( else: items_expression = items_raw.get("value", "") if isinstance(items_raw, dict) else str(items_raw) - # Preserve the original ADF expression text in ``condition_expression`` - # so the notebook can show it as a documentation comment. The - # *resolved* form lives separately in ``condition_code``. + # Preserve the original ADF expression text in condition_expression for a doc comment; the + # resolved form lives in condition_code. condition_expression = condition_raw.get("value", "") if isinstance(condition_raw, dict) else str(condition_raw) condition_result = resolve_expression(condition_raw, context) diff --git a/src/orchestra/translator/activity_translators/for_each.py b/src/flowx/translator/activity_translators/for_each.py similarity index 90% rename from src/orchestra/translator/activity_translators/for_each.py rename to src/flowx/translator/activity_translators/for_each.py index f13b655..12866d1 100644 --- a/src/orchestra/translator/activity_translators/for_each.py +++ b/src/flowx/translator/activity_translators/for_each.py @@ -41,12 +41,8 @@ def translate( if expr_result is not None and expr_result.kind in ("dab_ref", "literal"): items_expression = expr_result.value elif expr_result is not None and expr_result.kind == "notebook_code": - # C-31 (CF4-001): the preparer used to construct a bare - # TranslationContext() and re-resolve the items expression on the - # JSON-reload path, but ``variable_cache`` is empty there so the - # bridge never fired and DAB rejected the raw @split(...) call. - # Capture the resolved notebook_code here while the full context - # is available; the preparer reads it from these IR fields. + # C-31 (CF4-001): capture the resolved items notebook_code here while the full context is + # available; the JSON-reload path has an empty variable_cache so it can't re-resolve @split(...). if isinstance(items_raw, dict) and items_raw.get("type") == "Expression": items_expression = items_raw.get("value", "") elif isinstance(items_raw, str): diff --git a/src/orchestra/translator/activity_translators/if_condition.py b/src/flowx/translator/activity_translators/if_condition.py similarity index 82% rename from src/orchestra/translator/activity_translators/if_condition.py rename to src/flowx/translator/activity_translators/if_condition.py index c7db2dc..9131222 100644 --- a/src/orchestra/translator/activity_translators/if_condition.py +++ b/src/flowx/translator/activity_translators/if_condition.py @@ -156,9 +156,8 @@ def _parse_condition( if adf_op == "not": inner = m.group(2).strip() resolved, bridge = _resolve_operand(inner, context) - # C-15 (CF3-003 / VAREX3-004): when the operand bridges to a - # Python bool task value, compare against 'False' (not '') so - # the IfCondition can actually evaluate to FALSE. + # C-15 (CF3-003/VAREX3-004): when the operand bridges to a Python bool task value, compare + # against 'False' (not '') so the IfCondition can evaluate to FALSE. right_operand = "False" if bridge is not None else "" return "NOT_EQUAL", resolved, right_operand, bridge @@ -167,46 +166,29 @@ def _parse_condition( right, right_bridge = _resolve_operand(args[1], context) if len(args) > 1 else ("", None) return op, left, right, merge_bridge_requests(left_bridge, right_bridge) - # Fallback: treat the whole expression as a truthy check. C-07: route - # through the bridge path when the expression is an ADF function call - # so the operand ends up as a real task-value reference rather than - # the legacy NOT_EQUAL '0' against a raw expression string. + # Fallback truthy check. C-07: route an ADF function call through the bridge path so the operand + # becomes a real task-value ref rather than legacy NOT_EQUAL '0' against a raw expression string. resolved, bridge = _resolve_operand(expr_str, context) if bridge is not None: return "NOT_EQUAL", _bridge_task_value_placeholder(), "False", bridge - # C-15 (CF3-003 / VAREX3-004): when the truthy operand resolves to a - # task-value ref backed by a SetVariable that writes a Python bool - # (e.g. a previously-cached @variables('continue') with bridge-set - # value), compare against 'False' so the legacy truthy path doesn't - # silently invert behaviour. Detected by the presence of a - # __BRIDGE__:: placeholder or the lowercase 'true'/'false' literal - # body of the upstream SetVariable. + # C-15 (CF3-003/VAREX3-004): when the truthy operand is a task-value ref backed by a Python-bool + # SetVariable (detected via a __BRIDGE__:: placeholder or 'true'/'false' body), compare against 'False'. if isinstance(resolved, str) and "__BRIDGE__" in resolved: return "NOT_EQUAL", resolved, "False", None - # C-43 (CF5-001 / LSC5-001): when the operand is a known-Boolean - # variable that resolves to a parent-job task-value ref - # (``{{tasks._init_X.values.X}}``), prefer recomputing the boolean - # locally via a BridgeRequest, mirroring the Switch path. Without this - # an inner-ForEach IfCondition references a task that lives only in the - # parent job; the bundler then blanks the operand to '' and - # NOT_EQUAL('', '0') is always TRUE, running the true branch - # unconditionally with no SETUP.md signal. The bridge keeps the - # operand local so it survives the dangling-ref safety net. + # C-43 (CF5-001/LSC5-001): for a known-Boolean variable resolving to a parent-job task-value ref, + # recompute the boolean locally via a BridgeRequest (mirroring Switch). Otherwise an inner-ForEach + # IfCondition references a parent-only task, the bundler blanks the operand, and the true branch always runs. if _operand_is_known_boolean(expr_str, context): bridge = _boolean_variable_bridge(expr_str, resolved, context) if bridge is not None: return "NOT_EQUAL", _bridge_task_value_placeholder(), "False", bridge - # C-32 (CF4-002): compare against lowercase ``'false'`` (matching - # C-21 SetVariable rendering) instead of the legacy ``'0'`` — the - # latter is always true for a Boolean-string operand so the false - # branch becomes dead code. + # C-32 (CF4-002): compare against lowercase 'false' (matching C-21 SetVariable rendering), not + # legacy '0' which is always true for a Boolean-string operand (making the false branch dead code). return "NOT_EQUAL", resolved, "false", None return "NOT_EQUAL", resolved, "0", None -def _boolean_variable_bridge( - expr: str, resolved: str, context: TranslationContext -) -> BridgeRequest | None: +def _boolean_variable_bridge(expr: str, resolved: str, context: TranslationContext) -> BridgeRequest | None: """Builds a local-recompute BridgeRequest for a Boolean-variable operand. C-43 (CF5-001): the bridge re-derives the boolean inside whatever job @@ -253,8 +235,7 @@ def _operand_is_known_boolean(expr: str, context: TranslationContext) -> bool: cached = context.get_variable_dab_ref(var_name) if isinstance(cached, str) and cached.lower() in ("true", "false"): return True - # C-41 (CF5-001): a Boolean variable seeded only by a literal - # default init task never populates variable_value_cache as a + # C-41 (CF5-001): a Boolean variable seeded only by a literal default never caches as a # dab_ref, so fall back to its declared ADF type. declared = context.get_variable_type(var_name) if isinstance(declared, str) and declared.lower() in ("boolean", "bool"): @@ -269,9 +250,7 @@ def _bridge_task_value_placeholder() -> str: return f"__BRIDGE__::{_BRIDGE_TASK_VALUE_KEY}" -def _resolve_operand( - operand: str, context: TranslationContext -) -> tuple[str, BridgeRequest | None]: +def _resolve_operand(operand: str, context: TranslationContext) -> tuple[str, BridgeRequest | None]: """Converts an ADF expression operand to a Databricks task value reference or a :class:`BridgeRequest` when the operand requires a bridge task. @@ -343,23 +322,23 @@ def _split_args(args_str: str) -> list[str]: current: list[str] = [] in_quote = False - for ch in args_str: - if ch == "'" and depth == 0: + for char in args_str: + if char == "'" and depth == 0: in_quote = not in_quote - current.append(ch) + current.append(char) elif in_quote: - current.append(ch) - elif ch == "(": + current.append(char) + elif char == "(": depth += 1 - current.append(ch) - elif ch == ")": + current.append(char) + elif char == ")": depth -= 1 - current.append(ch) - elif ch == "," and depth == 0: + current.append(char) + elif char == "," and depth == 0: parts.append("".join(current).strip()) current = [] else: - current.append(ch) + current.append(char) if current: parts.append("".join(current).strip()) diff --git a/src/orchestra/translator/activity_translators/lookup.py b/src/flowx/translator/activity_translators/lookup.py similarity index 84% rename from src/orchestra/translator/activity_translators/lookup.py rename to src/flowx/translator/activity_translators/lookup.py index 7515250..1700e4f 100644 --- a/src/orchestra/translator/activity_translators/lookup.py +++ b/src/flowx/translator/activity_translators/lookup.py @@ -10,9 +10,7 @@ from flowx.translator.activity_translators.resolve import resolve_field -def _dataset_parameter_scope( - activity: AdfActivity, context: TranslationContext -) -> dict[str, str]: +def _dataset_parameter_scope(activity: AdfActivity, context: TranslationContext) -> dict[str, str]: """Resolve the Lookup dataset reference's ``parameters`` binding. C-47 (LSC5-001): a file-source dataset's ``folderPath`` / ``fileName`` @@ -54,8 +52,7 @@ def _substitute_dataset_refs(value: Any, scope: dict[str, str]) -> Any: for name, resolved in scope.items(): # dataset().X and dataset()['X'] / dataset()["X"] forms. result = re.sub( - r"dataset\(\)\s*(?:\.\s*" + re.escape(name) + r"\b|\[\s*['\"]" - + re.escape(name) + r"['\"]\s*\])", + r"dataset\(\)\s*(?:\.\s*" + re.escape(name) + r"\b|\[\s*['\"]" + re.escape(name) + r"['\"]\s*\])", resolved, result, ) @@ -75,6 +72,7 @@ def _unwrap_expression(value: Any) -> Any: return value["value"] return value + # File-source dataset types that a Lookup can read directly. Keeping # this list local avoids tugging the broader copy translator in. _FILE_DATASET_TYPES: frozenset[str] = frozenset( @@ -112,9 +110,8 @@ def translate( first_row_only = type_properties.get("firstRowOnly", True) - # Resolve the Lookup's dataset reference (lookup-translator-ignores-dataset-reference): - # typeProperties.dataset is the canonical place for ADF; activity.inputs - # is the legacy fall-back used by the loader for flattened activity shapes. + # Resolve the Lookup's dataset reference: typeProperties.dataset is canonical; activity.inputs is the + # legacy fall-back the loader uses for flattened activity shapes. dataset_ref = _resolve_lookup_dataset(activity, definitions) if dataset_ref is not None: dataset_props = dataset_ref["properties"] @@ -123,22 +120,15 @@ def translate( location = type_props.get("location") or {} if dataset_type in _FILE_DATASET_TYPES: source_properties.setdefault("dataset_type", dataset_type) - # Stash the dataset path components so the code generator can - # build the right spark.read call. Avoid pulling in the full - # copy translator dataset-path machinery — we only need the - # raw container + folder + filename to surface to the user. - # C-37 (LSC4-001): unwrap any ADF expression dict shapes so - # downstream code can treat these as plain strings. + # Stash the dataset path components (container/folder/filename) for the code generator's + # spark.read call. C-37 (LSC4-001): unwrap any ADF expression dict shapes to plain strings. container = _unwrap_expression( location.get("container") or location.get("fileSystem") or location.get("bucketName") ) folder = _unwrap_expression(location.get("folderPath")) filename = _unwrap_expression(location.get("fileName")) - # C-47 (LSC5-001): substitute dataset().X param refs using the - # Lookup dataset reference's parameter bindings, then resolve the - # result so the path default is a real literal / interpolated - # {{job.parameters.X}} string rather than a verbatim dataset() - # expression the code generator would bake into a broken path. + # C-47 (LSC5-001): substitute dataset().X param refs from the dataset reference's bindings, then + # resolve so the path is a real literal/{{job.parameters.X}} string, not a verbatim dataset() expr. ds_scope = _dataset_parameter_scope(activity, context) if ds_scope: if isinstance(folder, str) and folder: @@ -158,9 +148,8 @@ def translate( for key in ("multiLineJson", "filePattern"): if key in format_settings: source_properties.setdefault(key, format_settings[key]) - # LSC3-005: surface the linked service URL when present so the - # generator can assemble the abfss:// path for AzureBlobFS / ADLS - # backed file datasets. + # LSC3-005: surface the linked service URL when present so the generator can assemble the + # abfss:// path for AzureBlobFS/ADLS-backed file datasets. ls_url = dataset_props.get("linked_service_url") if ls_url: source_properties.setdefault("linked_service_url", ls_url) @@ -203,9 +192,8 @@ def _resolve_lookup_dataset( if dataset is None: return None properties = dict(dataset.properties or {}) - # Thread linkedService typeProperties.url through onto the properties so - # the lookup notebook can assemble the abfss:// file path for file-source - # datasets where the URL is only known on the linked service. + # Thread linkedService typeProperties.url onto the properties so the lookup notebook can assemble the + # abfss:// file path for file-source datasets where the URL is only on the linked service. linked_service = definitions.get_linked_service(dataset.linked_service_name) if linked_service is not None: ls_props = linked_service.properties or {} diff --git a/src/orchestra/translator/activity_translators/notebook.py b/src/flowx/translator/activity_translators/notebook.py similarity index 83% rename from src/orchestra/translator/activity_translators/notebook.py rename to src/flowx/translator/activity_translators/notebook.py index 6bb10a3..74dd5bc 100644 --- a/src/orchestra/translator/activity_translators/notebook.py +++ b/src/flowx/translator/activity_translators/notebook.py @@ -30,10 +30,8 @@ def translate( """ type_properties = activity.type_properties or {} - # C-28 (NB-ITER4-001): when notebookPath is an ADF expression that lowers - # to notebook_code (e.g. @trim(json(...).notebook_path)), preserve the raw - # expression and mark the activity so the preparer emits a dispatch stub - # rather than inlining Python source as the workspace path. + # C-28 (NB-ITER4-001): when notebookPath lowers to notebook_code (e.g. @trim(json(...).notebook_path)), + # preserve the raw expression and mark the activity so the preparer emits a dispatch stub, not inline Python. notebook_path_raw = type_properties.get("notebookPath", "") notebook_path, notebook_path_unresolved, notebook_path_expression = _resolve_notebook_path_field( notebook_path_raw, context @@ -41,12 +39,8 @@ def translate( raw_params = type_properties.get("baseParameters") or {} libraries, unresolved_libraries = _resolve_libraries(type_properties.get("libraries"), context) - # Resolve base_parameters at translate time so ADF expressions like - # @variables('runTimestamp') are inlined to DAB refs while the full - # translation context (with variable_value_cache) is available. Any - # caveat notes the resolver emits (e.g. utcnow() approximations) are - # captured into parameter_approximations so the bundler can surface - # them in SETUP.md. + # Resolve base_parameters at translate time (while variable_value_cache is available) so ADF + # expressions inline to DAB refs; resolver caveats (e.g. utcnow()) go to parameter_approximations for SETUP.md. resolved_params: dict[str, Any] = {} approximations: list[dict[str, str]] = [] for key, value in raw_params.items(): @@ -122,22 +116,14 @@ def _raw_expression_text(value: Any) -> str: return str(value) -# Library entry keys that may carry ADF expressions (jar/whl paths, -# maven coordinates with @concat, etc). PyPI uses ``package`` and CRAN -# uses ``package``; we walk all of them through the resolver and only -# emit the entry when every expression resolves to a clean literal/dab_ref. +# Library entry keys that may carry ADF expressions (jar/whl paths, maven coords with @concat); each is +# walked through the resolver and only emitted when every expression resolves to a clean literal/dab_ref. _LIBRARY_VALUE_KEYS: tuple[str, ...] = ("jar", "whl", "egg", "requirements") -_GLOBAL_PARAM_REF_RE = re.compile( - r"pipeline\(\s*\)\.globalParameters\.(\w+)", re.IGNORECASE -) -_PIPELINE_PARAM_REF_RE = re.compile( - r"pipeline\(\s*\)\.parameters\.(\w+)", re.IGNORECASE -) -_VARIABLE_REF_RE = re.compile( - r"variables\(\s*'([^']+)'\s*\)", re.IGNORECASE -) +_GLOBAL_PARAM_REF_RE = re.compile(r"pipeline\(\s*\)\.globalParameters\.(\w+)", re.IGNORECASE) +_PIPELINE_PARAM_REF_RE = re.compile(r"pipeline\(\s*\)\.parameters\.(\w+)", re.IGNORECASE) +_VARIABLE_REF_RE = re.compile(r"variables\(\s*'([^']+)'\s*\)", re.IGNORECASE) def _extract_missing_identifiers(expression_text: str, context: TranslationContext) -> list[str]: @@ -194,19 +180,15 @@ def _resolve_libraries( for key, value in lib.items(): if key in _LIBRARY_VALUE_KEYS and isinstance(value, (str, dict)): result = resolve_expression(value, context) - # C-13 (NB-ITER3-004): accept both literal and dab_ref so a - # jar path like @pipeline().parameters.libName collapses to - # {{job.parameters.libName}} (symmetric with custom_tags - # resolution in _resolve_ls_parameters). + # C-13 (NB-ITER3-004): accept both literal and dab_ref so a jar path like + # @pipeline().parameters.libName collapses to {{job.parameters.libName}}. if result is not None and result.kind in ("literal", "dab_ref"): resolved_entry[key] = result.value else: expression_text = _raw_expression_text(value) resolved_entry[key] = value - # Only surface library entries whose value carried an - # ADF expression (starts with ``@``). Bare literal - # paths that already resolved successfully don't need a - # SETUP.md callout. + # Only surface library entries whose value carried an ADF expression (starts with @); + # already-resolved literal paths need no SETUP.md callout. if isinstance(expression_text, str) and expression_text.startswith("@"): unresolved.append( { diff --git a/src/orchestra/translator/activity_translators/resolve.py b/src/flowx/translator/activity_translators/resolve.py similarity index 92% rename from src/orchestra/translator/activity_translators/resolve.py rename to src/flowx/translator/activity_translators/resolve.py index 7f8d434..b2f8e77 100644 --- a/src/orchestra/translator/activity_translators/resolve.py +++ b/src/flowx/translator/activity_translators/resolve.py @@ -63,12 +63,12 @@ def merge_bridge_requests(*requests: BridgeRequest | None) -> BridgeRequest | No single bridge task with a boolean truthiness result. Returns ``None`` when no non-None requests are supplied. """ - populated = [r for r in requests if r is not None] + populated = [request for request in requests if request is not None] if not populated: return None if len(populated) == 1: return populated[0] - expression = " and ".join(f"({r.notebook_code})" for r in populated) + expression = " and ".join(f"({request.notebook_code})" for request in populated) imports: list[str] = [] required: dict[str, str] = {} for req in populated: @@ -129,16 +129,16 @@ def resolve_field_int(value: Any, context: TranslationContext, default: int = 0) return default -def resolve_dict_values(d: dict[str, Any] | None, context: TranslationContext) -> dict[str, str]: +def resolve_dict_values(fields: dict[str, Any] | None, context: TranslationContext) -> dict[str, str]: """Resolves all values in a dict that may contain ADF expressions. Args: - d: Dict of field name to raw values. + fields: Dict of field name to raw values. context: Translation context for variable resolution. Returns: Dict with all values resolved to strings. """ - if not d: + if not fields: return {} - return {k: resolve_field(v, context) for k, v in d.items()} + return {key: resolve_field(value, context) for key, value in fields.items()} diff --git a/src/orchestra/translator/activity_translators/set_variable.py b/src/flowx/translator/activity_translators/set_variable.py similarity index 79% rename from src/orchestra/translator/activity_translators/set_variable.py rename to src/flowx/translator/activity_translators/set_variable.py index 6f127ea..9e93d5f 100644 --- a/src/orchestra/translator/activity_translators/set_variable.py +++ b/src/flowx/translator/activity_translators/set_variable.py @@ -77,14 +77,9 @@ def translate( variable_name = type_properties.get("variableName", "") value_raw = type_properties.get("value", "") - # C-42 (VAREX5-001): a Set Pipeline Return Value activity carries a - # list of {key, value} pairs (e.g. - # [{'key': 'result', 'value': {'type': 'Expression', - # 'content': "@variables('executionOutputs')"}}]). The legacy path - # fails _is_adf_expression and stringifies the whole list, which the - # bundler then blanks. The inner expression is resolvable, so unwrap a - # single pair's value and route it through the normal resolution - # pipeline instead of losing the reference. + # C-42 (VAREX5-001): a Set Pipeline Return Value activity carries a list of {key, value} pairs whose + # inner value is a resolvable expression. The legacy path stringifies the whole list (then blanked by + # the bundler), so unwrap a single pair's value and route it through normal resolution. value_raw = _unwrap_return_value_pairs(value_raw) expr_result = resolve_expression(value_raw, context) @@ -98,12 +93,9 @@ def translate( notebook_imports = list(expr_result.imports) if expr_result.kind == "notebook_code" else [] required_parameters = dict(expr_result.required_parameters) elif _is_adf_expression(value_raw): - # C-33 (VAREX4-001 / CF4-003): when the value is an ADF expression - # the resolver couldn't handle (e.g. a nested function call we - # don't model), do NOT stamp value_kind='literal' with the raw - # @concat text — that ships uninterpretable Python source through - # SETUP.md. Blank the value and mark it unresolved so the bundler - # emits a manual_variable_init SetupTask the user can act on. + # C-33 (VAREX4-001/CF4-003): when the resolver can't handle the value, do NOT stamp + # value_kind='literal' with raw @concat text (uninterpretable Python). Blank it and mark + # unresolved so the bundler emits a manual_variable_init SetupTask. variable_value = "" value_kind = "unresolved" notebook_code = None @@ -115,10 +107,8 @@ def translate( elif isinstance(value_raw, str): variable_value = value_raw elif isinstance(value_raw, bool): - # VAREX3-002: render Python bool as lowercase 'true'/'false' so - # downstream ADF comparisons like @equals(variables('X'), true) - # match consistently. ``str(True)`` would emit 'True' and silently - # invert the comparison. + # VAREX3-002: render Python bool as lowercase 'true'/'false' so @equals(variables('X'), true) + # matches; str(True) would emit 'True' and silently invert the comparison. variable_value = "true" if value_raw else "false" else: variable_value = str(value_raw) @@ -137,10 +127,8 @@ def translate( raw_expression=raw_expression_text if value_kind == "unresolved" else None, ) - # Register variable -> task_key mapping in context. - # When the value is a DAB ref (e.g. {{job.start_time.iso_datetime}} from - # @utcNow()), store it so downstream @variables() calls can inline it - # instead of routing through the task value. + # Register variable -> task_key mapping in context. When the value is a DAB ref (e.g. + # {{job.start_time.iso_datetime}} from @utcNow()), store it so downstream @variables() calls inline it. dab_ref_value = variable_value if value_kind == "dab_ref" else None new_context = context.with_variable( variable_name, diff --git a/src/orchestra/translator/activity_translators/spark_jar.py b/src/flowx/translator/activity_translators/spark_jar.py similarity index 100% rename from src/orchestra/translator/activity_translators/spark_jar.py rename to src/flowx/translator/activity_translators/spark_jar.py diff --git a/src/orchestra/translator/activity_translators/spark_python.py b/src/flowx/translator/activity_translators/spark_python.py similarity index 95% rename from src/orchestra/translator/activity_translators/spark_python.py rename to src/flowx/translator/activity_translators/spark_python.py index b0b230a..87c53de 100644 --- a/src/orchestra/translator/activity_translators/spark_python.py +++ b/src/flowx/translator/activity_translators/spark_python.py @@ -57,7 +57,7 @@ def translate( raw_parameters = type_properties.get("parameters") or [] libraries = type_properties.get("libraries") - parameters = [_resolve_parameter(p, context) for p in raw_parameters] + parameters = [_resolve_parameter(parameter, context) for parameter in raw_parameters] return SparkPythonActivity( **base_kwargs, diff --git a/src/orchestra/translator/activity_translators/switch.py b/src/flowx/translator/activity_translators/switch.py similarity index 97% rename from src/orchestra/translator/activity_translators/switch.py rename to src/flowx/translator/activity_translators/switch.py index 1ce80db..470407c 100644 --- a/src/orchestra/translator/activity_translators/switch.py +++ b/src/flowx/translator/activity_translators/switch.py @@ -17,9 +17,7 @@ _BRIDGE_PLACEHOLDER = "__BRIDGE__::result" -def _resolve_on_expression( - on_expression: str, context: TranslationContext -) -> tuple[str, BridgeRequest | None]: +def _resolve_on_expression(on_expression: str, context: TranslationContext) -> tuple[str, BridgeRequest | None]: """Resolves the ``on`` expression to a DAB dynamic value ref or a bridge request. C-07 (CF-iter2-001 / CF-iter2-003): when the expression involves an diff --git a/src/orchestra/translator/activity_translators/wait.py b/src/flowx/translator/activity_translators/wait.py similarity index 100% rename from src/orchestra/translator/activity_translators/wait.py rename to src/flowx/translator/activity_translators/wait.py diff --git a/src/flowx/translator/activity_translators/web_activity.py b/src/flowx/translator/activity_translators/web_activity.py new file mode 100644 index 0000000..c1da35b --- /dev/null +++ b/src/flowx/translator/activity_translators/web_activity.py @@ -0,0 +1,162 @@ +"""Translates ADF WebActivity activities to Databricks WebActivity IR.""" + +from __future__ import annotations + +import json +from typing import Any + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions +from flowx.models.ir import Activity, TranslationContext +from flowx.models.ir import WebActivity as WebActivityIR +from flowx.parser.expression_parser import ( + resolve_expression, + resolve_interpolated_string_for_notebook, +) +from flowx.translator.activity_translators.resolve import resolve_dict_values, resolve_field + + +def translate( + activity: AdfActivity, + base_kwargs: dict[str, Any], + context: TranslationContext, + definitions: AdfDefinitions, +) -> Activity: + """Translates a WebActivity. + + Args: + activity: The ADF activity AST node. + base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). + context: Current translation context. + definitions: Full ADF definitions for cross-referencing. + + Returns: + A :class:`WebActivity` IR node. + """ + type_properties = activity.type_properties or {} + + url = resolve_field(type_properties.get("url", ""), context) + method = type_properties.get("method", "GET") + headers = resolve_dict_values(type_properties.get("headers"), context) or None + body = type_properties.get("body") + body_code, body_imports, body_required = _resolve_body_to_code(body, context) + authentication = type_properties.get("authentication") + disable_cert_validation = type_properties.get("disableCertValidation", False) + http_request_timeout = type_properties.get("httpRequestTimeout") + + timeout_seconds: int | None = None + if http_request_timeout and isinstance(http_request_timeout, str): + timeout_seconds = _parse_timeout_to_seconds(http_request_timeout) + + return WebActivityIR( + **base_kwargs, + url=url, + method=method, + body=body, + headers=headers, + authentication=authentication, + disable_cert_validation=disable_cert_validation, + http_request_timeout_seconds=timeout_seconds, + body_code=body_code, + body_imports=body_imports, + body_required_parameters=body_required, + ) + + +def _py_literal(value: Any) -> str: + """Renders a resolved literal as a Python expression.""" + if isinstance(value, str): + return json.dumps(value) + if isinstance(value, bool) or value is None: + return repr(value) + return json.dumps(value) + + +def _value_to_code(value: Any, context: TranslationContext) -> tuple[str | None, list[str], dict[str, str]]: + """Lowers a single body value to a Python expression string. + + Returns ``(code, imports, required_parameters)`` where ``code`` is + ``None`` when the value is a plain literal the code generator can render + directly (no ``@``-expression present). + """ + if isinstance(value, str): + if "@{" in value: + resolved = resolve_interpolated_string_for_notebook(value, context) + return f"f{json.dumps(resolved)}", [], {} + if value.startswith("@"): + result = resolve_expression(value, context) + if result is None: + return None, [], {} + if result.kind == "notebook_code": + return result.value, list(result.imports), dict(result.required_parameters) + if result.kind == "dab_ref": + # A bare variable/pipeline ref is read from a widget at runtime; bind the DAB ref into + # base_parameters via the returned required_parameters mapping. + widget = result.value.strip("{}").split(".")[-1] + return f"dbutils.widgets.get({json.dumps(widget)})", [], {widget: result.value} + return _py_literal(result.value), [], dict(result.required_parameters) + return None, [], {} + if isinstance(value, dict): + if value.get("type") == "Expression" and "value" in value: + return _value_to_code(value["value"], context) + # Nested dict body (e.g. {"text": {"value": "@concat(...)"}}). + parts: list[str] = [] + imports: list[str] = [] + required: dict[str, str] = {} + any_code = False + for key, inner in value.items(): + code, imps, req = _value_to_code(inner, context) + if code is None: + parts.append(f"{json.dumps(key)}: {_py_literal(inner)}") + else: + any_code = True + parts.append(f"{json.dumps(key)}: {code}") + imports.extend(imps) + required.update(req) + if not any_code: + return None, [], {} + return "{" + ", ".join(parts) + "}", imports, required + return None, [], {} + + +def _resolve_body_to_code(body: Any, context: TranslationContext) -> tuple[str | None, list[str], dict[str, str]]: + """Pre-resolves an ADF request body to Python code at translate time. + + ADF web-activity bodies frequently embed ``@concat`` / ``@variables`` / + ``@{...}`` expressions, either at the top level or nested inside a dict + (``{"text": {"value": "@concat(...)"}}``). Resolving them here -- while + the real :class:`TranslationContext` (and its variable cache) is + available -- lets the code generator emit parsed Python instead of the + raw ADF token. + + Returns ``(body_code, imports, required_parameters)``. ``body_code`` is + ``None`` for a plain-literal body (the generator renders it directly). + """ + if body is None: + return None, [], {} + return _value_to_code(body, context) + + +def _parse_timeout_to_seconds(timeout_str: str) -> int | None: + """Parses an ADF timeout string to seconds. + + Args: + timeout_str: Timeout in ``"d.hh:mm:ss"`` or ``"hh:mm:ss"`` format. + + Returns: + Total seconds, or ``None`` if the format is unrecognised. + """ + try: + parts = timeout_str.split(".") + if len(parts) == 2: + days = int(parts[0]) + time_part = parts[1] + else: + days = 0 + time_part = parts[0] + time_parts = time_part.split(":") + hours = int(time_parts[0]) if len(time_parts) > 0 else 0 + minutes = int(time_parts[1]) if len(time_parts) > 1 else 0 + seconds = int(time_parts[2]) if len(time_parts) > 2 else 0 + return days * 86400 + hours * 3600 + minutes * 60 + seconds + except (ValueError, IndexError): + return None diff --git a/src/orchestra/translator/engine.py b/src/flowx/translator/engine.py similarity index 77% rename from src/orchestra/translator/engine.py rename to src/flowx/translator/engine.py index 69981f3..b99c318 100644 --- a/src/orchestra/translator/engine.py +++ b/src/flowx/translator/engine.py @@ -126,12 +126,9 @@ def translate_pipeline( global_parameters=MappingProxyType(dict(definitions.global_parameters)), ) - # C-41 (CF5-001): seed declared variable types so the IfCondition - # fallback can recognise Boolean variables that are backed only by a - # literal default init task (and thus never populate - # variable_value_cache). Without this a `continue`-style Boolean - # condition emits NOT_EQUAL(left, '0'), always true for a - # 'true'/'false' string, making the false branch dead code. + # C-41 (CF5-001): seed declared variable types so the IfCondition fallback recognises Boolean + # variables backed only by a literal default (which never populate variable_value_cache); else the + # false branch is dead code. if pipeline.variables: default_literals: dict[str, str] = {} for name, var in pipeline.variables.items(): @@ -148,11 +145,8 @@ def translate_pipeline( gaps: list[AgenticGap] = [] warnings: list[str] = [] - # C-05 (VAREX-002): synthesise init SetVariable tasks for pipeline - # variables carrying a defaultValue. This seeds variable_cache so - # downstream @variables('X') references resolve to the init task's - # value reference instead of falling back to a self-referential - # {{tasks.X.values.X}} dangler. + # C-05 (VAREX-002): synthesise init SetVariable tasks for variables with a defaultValue, seeding + # variable_cache so @variables('X') resolves to the init task's value instead of a self-referential dangler. init_variable_activities, context = _build_variable_init_activities(pipeline, context) translated_activities: list[Activity] = list(init_variable_activities) @@ -170,32 +164,21 @@ def translate_pipeline( deterministic_count += 1 elif strategy is TranslationStrategy.AGENTIC: agentic_count += 1 - gaps.append( - AgenticGap( - activity_name=adf_activity.name, - activity_type=adf_activity.type, - recommended_skill=skill, - raw_definition=adf_activity.type_properties, - ) - ) else: unsupported_count += 1 - gaps.append( - AgenticGap( - activity_name=adf_activity.name, - activity_type=adf_activity.type, - recommended_skill=None, - raw_definition=adf_activity.type_properties, - ) - ) - warnings.append(f"Activity '{adf_activity.name}' (type={adf_activity.type}) has no translation path.") + + # Collect every agentic/unsupported activity across the whole tree (including IfCondition/ForEach/Until + # children) so each gap reaches the agent with its full ADF/ARM JSON. + gaps = _collect_agentic_gaps(pipeline.activities, warnings) parameter_entries: list[dict[str, Any]] = [] if pipeline.parameters: for param_name, param_def in pipeline.parameters.items(): entry: dict[str, Any] = {"name": param_name, "type": param_def.type} if param_def.default_value is not None: - entry["default"] = _coerce_parameter_default(param_def.default_value, param_def.type) + entry["default"] = _resolve_parameter_default( + param_def.default_value, param_def.type, context, warnings + ) parameter_entries.append(entry) schedule = _compile_pipeline_schedule(pipeline, definitions) @@ -208,17 +191,12 @@ def translate_pipeline( schedule=schedule, ) - # Whole-IR expression rewrite: catches @{...} tokens the per-activity - # translators didn't address (raw SQL WHERE clauses inside source_properties, - # REST request bodies, dataset folder paths, ...). Unresolved tokens are - # surfaced as translation warnings. + # Whole-IR expression rewrite: catches @{...} tokens the per-activity translators missed (raw SQL + # WHERE, REST bodies, dataset folder paths, ...). Unresolved tokens become translation warnings. pipeline_ir = rewrite_pipeline_expressions(pipeline_ir, warnings=warnings) - # Motif detection: scan for known multi-activity patterns. Collapsing is - # gated on the per-motif preference -- when *motif_consolidations* is - # ``None`` we preserve back-compat behaviour and collapse every detected - # motif; otherwise only motifs whose motif_id maps to ``"consolidate"`` are - # collapsed. + # Motif detection. Collapsing is gated on motif_consolidations: None preserves back-compat (collapse + # every detected motif), otherwise only motifs mapped to "consolidate" are collapsed. detected_motifs = detect_motifs(pipeline, definitions) motifs_to_collapse = _filter_motifs_for_collapse(detected_motifs, motif_consolidations) if motifs_to_collapse: @@ -249,6 +227,9 @@ def translate_pipeline( ) +_OPT_IN_ONLY_MOTIFS: frozenset[str] = frozenset({"activity_and_notify"}) + + def _filter_motifs_for_collapse( detected_motifs: list, motif_consolidations: dict[str, str] | None, @@ -266,9 +247,45 @@ def _filter_motifs_for_collapse( The subset of motifs to pass to :func:`flowx.motifs.collapser.collapse_motifs`. """ + # activity_and_notify is destructive (drops the notify activities for job-task notifications), so it's + # never collapsed implicitly -- only when the user opts into a notification destination. if motif_consolidations is None: - return list(detected_motifs) - return [m for m in detected_motifs if motif_consolidations.get(m.definition.motif_id) == "consolidate"] + return [motif for motif in detected_motifs if motif.definition.motif_id not in _OPT_IN_ONLY_MOTIFS] + return [motif for motif in detected_motifs if motif_consolidations.get(motif.definition.motif_id) == "consolidate"] + + +def _collect_agentic_gaps(activities: list[AdfActivity], warnings: list[str]) -> list[AgenticGap]: + """Walk the activity tree and emit a gap for every agentic / unsupported activity. + + Recurses into IfCondition branches and ForEach / Until container children so + that nested activities (e.g. an ``Until`` inside an ``IfCondition``) are not + lost. Each gap carries the activity's full ADF/ARM JSON (``raw``) so the + agentic handler can translate directly from source. + """ + gaps: list[AgenticGap] = [] + seen: set[str] = set() + + def _walk(acts: list[AdfActivity] | None) -> None: + for act in acts or []: + strategy, skill = classify_activity(act.type) + if strategy is not TranslationStrategy.DETERMINISTIC and act.name not in seen: + seen.add(act.name) + gaps.append( + AgenticGap( + activity_name=act.name, + activity_type=act.type, + recommended_skill=skill, + raw_definition=act.raw if act.raw is not None else act.type_properties, + ) + ) + if strategy is TranslationStrategy.UNSUPPORTED: + warnings.append(f"Activity '{act.name}' (type={act.type}) has no translation path.") + _walk(act.if_true_activities) + _walk(act.if_false_activities) + _walk(act.activities) + + _walk(activities) + return gaps def _dispatch_activity( @@ -355,6 +372,8 @@ def _dispatch_activity( **base_kwargs, original_type=activity.type, comment=reason, + agentic_skill=skill, + raw_definition=activity.raw, ) context = context.with_activity(activity.name, placeholder) return placeholder, context @@ -385,10 +404,8 @@ def _translate_activity_list( return results, context -# C-10 (SCHED-001): map Windows timezone names ADF emits onto IANA names -# the Databricks DAB ``schedule.timezone_id`` field expects. Only the -# ones observed in the corpus are mapped explicitly; anything else passes -# through unchanged (Databricks accepts any IANA zone). +# C-10 (SCHED-001): map the Windows timezone names ADF emits onto the IANA names DAB's +# schedule.timezone_id expects. Only corpus-observed ones are mapped; anything else passes through. _ADF_TIMEZONE_TO_IANA: dict[str, str] = { "UTC": "UTC", "Coordinated Universal Time": "UTC", @@ -433,20 +450,17 @@ def _compile_pipeline_schedule( """ triggers = getattr(definitions, "triggers", None) or [] pipeline_name = pipeline.name - matching_triggers = [t for t in triggers if _trigger_references(t, pipeline_name)] + matching_triggers = [trigger for trigger in triggers if _trigger_references(trigger, pipeline_name)] if not matching_triggers: return None - # First matching trigger wins -- ADF allows multiple triggers per - # pipeline but DAB schedules are 1:1. Subsequent triggers can be - # surfaced via SETUP.md by downstream tooling. + # First matching trigger wins -- ADF allows multiple triggers per pipeline but DAB schedules are 1:1; + # downstream tooling can surface the rest via SETUP.md. trigger = matching_triggers[0] spec = _adf_trigger_to_schedule(trigger) if spec is not None: - # SCHED3-003: pull per-pipeline parameter overrides off the - # matching pipelineReference so trigger-injected params (e.g. - # ``{applicationName: 'app0001', negocio: 'GLP'}``) propagate to - # the job's default parameter values. + # SCHED3-003: pull per-pipeline parameter overrides off the matching pipelineReference so + # trigger-injected params propagate to the job's default parameter values. overrides = _extract_trigger_parameter_overrides(trigger, pipeline_name) if overrides: spec["parameter_overrides"] = overrides @@ -465,9 +479,7 @@ def _trigger_references(trigger: Any, pipeline_name: str) -> bool: return False -def _extract_trigger_parameter_overrides( - trigger: Any, pipeline_name: str -) -> dict[str, Any]: +def _extract_trigger_parameter_overrides(trigger: Any, pipeline_name: str) -> dict[str, Any]: """Returns the parameters block on the trigger's pipelineReference entry. SCHED3-003: ADF triggers attach per-pipeline parameter overrides at the @@ -501,9 +513,8 @@ def _adf_trigger_to_schedule(trigger: Any) -> dict[str, Any] | None: trigger_type = trigger.type if trigger_type == "ScheduleTrigger": recurrence = type_properties.get("recurrence") or {} - # SCHED3-002: Day/Week/Month with interval > 1 cannot be represented - # in quartz cron without enumerating every Nth occurrence; use the - # trigger.periodic primitive so it ships correctly. + # SCHED3-002: Day/Week/Month with interval > 1 can't be expressed in quartz cron without + # enumerating every Nth occurrence; use the trigger.periodic primitive instead. periodic = _recurrence_to_periodic(recurrence) if periodic is not None: spec: dict[str, Any] = { @@ -517,10 +528,8 @@ def _adf_trigger_to_schedule(trigger: Any) -> dict[str, Any] | None: if "time_of_day_note" in periodic: spec["time_of_day_note"] = periodic["time_of_day_note"] return spec - # C-45 (SCHED5-002): an interval > 1 Month recurrence has no - # monthly-cron-expressible form (cron fires every month, ignoring the - # interval) and the DAB periodic enum has no MONTHS unit, so surface a - # manual setup note instead of silently emitting a monthly cron. + # C-45 (SCHED5-002): interval > 1 Month has no monthly-cron form (cron ignores the interval) and + # the DAB periodic enum has no MONTHS unit, so surface a manual setup note instead of a wrong cron. if _is_multi_month_recurrence(recurrence): return { "kind": "manual_setup", @@ -600,12 +609,8 @@ def _recurrence_to_periodic(recurrence: dict[str, Any]) -> dict[str, Any] | None interval = int(interval) if not isinstance(interval, int) or interval <= 1: return None - # C-45 (SCHED5-002): the DAB PeriodicTriggerConfigurationTimeUnit enum - # only defines DAYS / HOURS / WEEKS — emitting MONTHS makes bundle - # validate/deploy reject the trigger. Month frequencies are routed to - # the quartz cron path (monthDays) instead; an interval > 1 Month, which - # is not monthly-cron-expressible, is surfaced as a setup note by the - # caller. + # C-45 (SCHED5-002): the DAB periodic enum only has DAYS/HOURS/WEEKS, so month frequencies route to the + # quartz cron path (monthDays); an interval > 1 Month (not monthly-cron-expressible) is a setup note. unit_map = {"Day": "DAYS", "Week": "WEEKS"} unit = unit_map.get(frequency or "") if unit is None: @@ -614,9 +619,7 @@ def _recurrence_to_periodic(recurrence: dict[str, Any]) -> dict[str, Any] | None schedule = recurrence.get("schedule") or {} if isinstance(schedule, dict): time_of_day = { - key: schedule.get(key) - for key in ("hours", "minutes", "weekDays", "monthDays") - if schedule.get(key) + key: schedule.get(key) for key in ("hours", "minutes", "weekDays", "monthDays") if schedule.get(key) } if time_of_day: spec["time_of_day_note"] = time_of_day @@ -653,12 +656,8 @@ def _recurrence_to_quartz_cron(recurrence: dict[str, Any]) -> str | None: week_days = schedule.get("weekDays") or [] month_days = schedule.get("monthDays") or [] - # C-44 (SCHED5-001): when the schedule block carries no explicit - # time-of-day, ADF defaults it to the first-execution time derived from - # ``startTime``. Reading only ``schedule.minutes/hours`` (falling back - # to '0'/'0') silently shifts a ``startTime`` of 21:00 to midnight. - # Derive the hour/minute from ``startTime`` so the cron fires at the - # ADF-intended time. + # C-44 (SCHED5-001): when the schedule has no explicit time-of-day, derive hour/minute from startTime + # (ADF's default); reading only schedule.minutes/hours would shift a 21:00 startTime to midnight. start_hour, start_minute = _start_time_hour_minute(recurrence.get("startTime")) minute_default = str(start_minute) if start_minute is not None else "0" hour_default = str(start_hour) if start_hour is not None else "0" @@ -679,7 +678,7 @@ def _recurrence_to_quartz_cron(recurrence: dict[str, Any]) -> str | None: if frequency == "Day": return f"0 {minute_field} {hour_field} * * ?" if frequency == "Week": - days = ",".join(_DAYS_OF_WEEK_MAP.get(d, d) for d in week_days) or "MON" + days = ",".join(_DAYS_OF_WEEK_MAP.get(day, day) for day in week_days) or "MON" return f"0 {minute_field} {hour_field} ? * {days}" if frequency == "Month": dom_field = _list_or_default(month_days, "1") @@ -694,7 +693,7 @@ def _list_or_default(value: Any, default: str) -> str: if isinstance(value, list): if not value: return default - return ",".join(str(v) for v in value) + return ",".join(str(item) for item in value) return str(value) @@ -764,10 +763,8 @@ def _build_variable_init_activities( notebook_imports = list(expr_result.imports) if expr_result.kind == "notebook_code" else [] required_parameters = dict(expr_result.required_parameters) else: - # VAREX3-002: Boolean defaults must render lowercase ('true'/'false') - # so downstream ``@equals(variables('continue'), true)`` evaluates - # consistently with ADF semantics. Python ``str(True)`` would - # produce title-case 'True' and silently invert the comparison. + # VAREX3-002: Boolean defaults render lowercase 'true'/'false' so @equals(variables('continue'), + # true) matches ADF; Python str(True) would emit 'True' and silently invert the comparison. if isinstance(default, bool): variable_value = "true" if default else "false" else: @@ -794,9 +791,8 @@ def _build_variable_init_activities( required_parameters=required_parameters, ) init_tasks.append(init_activity) - # Register the synthesised setter so @variables('X') resolves to - # {{tasks._init_X.values.X}}. When the value is itself a DAB ref - # (e.g. from @utcNow()), inline it directly per existing semantics. + # Register the synthesised setter so @variables('X') resolves to {{tasks._init_X.values.X}}; when + # the value is itself a DAB ref (e.g. @utcNow()), inline it directly. dab_ref_value = variable_value if value_kind == "dab_ref" else None context = context.with_variable(var_name, task_key, dab_ref_value=dab_ref_value) context = context.with_activity(init_activity.name, init_activity) @@ -965,7 +961,7 @@ def _map_dependency_conditions(conditions: list[str] | None) -> str | None: """ if not conditions: return None - normalized = [c for c in conditions if c] + normalized = [condition for condition in conditions if condition] if not normalized: return None if len(normalized) == 1: @@ -1032,14 +1028,12 @@ def _resolve_ls_parameters( if isinstance(activity_supplied, dict): for pname, pval in activity_supplied.items(): raw = _unwrap_expression_value(pval) - # C-03: route @-prefixed activity-supplied values through the - # expression parser so @pipeline().globalParameters.X collapses - # to the factory value when one is set. + # C-03: route @-prefixed activity-supplied values through the expression parser so + # @pipeline().globalParameters.X collapses to the factory value when one is set. if context is not None and isinstance(raw, str) and raw.startswith("@"): result = resolve_expression(raw, context) - # C-13 (NB-ITER3-002 / LSC3-003 / VAREX3-006): accept both - # literal and dab_ref so @pipeline().parameters.X collapses - # to {{job.parameters.X}} (valid in custom_tags map values). + # C-13 (NB-ITER3-002/LSC3-003/VAREX3-006): accept both literal and dab_ref so + # @pipeline().parameters.X collapses to {{job.parameters.X}} (valid in custom_tags values). if result is not None and result.kind in ("literal", "dab_ref"): raw = result.value resolved[pname] = raw @@ -1059,15 +1053,13 @@ def _unwrap_expression_value(value: Any) -> Any: # Bare {"value": X, "type": "Expression"} -- collapse to inner X. if "value" in value and value.get("type") == "Expression": return _unwrap_expression_value(value["value"]) - # Some payloads omit the explicit type marker but follow the same - # single-key shape. Conservatively unwrap only when the dict has - # the exact two keys {"value", "type"} so we don't corrupt regular - # nested config blocks like {"workspace": {"destination": ...}}. + # Some payloads omit the type marker but share the shape; conservatively unwrap only when the dict + # has exactly {value, type} so we don't corrupt nested config like {"workspace": {"destination": ...}}. if set(value.keys()) == {"value", "type"}: return _unwrap_expression_value(value["value"]) return {k: _unwrap_expression_value(v) for k, v in value.items()} if isinstance(value, list): - return [_unwrap_expression_value(v) for v in value] + return [_unwrap_expression_value(item) for item in value] return value @@ -1102,7 +1094,7 @@ def _sub(match: re.Match[str]) -> str: if isinstance(value, dict): return {k: _substitute_ls_params(v, params) for k, v in value.items()} if isinstance(value, list): - return [_substitute_ls_params(v, params) for v in value] + return [_substitute_ls_params(item, params) for item in value] return value @@ -1118,6 +1110,45 @@ def _coerce_int(value: Any) -> Any: return value +def _resolve_parameter_default( + value: Any, + declared_type: str, + context: TranslationContext, + warnings: list[str], +) -> Any: + """Resolve a pipeline parameter default for emission as a job parameter. + + ADF parameter defaults may themselves be ``@``-expressions -- most + commonly ``@utcNow('yyyy-MM-dd')``. Those must be lowered to a DAB + dynamic value reference (e.g. ``{{job.start_time.iso_date}}``) so the + generated job-parameter default is valid Databricks YAML rather than a + raw ADF token. Non-expression defaults fall through to type coercion. + + Args: + value: The raw ADF default value. + declared_type: The ADF parameter type (``String`` / ``Int`` / ...). + context: Current translation context (for variable/expression refs). + warnings: Mutable warning list; an entry is appended when an + ``@``-expression default cannot be lowered deterministically. + + Returns: + The resolved default -- a DAB ref / literal for ``@``-expressions, + otherwise the type-coerced value. + """ + if isinstance(value, str) and value.strip().startswith("@"): + from flowx.parser.expression_parser import resolve_expression + + result = resolve_expression(value, context) + if result is not None and result.kind in ("dab_ref", "literal"): + return result.value + warnings.append( + f"Pipeline parameter default '{value}' could not be lowered to a DAB value " + "reference; emitted as-is. Set it explicitly at deploy time if needed." + ) + return value + return _coerce_parameter_default(value, declared_type) + + def _coerce_parameter_default(value: Any, declared_type: str) -> Any: """Coerce an ADF parameter default into a Python type matching its declared type. @@ -1170,10 +1201,8 @@ def _extract_cluster_config( if overrides: ls_properties = _substitute_ls_params(ls_properties, overrides) - # C-02 (NB-ITER2-2 / LSC2-003): unwrap any {value, type:'Expression'} - # dicts that survived the substitution pass. Map fields like - # custom_tags and spark_env_vars must be plain Map[String, String] for - # Databricks to accept the cluster YAML. + # C-02 (NB-ITER2-2/LSC2-003): unwrap any {value, type:'Expression'} dicts that survived substitution - + # map fields like custom_tags / spark_env_vars must be plain Map[String, String] for the cluster YAML. ls_properties = _unwrap_expression_value(ls_properties) nested = ls_properties.get("typeProperties") or {} @@ -1216,10 +1245,8 @@ def _extract_cluster_config( if cluster_log_conf: config["cluster_log_conf"] = cluster_log_conf - # C-39 (LSC4-004): capture the ADF authentication shape (e.g. "MSI" or - # any CredentialReference) so the bundler can emit a manual_credential - # SetupTask warning that ``single_user_name`` was rewritten to the - # deploying user. + # C-39 (LSC4-004): capture the ADF auth shape (e.g. MSI / a CredentialReference) so the bundler emits + # a manual_credential SetupTask noting single_user_name was rewritten to the deploying user. authentication = fields.get("authentication") if authentication: config["_adf_authentication"] = authentication @@ -1246,30 +1273,30 @@ def _pipeline_to_dict(pipeline: Pipeline) -> dict[str, Any]: "tags": pipeline.tags, "tasks": [_activity_to_dict(task) for task in pipeline.tasks], } - if pipeline.translation_preferences is not None: - result["translation_preferences"] = _preferences_to_dict(pipeline.translation_preferences) + if pipeline.translation_configuration is not None: + result["translation_configuration"] = _configuration_to_dict(pipeline.translation_configuration) return result -def _preferences_to_dict(preferences: Any) -> dict[str, Any]: - """Serialise a TranslationPreferences instance to a JSON-friendly dictionary. +def _configuration_to_dict(configuration: Any) -> dict[str, Any]: + """Serialise a TranslationConfiguration instance to a JSON-friendly dictionary. Args: - preferences: The :class:`TranslationPreferences` snapshot to serialise. + configuration: The :class:`TranslationConfiguration` snapshot to serialise. Returns: Dictionary with each StrEnum field rendered as its string value and per-task overrides preserved verbatim. """ return { - "copy_activity_paradigm": str(preferences.copy_activity_paradigm), - "non_databricks_task_compute": str(preferences.non_databricks_task_compute), - "use_lakeflow_connectors": str(preferences.use_lakeflow_connectors), - "lakeflow_connector_type": str(preferences.lakeflow_connector_type), + "copy_activity_paradigm": str(configuration.copy_activity_paradigm), + "non_databricks_task_compute": str(configuration.non_databricks_task_compute), + "use_lakeflow_connectors": str(configuration.use_lakeflow_connectors), + "lakeflow_connector_type": str(configuration.lakeflow_connector_type), "motif_consolidations": { - motif_id: str(choice) for motif_id, choice in preferences.motif_consolidations.items() + motif_id: str(choice) for motif_id, choice in configuration.motif_consolidations.items() }, - "per_task": dict(preferences.per_task), + "per_task": dict(configuration.per_task), } @@ -1305,6 +1332,8 @@ def _activity_to_dict(task: Activity) -> dict[str, Any]: task_dict["existing_cluster_id"] = task.existing_cluster_id if task.compute_mode: task_dict["compute_mode"] = task.compute_mode + if task.notifications: + task_dict["notifications"] = task.notifications if task.libraries: task_dict["libraries"] = task.libraries if task.parameter_approximations: @@ -1448,6 +1477,16 @@ def _activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["headers"] = activity.headers if activity.authentication: extra["authentication"] = activity.authentication + if activity.body_code is not None: + extra["body_code"] = activity.body_code + if activity.body_imports: + extra["body_imports"] = activity.body_imports + if activity.body_required_parameters: + extra["body_required_parameters"] = activity.body_required_parameters + if activity.disable_cert_validation: + extra["disable_cert_validation"] = activity.disable_cert_validation + if activity.http_request_timeout_seconds: + extra["http_request_timeout_seconds"] = activity.http_request_timeout_seconds case DeleteActivity(): extra["dataset_name"] = activity.dataset_name if activity.folder_path: @@ -1560,14 +1599,104 @@ def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: } -if __name__ == "__main__": +def _find_and_replace_task(tasks: list[dict[str, Any]], activity_name: str, replacement: dict[str, Any]) -> bool: + """Replace the task named *activity_name* with *replacement*, recursing into containers. + + Searches top-level tasks and the nested activity lists of IfCondition / + ForEach / Switch containers. Preserves the placeholder's ``task_key`` and + ``depends_on`` when the replacement omits them so downstream dependency + edges stay intact. Returns True when a match was replaced. + """ + nested_keys = ("inner_activities", "if_true_activities", "if_false_activities", "default_activities") + for index, task in enumerate(tasks): + if task.get("name") == activity_name: + replacement.setdefault("task_key", task.get("task_key")) + replacement.setdefault("name", activity_name) + if "depends_on" not in replacement and task.get("depends_on"): + replacement["depends_on"] = task["depends_on"] + tasks[index] = replacement + return True + for key in nested_keys: + child = task.get(key) + if isinstance(child, list) and _find_and_replace_task(child, activity_name, replacement): + return True + for case in task.get("cases") or []: + if isinstance(case, dict) and isinstance(case.get("activities"), list): + if _find_and_replace_task(case["activities"], activity_name, replacement): + return True + return False + + +def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Path | None = None) -> tuple[int, int]: + """Merge agent-produced per-activity translations into a translation report. + + Each ``*.json`` file in *results_dir* describes one resolved agentic gap:: + + { + "activity_name": "", # required + "pipeline": "", # optional; for multi-pipeline reports + "task": { ...IR task dict... } # required; replacement task + } + + The matching placeholder task (located by ``name``, recursing into + IfCondition / ForEach / Switch containers) is replaced by ``task``. Use a + ``NotebookActivity`` whose ``notebook_path`` points at a notebook the agent + wrote to the workspace; the prepare phase then references it directly. + + Args: + report_path: ``translation_report.json`` produced by the translate phase. + results_dir: Directory of per-activity result JSON files. + output_path: Where to write the merged report; defaults to overwriting + *report_path*. + + Returns: + ``(merged, unmatched)`` counts. + """ + report = json.loads(report_path.read_text(encoding="utf-8")) + pipelines = report["pipelines"] if isinstance(report, dict) and "pipelines" in report else [report] + + merged = 0 + unmatched = 0 + for result_file in sorted(results_dir.glob("*.json")): + data = json.loads(result_file.read_text(encoding="utf-8")) + activity_name = data.get("activity_name") or data.get("activity") + task = data.get("task") or data.get("ir") + if not activity_name or not isinstance(task, dict): + logger.warning("Skipping %s: missing 'activity_name' or 'task'.", result_file.name) + unmatched += 1 + continue + wanted = data.get("pipeline") + candidates = [pipeline for pipeline in pipelines if not wanted or pipeline.get("name") == wanted] + if any(_find_and_replace_task(pipeline.get("tasks", []), activity_name, dict(task)) for pipeline in candidates): + merged += 1 + logger.info("Merged agentic result for '%s' from %s", activity_name, result_file.name) + else: + logger.warning("No placeholder named '%s' found for %s", activity_name, result_file.name) + unmatched += 1 + + destination = output_path or report_path + destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") + logger.info("Wrote merged report to %s (%d merged, %d unmatched)", destination, merged, unmatched) + return merged, unmatched + + +def main(argv: list[str] | None = None) -> int: + """Convert-phase entry point: translate ADF pipelines to IR (or merge agentic results). + + Exposed as a callable so the adapter can run the phase in-process instead of spawning a + second interpreter. + """ parser = argparse.ArgumentParser(description="Translate ADF pipelines to Databricks IR.") - parser.add_argument("--source-dir", required=True, type=Path, help="Root directory containing ADF JSON exports.") + parser.add_argument("--source-dir", required=False, type=Path, help="Root directory containing ADF JSON exports.") parser.add_argument( "--output-dir", type=Path, - default=Path("./orchestra_output/translate"), - help="Directory to write translation results into.", + default=Path("./flowx_output"), + help=( + "Migration output directory. The translation report and other " + "intermediate IR are written to its transient .work/ subfolder " + "(consumed by the adapter/prepare phases; pruned by prepare)." + ), ) parser.add_argument( "--pipeline", @@ -1580,20 +1709,60 @@ def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: action="store_true", help="Write a full debug IR dump alongside the normal output.", ) - args = parser.parse_args() + parser.add_argument( + "--merge-agentic", + action="store_true", + help="Merge agent-produced results from --agentic-results into --report instead of translating.", + ) + parser.add_argument( + "--report", + type=Path, + default=None, + help="Translation report to merge agentic results into (with --merge-agentic).", + ) + parser.add_argument( + "--agentic-results", + type=Path, + default=None, + help="Directory of per-activity agentic result JSON files (with --merge-agentic).", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Where to write the merged report (default: overwrite --report).", + ) + args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + if args.merge_agentic: + if not args.report or not args.agentic_results: + parser.error("--merge-agentic requires --report and --agentic-results") + merged_count, unmatched_count = merge_agentic_results(args.report, args.agentic_results, args.output) + print("\nAgentic Merge Summary") + print("=====================") + print(f"Merged: {merged_count}") + print(f"Unmatched: {unmatched_count}") + return 0 if unmatched_count == 0 else 1 + + if not args.source_dir: + parser.error("--source-dir is required (unless using --merge-agentic)") + definitions = load_adf_definitions(args.source_dir) logger.info("Loaded %d pipeline(s) from %s", len(definitions.pipelines), args.source_dir) output_dir: Path = args.output_dir.resolve() - output_dir.mkdir(parents=True, exist_ok=True) + # Translation IR is intermediate: write it to a transient .work/ subfolder; the prepare phase consumes + # the report from there and prunes .work/, leaving metadata/ curated. + work_dir = output_dir / ".work" + work_dir.mkdir(parents=True, exist_ok=True) total_deterministic = 0 total_agentic = 0 total_unsupported = 0 all_gaps: list[dict[str, Any]] = [] + all_pipeline_dicts: list[dict[str, Any]] = [] for pipeline in definitions.pipelines: if args.pipeline and pipeline.name != args.pipeline: @@ -1604,14 +1773,15 @@ def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: total_agentic += report.agentic_count total_unsupported += report.unsupported_count - pipeline_file = output_dir / f"{_sanitize_task_key(pipeline.name)}.json" + pipeline_file = work_dir / f"{_sanitize_task_key(pipeline.name)}.json" pipeline_dict = _pipeline_to_dict(report.pipeline) pipeline_file.write_text(json.dumps(pipeline_dict, indent=2, default=str), encoding="utf-8") logger.info("Wrote pipeline IR to %s", pipeline_file) + all_pipeline_dicts.append(pipeline_dict) # Write debug IR if requested if args.debug: - debug_file = output_dir / f"{_sanitize_task_key(pipeline.name)}.debug.json" + debug_file = work_dir / f"{_sanitize_task_key(pipeline.name)}.debug.json" debug_dict = _pipeline_to_debug_dict(report.pipeline) debug_file.write_text(json.dumps(debug_dict, indent=2, default=str), encoding="utf-8") logger.info("Wrote debug IR to %s", debug_file) @@ -1623,8 +1793,18 @@ def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: for warning in report.warnings: logger.warning(warning) + # Write canonical translation_report.json so downstream tools (inspect, workspace-paths, dab_writer) + # can reference a well-known filename regardless of --pipeline. + report_file = work_dir / "translation_report.json" + if len(all_pipeline_dicts) == 1: + report_payload = all_pipeline_dicts[0] + else: + report_payload = {"pipelines": all_pipeline_dicts} + report_file.write_text(json.dumps(report_payload, indent=2, default=str), encoding="utf-8") + logger.info("Wrote translation_report.json to %s", report_file) + if all_gaps: - gaps_file = output_dir / "gaps.json" + gaps_file = work_dir / "gaps.json" gaps_file.write_text(json.dumps(all_gaps, indent=2, default=str), encoding="utf-8") logger.info("Wrote %d gap(s) to %s", len(all_gaps), gaps_file) @@ -1635,3 +1815,9 @@ def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: print(f"Agentic: {total_agentic}") print(f"Unsupported: {total_unsupported}") print(f"Total: {total}") + print(f"\nTranslation report (intermediate): {report_file}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/orchestra/translator/query_analysis.py b/src/flowx/translator/query_analysis.py similarity index 100% rename from src/orchestra/translator/query_analysis.py rename to src/flowx/translator/query_analysis.py diff --git a/src/orchestra/utils.py b/src/flowx/utils.py similarity index 94% rename from src/orchestra/utils.py rename to src/flowx/utils.py index fbbd882..7fa6c61 100644 --- a/src/orchestra/utils.py +++ b/src/flowx/utils.py @@ -7,9 +7,7 @@ from flowx.models.adf_ast import AdfPolicy -# --------------------------------------------------------------------------- -# Default ADF timeout (12 hours) used when a timeout string cannot be parsed. -# --------------------------------------------------------------------------- +# Default ADF timeout (12 hours), used when a timeout string cannot be parsed. DEFAULT_TIMEOUT_SECONDS = 43_200 # --------------------------------------------------------------------------- diff --git a/src/flowx/validate/__init__.py b/src/flowx/validate/__init__.py new file mode 100644 index 0000000..a6544ce --- /dev/null +++ b/src/flowx/validate/__init__.py @@ -0,0 +1,27 @@ +"""Tier-0 static validation: motif-aware DAG equivalence between ADF and IR.""" + +from __future__ import annotations + +from flowx.validate.bundle_invariants import ( + BundleFinding, + BundleInvariantResult, + check_bundle_dir, + check_job, +) +from flowx.validate.dag_equivalence import ( + DagEquivalenceResult, + DagFinding, + check_dag_equivalence, + format_result, +) + +__all__ = [ + "DagEquivalenceResult", + "DagFinding", + "check_dag_equivalence", + "format_result", + "BundleFinding", + "BundleInvariantResult", + "check_bundle_dir", + "check_job", +] diff --git a/src/flowx/validate/bundle_invariants.py b/src/flowx/validate/bundle_invariants.py new file mode 100644 index 0000000..c822802 --- /dev/null +++ b/src/flowx/validate/bundle_invariants.py @@ -0,0 +1,181 @@ +"""Structural-invariant checks for a generated Databricks Asset Bundle. + +These guard against output that is valid YAML / valid Python but invalid as a +Databricks job -- e.g. a job parameter declared twice (the duplicate-``region`` +regression), a duplicate task key, a ``{{job.parameters.X}}`` reference to an +undeclared parameter, a ``depends_on`` edge to a missing task, or a leaked YAML +anchor/alias (the fingerprint of a shared mutable object reaching serialization). + +Run :func:`check_bundle_dir` over a generated bundle in tests (and optionally as +a Tier-0 prepare step) so these never ship silently. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +# PyYAML emits anchors/aliases as ``&id001`` / ``*id001`` when the same object +# appears more than once in the tree. flowx never intends to emit these. +_ANCHOR_RE = re.compile(r"[&*]id\d+\b") +_JOB_PARAM_REF_RE = re.compile(r"\{\{\s*job\.parameters\.([A-Za-z0-9_]+)\s*\}\}") + + +@dataclass(slots=True, kw_only=True) +class BundleFinding: + """A single invariant violation. + + Attributes: + code: Stable machine-readable identifier. + message: Human-readable explanation. + location: File / job / task the finding concerns. + """ + + code: str + message: str + location: str = "" + severity: str = "violation" + + +@dataclass(slots=True, kw_only=True) +class BundleInvariantResult: + """Outcome of :func:`check_bundle_dir` / :func:`check_job`.""" + + findings: list[BundleFinding] = field(default_factory=list) + + @property + def violations(self) -> list[BundleFinding]: + """Hard, always-invalid findings (these fail a bundle).""" + return [finding for finding in self.findings if finding.severity == "violation"] + + @property + def warnings(self) -> list[BundleFinding]: + """Soft findings worth surfacing but not build-failing.""" + return [finding for finding in self.findings if finding.severity == "warning"] + + @property + def ok(self) -> bool: + """True when no hard invariant was violated (warnings are allowed).""" + return not self.violations + + +def _collect_task_keys(tasks: list[dict[str, Any]]) -> list[str]: + """Top-level task keys plus any single nested ``for_each_task.task`` key.""" + keys: list[str] = [] + for task in tasks or []: + if "task_key" in task: + keys.append(task["task_key"]) + nested = (task.get("for_each_task") or {}).get("task") + if isinstance(nested, dict) and "task_key" in nested: + keys.append(nested["task_key"]) + return keys + + +def _dump(obj: Any) -> str: + """Serialise a structure to a string for reference scanning.""" + return yaml.safe_dump(obj, default_flow_style=False) + + +def check_job(job_key: str, job: dict[str, Any]) -> list[BundleFinding]: + """Check the structural invariants of a single job resource dict.""" + findings: list[BundleFinding] = [] + where = f"job '{job_key}'" + + # 1. No duplicate job-parameter names. + param_names = [param.get("name") for param in (job.get("parameters") or []) if isinstance(param, dict)] + duplicate_params = sorted({name for name in param_names if name is not None and param_names.count(name) > 1}) + for name in duplicate_params: + findings.append( + BundleFinding( + code="duplicate_job_parameter", + location=where, + message=f"Job parameter '{name}' is declared more than once.", + ) + ) + + # 2. No duplicate task keys. + task_keys = _collect_task_keys(job.get("tasks") or []) + duplicate_keys = sorted({key for key in task_keys if task_keys.count(key) > 1}) + for key in duplicate_keys: + findings.append( + BundleFinding( + code="duplicate_task_key", location=where, message=f"Task key '{key}' is used more than once." + ) + ) + + # 3. Every {{job.parameters.X}} reference is declared. + declared = {name for name in param_names if name is not None} + referenced = set(_JOB_PARAM_REF_RE.findall(_dump(job))) + for name in sorted(referenced - declared): + findings.append( + BundleFinding( + code="undeclared_job_parameter", + severity="warning", + location=where, + message=f"'{{{{job.parameters.{name}}}}}' is referenced but '{name}' is not a declared job parameter.", + ) + ) + + # 4. Every top-level depends_on target exists. + top_level_keys = {task.get("task_key") for task in (job.get("tasks") or []) if isinstance(task, dict)} + for task in job.get("tasks") or []: + for dep in task.get("depends_on") or []: + target = dep.get("task_key") + if target and target not in top_level_keys: + findings.append( + BundleFinding( + code="dangling_depends_on", + location=f"{where}, task '{task.get('task_key')}'", + message=f"depends_on references unknown task '{target}'.", + ) + ) + return findings + + +def check_resource_text(text: str, *, filename: str = "") -> list[BundleFinding]: + """Check one resource YAML document (raw text): anchors + per-job invariants.""" + findings: list[BundleFinding] = [] + if _ANCHOR_RE.search(text): + findings.append( + BundleFinding( + code="yaml_anchor", + location=filename, + message=( + "Emitted YAML contains an anchor/alias (&idN/*idN); a shared mutable object " + "leaked into the bundle structure. This usually means a value was added twice." + ), + ) + ) + doc = yaml.safe_load(text) or {} + jobs = ((doc.get("resources") or {}).get("jobs") or {}) if isinstance(doc, dict) else {} + for job_key, job in jobs.items(): + if isinstance(job, dict): + findings.extend(check_job(job_key, job)) + return findings + + +def check_bundle_dir(bundle_dir: Path) -> BundleInvariantResult: + """Run all structural invariants over every resource YAML in a bundle directory.""" + bundle_dir = Path(bundle_dir) + findings: list[BundleFinding] = [] + resources_dir = bundle_dir / "resources" + yaml_files = sorted(resources_dir.glob("*.yml")) if resources_dir.exists() else [] + databricks_yml = bundle_dir / "databricks.yml" + if databricks_yml.exists(): + yaml_files.append(databricks_yml) + for path in yaml_files: + findings.extend(check_resource_text(path.read_text(encoding="utf-8"), filename=path.name)) + return BundleInvariantResult(findings=findings) + + +def format_result(result: BundleInvariantResult) -> str: + """Render a result as a compact human-readable report.""" + if result.ok: + return "Bundle invariants: OK" + lines = ["Bundle invariants: FAILED"] + lines.extend(f" - [{finding.code}] {finding.location}: {finding.message}" for finding in result.findings) + return "\n".join(lines) diff --git a/src/flowx/validate/dag_equivalence.py b/src/flowx/validate/dag_equivalence.py new file mode 100644 index 0000000..29b41c4 --- /dev/null +++ b/src/flowx/validate/dag_equivalence.py @@ -0,0 +1,408 @@ +"""Motif-aware DAG equivalence check (Tier-0 static validation). + +flowx translates an Azure Data Factory pipeline into a Databricks IR +pipeline. The two top-level dependency DAGs are *not* expected to be +identical, because motif collapsing rewrites the graph: each detected motif +contracts its matched activity set ``S`` into a single +:class:`~flowx.models.ir.MotifActivity`, dropping the edges internal to +``S`` and rewiring every cross-boundary edge onto the collapsed node. + +This module checks that the IR DAG equals the *quotient* of the ADF DAG under +the motif partition (so motif-induced differences are tolerated), and that +each contraction is **safe** -- i.e. it did not silently reorder anything. + +The safety condition is graph-theoretic convexity: contracting a set ``S`` to +a point preserves every ordering constraint **iff** no activity outside ``S`` +lies on a dependency path *between* two members of ``S``. If such an external +activity exists, the collapse would force it to run both before and after the +motif (a reordering / cycle); that is the invalidation we flag. + +Findings are graded: + +* ``violation`` -- the migrated DAG is not a faithful quotient of the source + (a cross-boundary ordering edge was dropped, an activity vanished, a motif + set was non-convex, or the quotient is cyclic). These block equivalence. +* ``warning`` -- a difference that is over-constraining or lossy but not + unsafe (an extra ordering edge, a merged/changed dependency outcome, an + unexplained IR-only task). +* ``tolerated`` -- a difference fully explained by motif contraction or by a + synthesised IR-only helper task; recorded only for transparency. +""" + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field + +from flowx.models.adf_ast import AdfPipeline +from flowx.models.ir import MotifActivity, Pipeline +from flowx.utils import normalize_task_key + +_DEFAULT_OUTCOME = "Succeeded" +# Task-key prefixes of synthesised variable-initialiser tasks the translator injects (engine +# ``_init_``); these have no ADF preimage and are expected to be IR-only. +_SYNTHESISED_KEY_PREFIXES = ("_init_",) + + +@dataclass(slots=True, kw_only=True) +class DagFinding: + """A single observation from the equivalence check. + + Attributes: + code: Stable machine-readable identifier (e.g. ``"missing_edge"``). + severity: One of ``"violation"`` / ``"warning"`` / ``"tolerated"``. + message: Human-readable explanation. + nodes: Block labels / activity names the finding concerns. + """ + + code: str + severity: str + message: str + nodes: tuple[str, ...] = () + + +@dataclass(slots=True, kw_only=True) +class DagEquivalenceResult: + """Outcome of :func:`check_dag_equivalence`. + + Attributes: + equivalent: True when there are no ``violation`` findings. + findings: All findings, in detection order. + """ + + equivalent: bool + findings: list[DagFinding] = field(default_factory=list) + + @property + def violations(self) -> list[DagFinding]: + """Findings that block equivalence.""" + return [finding for finding in self.findings if finding.severity == "violation"] + + @property + def warnings(self) -> list[DagFinding]: + """Non-blocking findings worth surfacing to the user.""" + return [finding for finding in self.findings if finding.severity == "warning"] + + @property + def tolerated(self) -> list[DagFinding]: + """Differences explained by motif collapse or synthesised tasks.""" + return [finding for finding in self.findings if finding.severity == "tolerated"] + + +# --------------------------------------------------------------------------- +# Graph helpers +# --------------------------------------------------------------------------- + + +def _reachable(adjacency: dict[str, set[str]], sources: set[str]) -> set[str]: + """Returns every node reachable from *sources* (sources excluded).""" + seen: set[str] = set() + queue: deque[str] = deque(sources) + while queue: + node = queue.popleft() + for successor in adjacency.get(node, ()): # noqa: B007 + if successor not in seen: + seen.add(successor) + queue.append(successor) + return seen + + +def _has_cycle(nodes: set[str], edges: set[tuple[str, str]]) -> bool: + """Kahn's algorithm: True when the directed graph has a cycle.""" + adjacency: dict[str, set[str]] = {node: set() for node in nodes} + in_degree: dict[str, int] = {node: 0 for node in nodes} + for upstream, downstream in edges: + if downstream not in adjacency[upstream]: + adjacency[upstream].add(downstream) + in_degree[downstream] = in_degree.get(downstream, 0) + 1 + queue: deque[str] = deque(node for node in nodes if in_degree[node] == 0) + visited = 0 + while queue: + node = queue.popleft() + visited += 1 + for successor in adjacency[node]: + in_degree[successor] -= 1 + if in_degree[successor] == 0: + queue.append(successor) + return visited != len(nodes) + + +def _is_synthesised(task_key: str) -> bool: + return any(task_key.startswith(prefix) for prefix in _SYNTHESISED_KEY_PREFIXES) + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def check_dag_equivalence(adf: AdfPipeline, ir: Pipeline) -> DagEquivalenceResult: + """Check that the IR DAG is a safe motif-quotient of the ADF DAG. + + Args: + adf: The source ADF pipeline (typed AST). + ir: The translated, *motif-collapsed* IR pipeline. + + Returns: + A :class:`DagEquivalenceResult`. ``equivalent`` is True when the IR + top-level DAG equals the quotient of the ADF top-level DAG under the + motif partition recorded in the IR, and every collapsed motif set is + convex (so nothing was reordered) and the result is acyclic. + """ + findings: list[DagFinding] = [] + + adf_names = [activity.name for activity in adf.activities] + adf_name_set = set(adf_names) + + # ADF top-level edges (upstream -> downstream) and per-edge conditions. + adf_adj: dict[str, set[str]] = {name: set() for name in adf_names} + adf_edge_conditions: dict[tuple[str, str], set[str]] = {} + # Per (downstream, upstream) the declared condition set -- used by the + # merged-outcome check, which mirrors the collapser's dedupe-by-source. + inbound_conditions: dict[str, dict[str, frozenset[str]]] = {} + for activity in adf.activities: + for adf_dep in activity.depends_on or []: + upstream, downstream = adf_dep.activity, activity.name + conditions = set(adf_dep.dependency_conditions or [_DEFAULT_OUTCOME]) + if upstream not in adf_name_set: + findings.append( + DagFinding( + code="dangling_adf_dependency", + severity="warning", + message=( + f"ADF activity '{downstream}' depends on '{upstream}', " + "which is not a top-level activity; edge ignored." + ), + nodes=(upstream, downstream), + ) + ) + continue + adf_adj[upstream].add(downstream) + adf_edge_conditions.setdefault((upstream, downstream), set()).update(conditions) + inbound_conditions.setdefault(downstream, {})[upstream] = frozenset(conditions) + + # Motif partition + IR block bookkeeping, read straight from the IR so we + # validate the *actual* collapse rather than re-running detection. + block_of_name: dict[str, str] = {} + motif_members: dict[str, set[str]] = {} + ir_blocks: set[str] = set() + synthesised_blocks: set[str] = set() + key_to_block: dict[str, str] = {} + + for task in ir.tasks: + label = task.task_key + ir_blocks.add(label) + key_to_block[task.task_key] = label + if isinstance(task, MotifActivity): + member_names = set(task.matched_activity_names) + motif_members[label] = member_names + for name in member_names: + block_of_name[name] = label + elif _is_synthesised(task.task_key): + synthesised_blocks.add(label) + elif task.name in adf_name_set: + # task.name is the original ADF activity name; key the singleton block by the IR task's + # actual task_key so labels match the engine's case-preserving sanitiser. Anything else is an + # unexpected IR-only task, left to surface as unmapped_ir_task. + block_of_name[task.name] = label + + # Any ADF activity with no corresponding IR task at all -> sentinel block. + # The label is absent from ``ir_blocks`` so it surfaces as ``missing_node``. + for name in adf_names: + block_of_name.setdefault(name, normalize_task_key(name)) + + adf_blocks = set(block_of_name.values()) + + # ----- Quotient of the ADF DAG under the partition ----- + quotient_edges: set[tuple[str, str]] = set() + quotient_conditions: dict[tuple[str, str], set[str]] = {} + collapsed_internal = 0 + for (upstream, downstream), conditions in adf_edge_conditions.items(): + block_upstream, block_downstream = block_of_name[upstream], block_of_name[downstream] + if block_upstream == block_downstream: + collapsed_internal += 1 + continue + quotient_edges.add((block_upstream, block_downstream)) + quotient_conditions.setdefault((block_upstream, block_downstream), set()).update(conditions) + if collapsed_internal: + findings.append( + DagFinding( + code="collapsed_internal_edges", + severity="tolerated", + message=( + f"{collapsed_internal} intra-motif edge(s) absorbed into collapsed " + "motif node(s); expected and ignored." + ), + ) + ) + + # ----- IR block-space edges ----- + ir_edges: set[tuple[str, str]] = set() + ir_conditions: dict[tuple[str, str], set[str]] = {} + for task in ir.tasks: + block_downstream = key_to_block[task.task_key] + for ir_dep in task.depends_on or []: + block_upstream = key_to_block.get(ir_dep.task_key, ir_dep.task_key) + if block_upstream == block_downstream: + continue + if block_upstream in synthesised_blocks or block_downstream in synthesised_blocks: + findings.append( + DagFinding( + code="synthesised_edge", + severity="tolerated", + message=f"Edge involving synthesised task '{block_upstream}' -> '{block_downstream}' ignored.", + nodes=(block_upstream, block_downstream), + ) + ) + continue + ir_edges.add((block_upstream, block_downstream)) + ir_conditions.setdefault((block_upstream, block_downstream), set()).add(ir_dep.outcome or _DEFAULT_OUTCOME) + + # ----- Node comparison ----- + for block in sorted(adf_blocks - ir_blocks): + claimed = motif_members.get(block) + label = block if claimed is None else f"motif[{', '.join(sorted(claimed))}]" + findings.append( + DagFinding( + code="missing_node", + severity="violation", + message=f"ADF activity/block '{label}' has no corresponding IR task.", + nodes=(block,), + ) + ) + for block in sorted(ir_blocks - adf_blocks - synthesised_blocks): + findings.append( + DagFinding( + code="unmapped_ir_task", + severity="warning", + message=f"IR task '{block}' has no ADF preimage and is not a recognised synthesised task.", + nodes=(block,), + ) + ) + for block in sorted(synthesised_blocks): + findings.append( + DagFinding( + code="synthesised_task", + severity="tolerated", + message=f"IR-only synthesised task '{block}' ignored.", + nodes=(block,), + ) + ) + + # ----- Edge comparison (only between blocks present on both sides) ----- + comparable = adf_blocks & ir_blocks + for edge in sorted(quotient_edges - ir_edges): + if edge[0] in comparable and edge[1] in comparable: + findings.append( + DagFinding( + code="missing_edge", + severity="violation", + message=f"Ordering edge '{edge[0]}' -> '{edge[1]}' present in ADF is missing from the IR DAG.", + nodes=edge, + ) + ) + for edge in sorted(ir_edges - quotient_edges): + findings.append( + DagFinding( + code="extra_edge", + severity="warning", + message=( + f"IR DAG adds ordering edge '{edge[0]}' -> '{edge[1]}' not implied by the ADF DAG " + "(over-constraining)." + ), + nodes=edge, + ) + ) + for edge in sorted(quotient_edges & ir_edges): + adf_outcomes = quotient_conditions.get(edge, set()) + ir_outcomes = ir_conditions.get(edge, set()) + if adf_outcomes and adf_outcomes != ir_outcomes: + findings.append( + DagFinding( + code="outcome_mismatch", + severity="warning", + message=( + f"Edge '{edge[0]}' -> '{edge[1]}' dependency condition changed: " + f"ADF {sorted(adf_outcomes)} vs IR {sorted(ir_outcomes)}." + ), + nodes=edge, + ) + ) + + # ----- Convexity: nothing reordered by any contraction ----- + reverse_adj: dict[str, set[str]] = {name: set() for name in adf_names} + for upstream, downstreams in adf_adj.items(): + for downstream in downstreams: + reverse_adj[downstream].add(upstream) + + for label, members in motif_members.items(): + valid_members = members & adf_name_set + if len(valid_members) < 2: + continue + descendants = _reachable(adf_adj, set(valid_members)) + ancestors = _reachable(reverse_adj, set(valid_members)) + between = (descendants & ancestors) - valid_members + if between: + findings.append( + DagFinding( + code="non_convex_motif", + severity="violation", + message=( + f"Motif '{label}' is non-convex: external activity(ies) " + f"{sorted(between)} lie on a dependency path between collapsed " + "members, so collapsing reorders them. Collapse is unsafe." + ), + nodes=tuple(sorted(between)), + ) + ) + + # Merged-outcome: collapser dedupes external deps by source, keeping + # the first outcome -- flag when members disagree on a shared source. + per_source: dict[str, set[frozenset[str]]] = {} + for member in valid_members: + for source, conds in inbound_conditions.get(member, {}).items(): + if source not in valid_members: + per_source.setdefault(source, set()).add(conds) + for source, condition_sets in per_source.items(): + if len(condition_sets) > 1: + findings.append( + DagFinding( + code="merged_outcome", + severity="warning", + message=( + f"Motif '{label}' collapses edges from '{source}' that carried " + f"differing conditions {[sorted(conditions) for conditions in condition_sets]}; " + "collapse keeps only one." + ), + nodes=(source, label), + ) + ) + + # ----- Acyclicity backstop on the deployable IR DAG ----- + if _has_cycle(ir_blocks, ir_edges): + findings.append( + DagFinding( + code="cycle", + severity="violation", + message="The translated IR DAG contains a dependency cycle.", + ) + ) + + equivalent = not any(finding.severity == "violation" for finding in findings) + return DagEquivalenceResult(equivalent=equivalent, findings=findings) + + +def format_result(result: DagEquivalenceResult) -> str: + """Render a result as a compact human-readable report.""" + status = "EQUIVALENT" if result.equivalent else "NOT EQUIVALENT" + lines = [f"DAG equivalence: {status}"] + for label, items in ( + ("Violations", result.violations), + ("Warnings", result.warnings), + ("Tolerated", result.tolerated), + ): + if not items: + continue + lines.append(f" {label} ({len(items)}):") + lines.extend(f" - [{finding.code}] {finding.message}" for finding in items) + return "\n".join(lines) diff --git a/src/orchestra/__init__.py b/src/orchestra/__init__.py deleted file mode 100644 index 16c449e..0000000 --- a/src/orchestra/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Flowx - ADF to Databricks translation plugin for Claude Code.""" - -__version__ = "0.1.0" diff --git a/src/orchestra/adapter/__init__.py b/src/orchestra/adapter/__init__.py deleted file mode 100644 index 6add5d5..0000000 --- a/src/orchestra/adapter/__init__.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Agent-facing surfaces and the matching pipeline modifier for flowx translation. - -This package draws a deliberate line between two roles: - -* **Agent adapter** -- :mod:`flowx.adapter.session` plus the question - shapes in :mod:`flowx.adapter.models`. This is the layer an agent - calls. It converts tool-call arguments into deterministic service calls - and maps "need more input" signals into structured objects (and the - :exc:`TranslationInputRequired` exception) the agent can hand back to - the user. - -* **Pipeline modifier** -- :mod:`flowx.adapter.operations`. The - deterministic transformation that consumes a validated - :class:`TranslationPreferences` snapshot and stamps concrete decisions - onto a Pipeline IR. It has no awareness of agents or user prompts and - is safely importable from non-agent contexts (CLI, tests, batch jobs). - -The package is organised into three primary modules plus the predicates -and session helpers: - -* :mod:`~flowx.adapter.models` -- StrEnums and dataclasses. -* :mod:`~flowx.adapter.operations` -- Free functions - (``gather_questions``, ``apply_preferences``, ``validate_answer``, - ``allowed_values_for``). -* :mod:`~flowx.adapter.constants` -- Question IDs, compute-mode - strings, replacement names, and other shared constants. -* :mod:`~flowx.adapter.predicates` -- Pure IR predicates used by - both ``operations`` and the bundler. -* :mod:`~flowx.adapter.session` -- The agent adapter class. -""" - -from __future__ import annotations - -from flowx.adapter.models import ( - DEFAULT_PREFERENCES, - CopyActivityParadigm, - LakeflowConnectorType, - MetadataDrivenAccess, - MetadataDrivenConsolidate, - MetadataDrivenLookupTool, - MetadataDrivenSize, - MigrationInputQuestion, - MotifConsolidate, - NonDatabricksTaskCompute, - PendingMigrationInputs, - PendingQuestions, - QuestionOption, - TranslationPreferences, - TranslationQuestion, - UseLakeflowConnectors, -) -from flowx.adapter.operations import ( - allowed_values_for, - apply_preferences, - collect_workspace_artifact_paths, - detect_databricks_hosts, - enum_for, - gather_questions, - validate_answer, -) -from flowx.adapter.session import ( - MigrationInputSession, - TranslationInputRequired, - TranslationSession, - UnknownMigrationPhaseError, -) - -__all__ = [ - "DEFAULT_PREFERENCES", - "CopyActivityParadigm", - "LakeflowConnectorType", - "MetadataDrivenAccess", - "MetadataDrivenConsolidate", - "MetadataDrivenLookupTool", - "MetadataDrivenSize", - "MigrationInputQuestion", - "MigrationInputSession", - "MotifConsolidate", - "NonDatabricksTaskCompute", - "PendingMigrationInputs", - "PendingQuestions", - "QuestionOption", - "TranslationInputRequired", - "TranslationPreferences", - "TranslationQuestion", - "TranslationSession", - "UnknownMigrationPhaseError", - "UseLakeflowConnectors", - "allowed_values_for", - "apply_preferences", - "collect_workspace_artifact_paths", - "detect_databricks_hosts", - "enum_for", - "gather_questions", - "validate_answer", -] diff --git a/src/orchestra/adapter/__main__.py b/src/orchestra/adapter/__main__.py deleted file mode 100644 index 490de59..0000000 --- a/src/orchestra/adapter/__main__.py +++ /dev/null @@ -1,597 +0,0 @@ -"""CLI bridge that lets the flowx skills drive the adapter via subprocesses. - -The skills (`/flowx:translate`, `/flowx:migrate`) cannot keep a -Python session alive across user prompts, so this module exposes two -stateless subcommands: - -* ``inspect`` reads a translation report and emits the pending questions - as JSON for the agent to surface to the user. -* ``modify`` reads the same report plus a JSON file of answers and writes - a preference-stamped report the prepare phase consumes verbatim. -""" - -from __future__ import annotations - -import argparse -import json -import sys -from dataclasses import asdict -from pathlib import Path -from typing import Any - -from flowx.adapter.constants import MOTIF_CONSOLIDATE_QUESTION_PREFIX -from flowx.adapter.models import ( - DEFAULT_PREFERENCES, - CopyActivityParadigm, - LakeflowConnectorType, - MetadataDrivenAccess, - MetadataDrivenConsolidate, - MetadataDrivenLookupTool, - MetadataDrivenSize, - MotifConsolidate, - NonDatabricksTaskCompute, - PendingQuestions, - TranslationPreferences, - TranslationQuestion, - UseLakeflowConnectors, -) -from flowx.adapter.operations import ( - apply_preferences, - collect_workspace_artifact_paths, - detect_databricks_hosts, - gather_questions, - validate_answer, -) -from flowx.bundler.dab_writer import pipeline_dict_to_ir -from flowx.translator.engine import _pipeline_to_dict - - -def main(argv: list[str] | None = None) -> int: - """Dispatches an ``inspect`` or ``modify`` subcommand. - - Args: - argv: CLI arguments to parse. Defaults to :data:`sys.argv` when - ``None``. - - Returns: - Exit code (0 on success, non-zero on usage or runtime errors). - """ - parser = _build_parser() - args = parser.parse_args(argv) - if args.command == "inspect": - return _run_inspect(args) - if args.command == "modify": - return _run_modify(args) - if args.command == "materialize-lookup": - return _run_materialize_lookup(args) - if args.command == "inputs": - return _run_inputs(args) - if args.command == "workspace-paths": - return _run_workspace_paths(args) - parser.print_help(sys.stderr) - return 2 - - -def _run_workspace_paths(args: argparse.Namespace) -> int: - """Implements the ``workspace-paths`` subcommand. - - Args: - args: Parsed CLI namespace carrying ``report``, ``source_dir``, - and ``out``. - - Returns: - ``0`` on success. The command always succeeds when the report - can be read; missing or unreadable inputs simply produce empty - path / host lists so the skill can detect the no-op case. - """ - paths = collect_workspace_artifact_paths(args.report) - suggested_hosts = detect_databricks_hosts(args.source_dir) if args.source_dir else [] - payload = { - "paths": paths, - "suggested_hosts": suggested_hosts, - "needs_auth": bool(paths), - } - _emit_json(payload, args.out) - return 0 - - -def _run_inputs(args: argparse.Namespace) -> int: - """Implements the ``inputs`` subcommand. - - Args: - args: Parsed CLI namespace carrying ``phase`` and ``out``. - - Returns: - ``0`` on success. The CLI never raises here because the phase - argument is constrained by argparse. - """ - from flowx.adapter.session import MigrationInputSession - - session = MigrationInputSession(phase=args.phase) - pending = session.pending() - payload = { - "phase": pending.phase, - "questions": [ - { - "question_id": question.question_id, - "prompt": question.prompt, - "description": question.description, - "default": question.default, - "required": question.required, - } - for question in pending.questions - ], - } - _emit_json(payload, args.out) - return 0 - - -def _build_parser() -> argparse.ArgumentParser: - """Builds the top-level argparse parser with the two subcommands. - - Returns: - Configured :class:`argparse.ArgumentParser`. - """ - parser = argparse.ArgumentParser( - prog="python -m flowx.adapter", - description="Inspect and modify a translated flowx pipeline IR.", - ) - subparsers = parser.add_subparsers(dest="command", required=True) - - inspect = subparsers.add_parser( - "inspect", - help="Emit pending translation questions for a report as JSON.", - ) - inspect.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") - inspect.add_argument( - "--answers", - type=Path, - default=None, - help=( - "Optional path to a JSON file of answers already collected; " - "questions whose conditions depend on those answers will surface " - "only when their conditions are met." - ), - ) - inspect.add_argument( - "--out", - type=Path, - default=None, - help="Optional output file; defaults to stdout.", - ) - - modify = subparsers.add_parser( - "modify", - help="Apply collected answers to a translation report and write the stamped IR.", - ) - modify.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") - modify.add_argument("answers", type=Path, help="Path to a JSON file mapping question_id to answer string.") - modify.add_argument( - "--out", - type=Path, - required=True, - help="Destination path for the preference-stamped IR JSON.", - ) - modify.add_argument( - "--lookup-values", - type=Path, - default=None, - help=( - "Optional path to a JSON list of lookup-value rows that consolidated " - "metadata-driven motifs should ingest. Each row is a dict mirroring " - "a row from the source Lookup query." - ), - ) - - workspace_paths = subparsers.add_parser( - "workspace-paths", - help=( - "Detect absolute workspace paths in a stamped report and suggest " - "Databricks workspace hosts from the ADF linked services." - ), - ) - workspace_paths.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") - workspace_paths.add_argument( - "--source-dir", - type=Path, - default=None, - help=( - "Optional path to the ADF JSON export directory. When supplied, " - "the command reads ``linked_services/*.json`` to suggest the " - "workspace host that ``databricks auth login --host`` should use." - ), - ) - workspace_paths.add_argument( - "--out", - type=Path, - default=None, - help="Optional output file; defaults to stdout.", - ) - - inputs = subparsers.add_parser( - "inputs", - help="Emit the migration-phase input questions for an flowx phase as JSON.", - ) - inputs.add_argument( - "phase", - choices=("ingest", "translate", "prepare"), - help="Migration phase whose input prompts the agent should surface.", - ) - inputs.add_argument( - "--out", - type=Path, - default=None, - help="Optional output file; defaults to stdout.", - ) - - materialize = subparsers.add_parser( - "materialize-lookup", - help="Parse CSV-shaped lookup values into the JSON shape modify consumes.", - ) - materialize.add_argument( - "source", - help=( - "Either a path to a CSV file or a literal CSV string. The first row " - "is treated as headers and every subsequent row is emitted as one dict." - ), - ) - materialize.add_argument( - "--out", - type=Path, - required=True, - help="Destination path for the lookup-values JSON list.", - ) - return parser - - -def _run_inspect(args: argparse.Namespace) -> int: - """Implements the ``inspect`` subcommand. - - Args: - args: Parsed CLI namespace carrying ``report``, ``answers``, and - ``out``. - - Returns: - ``0`` when the report was inspected successfully, ``1`` when the - report could not be loaded. - """ - pipelines = _load_pipelines(args.report) - if pipelines is None: - return 1 - answers = _read_answers_optional(args.answers) if getattr(args, "answers", None) else {} - payload = { - "pipelines": [_pending_to_payload(gather_questions(pipeline, [], answers=answers)) for pipeline in pipelines], - } - _emit_json(payload, args.out) - return 0 - - -def _read_answers_optional(answers_path: Path) -> dict[str, str]: - """Loads an answers JSON file supplied to ``inspect``. - - Args: - answers_path: Path to a JSON file mapping question_id to answer. - - Returns: - Mapping of question_id to answer string. Returns an empty dict - when the file is missing or unparseable so ``inspect`` still - succeeds (question gating just sees no prior answers). - """ - try: - raw = json.loads(answers_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return {} - return {key: str(value) for key, value in raw.items() if isinstance(value, str)} - - -def _run_modify(args: argparse.Namespace) -> int: - """Implements the ``modify`` subcommand. - - Args: - args: Parsed CLI namespace carrying ``report``, ``answers``, - ``out``, and the optional ``lookup_values``. - - Returns: - ``0`` when the modified IR was written successfully, ``1`` when - the report could not be loaded, ``2`` when the answers failed - validation. - """ - pipelines = _load_pipelines(args.report) - if pipelines is None: - return 1 - try: - answers = _load_answers(args.answers) - preferences = _preferences_from_answers(answers) - except ValueError as error: - print(f"Invalid answers payload: {error}", file=sys.stderr) - return 2 - lookup_values = _load_lookup_values(args.lookup_values) if args.lookup_values else [] - stamped_pipelines = [ - _stamp_lookup_values_into_metadata_driven_motifs(apply_preferences(pipeline, preferences), lookup_values) - for pipeline in pipelines - ] - modified = [_pipeline_to_dict(pipeline) for pipeline in stamped_pipelines] - _write_modified_report(args.report, modified, args.out) - return 0 - - -def _run_materialize_lookup(args: argparse.Namespace) -> int: - """Implements the ``materialize-lookup`` subcommand. - - Args: - args: Parsed CLI namespace carrying ``source`` (file path or - literal CSV string) and ``out``. - - Returns: - ``0`` when the JSON was written successfully, ``2`` when the - source could not be parsed as CSV. - """ - try: - rows = _parse_csv_source(args.source) - except ValueError as error: - print(f"Invalid CSV source: {error}", file=sys.stderr) - return 2 - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8") - return 0 - - -def _parse_csv_source(source: str) -> list[dict[str, str]]: - """Parses a CSV file path or literal CSV string into a list of row dicts. - - Args: - source: Either a path to a CSV file or a literal CSV string with - a header row. - - Returns: - List of dicts, one per data row, keyed by the header names. - - Raises: - ValueError: When the CSV has no header row or is empty. - """ - import csv - - source_path = Path(source) - text = source_path.read_text(encoding="utf-8") if source_path.exists() else source - reader = csv.DictReader(text.splitlines()) - if reader.fieldnames is None: - raise ValueError("Source CSV is empty or missing a header row") - return [dict(row) for row in reader] - - -def _load_lookup_values(lookup_values_path: Path) -> list[dict[str, Any]]: - """Loads materialised lookup values from a JSON file. - - Args: - lookup_values_path: Path to a JSON list of row dicts. - - Returns: - The parsed list of row dicts. Returns an empty list when the - file is missing or unparseable so the modify pass still succeeds - (consolidated motifs will warn and fall back to the scaffold). - """ - try: - raw = json.loads(lookup_values_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return [] - if not isinstance(raw, list): - return [] - return [row for row in raw if isinstance(row, dict)] - - -def _stamp_lookup_values_into_metadata_driven_motifs(pipeline, lookup_values: list[dict[str, Any]]): - """Stamps lookup values onto every metadata-driven motif marked for consolidation. - - Args: - pipeline: Preference-stamped pipeline IR. - lookup_values: Rows materialised by the agent or the user. - - Returns: - A new :class:`Pipeline` whose metadata-driven motif activities - carry the supplied lookup rows. When *lookup_values* is empty - the pipeline is returned unchanged. - """ - if not lookup_values: - return pipeline - import dataclasses as _dataclasses - - from flowx.models.ir import MotifActivity as _MotifActivity - - stamped_tasks = [] - for task in pipeline.tasks: - if isinstance(task, _MotifActivity) and task.consolidate_metadata_driven: - stamped_tasks.append(_dataclasses.replace(task, lookup_values=list(lookup_values))) - else: - stamped_tasks.append(task) - return _dataclasses.replace(pipeline, tasks=stamped_tasks) - - -def _load_pipelines(report_path: Path) -> list[Any] | None: - """Loads every pipeline IR contained in a report file. - - Args: - report_path: Path to a translation report or pipeline IR JSON. - - Returns: - List of rehydrated :class:`Pipeline` objects, or ``None`` when - the file could not be parsed. - """ - try: - raw = json.loads(report_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as error: - print(f"Failed to read {report_path}: {error}", file=sys.stderr) - return None - pipeline_dicts = _extract_pipeline_dicts(raw) - return [pipeline_dict_to_ir(pipeline_dict)[0] for pipeline_dict in pipeline_dicts] - - -def _extract_pipeline_dicts(raw: Any) -> list[dict[str, Any]]: - """Normalises a translation report into a list of pipeline IR dicts. - - Args: - raw: Parsed JSON content from a report file. - - Returns: - List of dicts, each in the shape ``engine._pipeline_to_dict`` - produces. Empty when *raw* does not contain a recognisable - pipeline payload. - """ - if isinstance(raw, dict) and "tasks" in raw and "name" in raw: - return [raw] - if isinstance(raw, dict) and "translations" in raw: - return [ - {"name": entry["pipeline"], **entry["ir"]} - for entry in raw.get("translations", []) - if entry.get("status") == "translated" and entry.get("ir") - ] - return [] - - -def _load_answers(answers_path: Path) -> dict[str, str]: - """Loads a JSON answers file and validates its top-level shape. - - Args: - answers_path: Path to a JSON file mapping question_id to answer. - - Returns: - Mapping of question_id to answer string. - - Raises: - ValueError: When the file is not a JSON object of string values. - """ - raw = json.loads(answers_path.read_text(encoding="utf-8")) - if not isinstance(raw, dict): - raise ValueError(f"Expected a JSON object at {answers_path}; got {type(raw).__name__}") - coerced: dict[str, str] = {} - for key, value in raw.items(): - if not isinstance(value, str): - raise ValueError(f"Answer for {key!r} must be a string; got {type(value).__name__}") - coerced[key] = value - return coerced - - -def _preferences_from_answers(answers: dict[str, str]) -> TranslationPreferences: - """Builds a :class:`TranslationPreferences` from a validated answers dict. - - Args: - answers: Validated mapping of question_id to answer string. - - Returns: - Preferences with every answered field overridden and every - unanswered field defaulted. - - Raises: - ValueError: When an answer is not in the allowed set for its - question. - """ - validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} - motif_consolidations: dict[str, MotifConsolidate] = {} - for qid, value in validated.items(): - if qid.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): - motif_consolidations[qid[len(MOTIF_CONSOLIDATE_QUESTION_PREFIX) :]] = MotifConsolidate(value) - return TranslationPreferences( - copy_activity_paradigm=CopyActivityParadigm( - validated.get("copy_activity_paradigm", DEFAULT_PREFERENCES.copy_activity_paradigm) - ), - non_databricks_task_compute=NonDatabricksTaskCompute( - validated.get("non_databricks_task_compute", DEFAULT_PREFERENCES.non_databricks_task_compute) - ), - use_lakeflow_connectors=UseLakeflowConnectors( - validated.get("use_lakeflow_connectors", DEFAULT_PREFERENCES.use_lakeflow_connectors) - ), - lakeflow_connector_type=LakeflowConnectorType( - validated.get("lakeflow_connector_type", DEFAULT_PREFERENCES.lakeflow_connector_type) - ), - metadata_driven_consolidate=MetadataDrivenConsolidate( - validated.get("metadata_driven_consolidate", DEFAULT_PREFERENCES.metadata_driven_consolidate) - ), - metadata_driven_access=MetadataDrivenAccess( - validated.get("metadata_driven_access", DEFAULT_PREFERENCES.metadata_driven_access) - ), - metadata_driven_size=MetadataDrivenSize( - validated.get("metadata_driven_size", DEFAULT_PREFERENCES.metadata_driven_size) - ), - metadata_driven_lookup_tool=MetadataDrivenLookupTool( - validated.get("metadata_driven_lookup_tool", DEFAULT_PREFERENCES.metadata_driven_lookup_tool) - ), - motif_consolidations=motif_consolidations, - ) - - -def _pending_to_payload(pending: PendingQuestions) -> dict[str, Any]: - """Serialises pending questions for transmission over stdout. - - Args: - pending: Outstanding questions for a single pipeline. - - Returns: - JSON-friendly dict the agent can iterate over to prompt the user. - """ - return { - "pipeline_name": pending.pipeline_name, - "questions": [_question_to_payload(question) for question in pending.questions], - } - - -def _question_to_payload(question: TranslationQuestion) -> dict[str, Any]: - """Serialises a single :class:`TranslationQuestion` to a JSON-friendly dict. - - Args: - question: Question to serialise. - - Returns: - Dict containing the question's fields with options flattened to - plain dicts. - """ - return { - "question_id": question.question_id, - "prompt": question.prompt, - "rationale": question.rationale, - "options": [asdict(option) for option in question.options], - "affected_task_keys": list(question.affected_task_keys), - "default": question.default, - } - - -def _emit_json(payload: dict[str, Any], out: Path | None) -> None: - """Writes a JSON payload to a file or to stdout. - - Args: - payload: JSON-serialisable mapping to emit. - out: Destination path; ``None`` selects stdout. - """ - encoded = json.dumps(payload, indent=2, default=str) - if out is None: - print(encoded) - return - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(encoded + "\n", encoding="utf-8") - - -def _write_modified_report(report_path: Path, pipelines: list[dict[str, Any]], out: Path) -> None: - """Writes the preference-stamped IR to *out* using the input report's shape. - - Args: - report_path: Path the modified report was sourced from. Used - only to detect whether the input was a single pipeline IR - or an aggregated translation report. - pipelines: Stamped pipeline IR dicts to write. - out: Destination path for the modified report. - """ - raw = json.loads(report_path.read_text(encoding="utf-8")) - if isinstance(raw, dict) and "translations" in raw: - by_name = {pipeline["name"]: pipeline for pipeline in pipelines} - for entry in raw.get("translations", []): - stamped = by_name.get(entry.get("pipeline")) - if stamped is not None and entry.get("ir") is not None: - entry["ir"] = {key: value for key, value in stamped.items() if key != "name"} - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(raw, indent=2, default=str) + "\n", encoding="utf-8") - return - payload = pipelines[0] if len(pipelines) == 1 else {"pipelines": pipelines} - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/orchestra/adapter/session.py b/src/orchestra/adapter/session.py deleted file mode 100644 index 318622d..0000000 --- a/src/orchestra/adapter/session.py +++ /dev/null @@ -1,450 +0,0 @@ -"""Agent adapter that drives the ask-validate-resume loop. - -:class:`TranslationSession` is the entry point an agent uses to -translate tool-call arguments into validated preferences. When the IR -raises questions the agent cannot answer from context alone, the -session surfaces them as structured :class:`TranslationQuestion` -objects (and, via :exc:`TranslationInputRequired`, as exceptions) so -the agent can route them back to the user. The pipeline modifier is -invoked only once every question has an answer. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field - -from flowx.adapter.constants import ( - INPUT_ADF_RESOURCE_URL, - INPUT_ADF_SOURCE_PATH, - INPUT_BUNDLE_NAME, - INPUT_CATALOG, - INPUT_DATABRICKS_PROFILE, - INPUT_INVENTORY_PATH, - INPUT_OUTPUT_BUNDLE_PATH, - INPUT_OUTPUT_DIR, - INPUT_SCHEMA, - INPUT_TRANSLATION_REPORT_PATH, - PHASE_INGEST, - PHASE_PREPARE, - PHASE_TRANSLATE, -) -from flowx.adapter.models import ( - DEFAULT_PREFERENCES, - CopyActivityParadigm, - LakeflowConnectorType, - MetadataDrivenAccess, - MetadataDrivenConsolidate, - MetadataDrivenLookupTool, - MetadataDrivenSize, - MigrationInputQuestion, - MotifConsolidate, - NonDatabricksTaskCompute, - PendingMigrationInputs, - PendingQuestions, - TranslationPreferences, - TranslationQuestion, - UseLakeflowConnectors, -) -from flowx.adapter.operations import ( - apply_preferences, - gather_questions, - validate_answer, -) -from flowx.models.ir import Pipeline -from flowx.models.motifs import DetectedMotif - - -class TranslationInputRequired(Exception): - """Raised by :meth:`TranslationSession.run` when answers are still missing. - - Attributes: - pending: The outstanding questions the agent should route to the - user before retrying :meth:`TranslationSession.run`. - """ - - def __init__(self, pending: PendingQuestions) -> None: - """Stores the pending questions on the exception. - - Args: - pending: Outstanding questions surfaced by the session. - """ - super().__init__( - f"{len(pending.questions)} translation question(s) require user input " - f"for pipeline {pending.pipeline_name!r}" - ) - self.pending = pending - - -@dataclass(slots=True, kw_only=True) -class TranslationSession: - """Coordinates the ask-validate-resume loop for one translated pipeline. - - A session is single-use: the caller drives it by either polling via - :meth:`pending` and :meth:`answer`, or calling :meth:`run` and - handling :exc:`TranslationInputRequired`. When every question is - answered, :meth:`run` (or :meth:`resume`) returns the - preference-stamped pipeline. - - Attributes: - pipeline: Translated pipeline IR after motif collapsing. - motifs: Detected motifs for the pipeline. Optional; only used to - decide whether the Lakeflow Connect question applies. - defaults: Baseline preferences applied when the caller skips a - question. Per-task overrides on this object are preserved - verbatim when :meth:`build_preferences` composes the final - snapshot. - """ - - pipeline: Pipeline - motifs: list[DetectedMotif] = field(default_factory=list) - defaults: TranslationPreferences = DEFAULT_PREFERENCES - _answers: dict[str, str] = field(default_factory=dict) - - def pending(self) -> PendingQuestions: - """Returns the questions still awaiting an answer. - - Returns: - A :class:`PendingQuestions` instance containing only the - questions whose preconditions are met by the IR and whose - IDs are not yet in the answer set. - """ - return gather_questions( - self.pipeline, - self.motifs, - answers=self._answers, - ) - - def answer(self, question_id: str, value: str) -> None: - """Validates and records a single answer. - - Args: - question_id: Stable question identifier from - :class:`TranslationQuestion`. - value: Caller-supplied answer string. - - Raises: - ValueError: When *question_id* is unknown or *value* is not - in the allowed set for the question. - """ - self._answers[question_id] = validate_answer(question_id, value) - - def answer_many(self, answers: dict[str, str]) -> None: - """Validates and records multiple answers atomically. - - Args: - answers: Mapping of question_id to the caller-supplied answer. - - Raises: - ValueError: When any pair fails validation. No answers from - the batch are recorded when the call raises. - """ - validated = {qid: validate_answer(qid, value) for qid, value in answers.items()} - self._answers.update(validated) - - def find_question(self, question_id: str) -> TranslationQuestion | None: - """Looks up a pending question by its identifier. - - Args: - question_id: Stable question identifier. - - Returns: - The matching :class:`TranslationQuestion` if it is still - pending, otherwise ``None``. - """ - return next( - (question for question in self.pending().questions if question.question_id == question_id), - None, - ) - - def build_preferences(self) -> TranslationPreferences: - """Composes the validated preferences snapshot from collected answers. - - Returns: - A :class:`TranslationPreferences` where every answered field - takes the caller-supplied value and every unanswered field - falls back to the corresponding value on ``defaults``. - """ - return TranslationPreferences( - copy_activity_paradigm=CopyActivityParadigm( - self._answers.get("copy_activity_paradigm", self.defaults.copy_activity_paradigm) - ), - non_databricks_task_compute=NonDatabricksTaskCompute( - self._answers.get("non_databricks_task_compute", self.defaults.non_databricks_task_compute) - ), - use_lakeflow_connectors=UseLakeflowConnectors( - self._answers.get("use_lakeflow_connectors", self.defaults.use_lakeflow_connectors) - ), - lakeflow_connector_type=LakeflowConnectorType( - self._answers.get("lakeflow_connector_type", self.defaults.lakeflow_connector_type) - ), - metadata_driven_consolidate=MetadataDrivenConsolidate( - self._answers.get("metadata_driven_consolidate", self.defaults.metadata_driven_consolidate) - ), - metadata_driven_access=MetadataDrivenAccess( - self._answers.get("metadata_driven_access", self.defaults.metadata_driven_access) - ), - metadata_driven_size=MetadataDrivenSize( - self._answers.get("metadata_driven_size", self.defaults.metadata_driven_size) - ), - metadata_driven_lookup_tool=MetadataDrivenLookupTool( - self._answers.get("metadata_driven_lookup_tool", self.defaults.metadata_driven_lookup_tool) - ), - motif_consolidations=self._collect_motif_consolidations(), - per_task=self.defaults.per_task, - ) - - def _collect_motif_consolidations(self) -> dict[str, MotifConsolidate]: - """Returns the per-motif consolidation answers gathered so far. - - Returns: - Dict mapping ``motif_id`` to the user's :class:`MotifConsolidate` - answer. Motifs the user did not answer fall back to the - value carried on ``self.defaults`` (default - :data:`MotifConsolidate.KEEP`). The dict is the union of - the defaults and any answers whose ``question_id`` starts - with ``consolidate_motif:``. - """ - from flowx.adapter.constants import MOTIF_CONSOLIDATE_QUESTION_PREFIX - - consolidations: dict[str, MotifConsolidate] = dict(self.defaults.motif_consolidations) - for question_id, answer in self._answers.items(): - if not question_id.startswith(MOTIF_CONSOLIDATE_QUESTION_PREFIX): - continue - motif_id = question_id[len(MOTIF_CONSOLIDATE_QUESTION_PREFIX) :] - consolidations[motif_id] = MotifConsolidate(answer) - return consolidations - - def resume(self) -> Pipeline: - """Returns the preference-stamped pipeline IR. - - Returns: - A new :class:`Pipeline` produced by applying the composed - preferences to ``self.pipeline``. The input pipeline is not - mutated. - """ - return apply_preferences(self.pipeline, self.build_preferences()) - - def run(self) -> Pipeline: - """Returns the modified pipeline, raising when input is still required. - - Returns: - The preference-stamped pipeline IR when every applicable - question has an answer. - - Raises: - TranslationInputRequired: When one or more questions are - still outstanding. The exception carries the pending - questions so the agent can route them to the user. - """ - pending = self.pending() - if pending.questions: - raise TranslationInputRequired(pending) - return self.resume() - - -_INGEST_QUESTIONS: tuple[MigrationInputQuestion, ...] = ( - MigrationInputQuestion( - question_id=INPUT_ADF_SOURCE_PATH, - prompt="Where are the ADF JSON exports?", - description=( - "Unity Catalog volume path (``/Volumes///``) " - "or a local directory containing the ADF ARM/JSON export." - ), - required=True, - ), - MigrationInputQuestion( - question_id=INPUT_ADF_RESOURCE_URL, - prompt="ADF resource URL?", - description=( - "Azure portal URL of the source Data Factory. Captured for " - "traceability and surfaced in the generated bundle README; " - "leave blank when the source is exported from a local copy." - ), - default="", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_OUTPUT_DIR, - prompt="Where should flowx write the ingest output?", - description="Directory the ingest phase writes ``inventory.json`` and ``ast/`` into.", - default="./orchestra_output/ingest", - required=False, - ), -) - -_TRANSLATE_QUESTIONS: tuple[MigrationInputQuestion, ...] = ( - MigrationInputQuestion( - question_id=INPUT_INVENTORY_PATH, - prompt="Path to the inventory.json from the ingest phase?", - description="Inventory produced by the ingest phase that the translator consumes.", - default="./orchestra_output/ingest/inventory.json", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_ADF_SOURCE_PATH, - prompt="Path to the ADF JSON exports?", - description="Same source directory the ingest phase consumed; needed for cross-references.", - required=True, - ), - MigrationInputQuestion( - question_id=INPUT_OUTPUT_DIR, - prompt="Where should flowx write the translate output?", - description="Directory the translate phase writes the report and IR into.", - default="./orchestra_output/translate", - required=False, - ), -) - -_PREPARE_QUESTIONS: tuple[MigrationInputQuestion, ...] = ( - MigrationInputQuestion( - question_id=INPUT_TRANSLATION_REPORT_PATH, - prompt="Path to the translation report?", - description=( - "Preference-stamped report from `python -m flowx.adapter modify`, " - "or the raw translate-phase report when no preferences were applied." - ), - default="./orchestra_output/translate/translation_report.stamped.json", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_OUTPUT_BUNDLE_PATH, - prompt="Where should the generated DAB bundle be written?", - description="Root directory for the emitted Databricks Declarative Automation Bundle.", - default="./dab_output", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_CATALOG, - prompt="Target Unity Catalog catalog?", - description="Default ``catalog`` bundle variable used by emitted notebooks and pipelines.", - default="main", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_SCHEMA, - prompt="Target Unity Catalog schema?", - description="Default ``schema`` bundle variable used by emitted notebooks and pipelines.", - default="default", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_BUNDLE_NAME, - prompt="Bundle name override?", - description="Defaults to the first translated pipeline's resource key when blank.", - default="", - required=False, - ), - MigrationInputQuestion( - question_id=INPUT_DATABRICKS_PROFILE, - prompt="Databricks CLI profile?", - description=( - "Profile used to download workspace-resident notebooks during the " - "prepare phase. Leave blank to use the default profile from " - "``~/.databrickscfg`` or the active ``DATABRICKS_*`` env vars." - ), - default="", - required=False, - ), -) - -_QUESTIONS_BY_PHASE: dict[str, tuple[MigrationInputQuestion, ...]] = { - PHASE_INGEST: _INGEST_QUESTIONS, - PHASE_TRANSLATE: _TRANSLATE_QUESTIONS, - PHASE_PREPARE: _PREPARE_QUESTIONS, -} - - -class UnknownMigrationPhaseError(ValueError): - """Raised when a MigrationInputSession is constructed with an unrecognised phase.""" - - -@dataclass(slots=True, kw_only=True) -class MigrationInputSession: - """Coordinates the free-text input prompts at the top of an flowx phase. - - A session is single-use: the caller drives it by polling - :meth:`pending` and recording answers via :meth:`answer`, then reads - them out with :meth:`collected` once every required input has a - value. The session is intentionally distinct from - :class:`TranslationSession` because the inputs it gathers are - free-text paths and identifiers rather than enum-backed choices. - - Attributes: - phase: One of ``"ingest"``, ``"translate"``, ``"prepare"``. - """ - - phase: str - _answers: dict[str, str] = field(default_factory=dict) - - def __post_init__(self) -> None: - """Validates that *phase* is one of the supported migration phases. - - Raises: - UnknownMigrationPhaseError: When *phase* is not registered in - :data:`_QUESTIONS_BY_PHASE`. - """ - if self.phase not in _QUESTIONS_BY_PHASE: - raise UnknownMigrationPhaseError( - f"Unknown migration phase {self.phase!r}; expected one of {sorted(_QUESTIONS_BY_PHASE)}" - ) - - def pending(self) -> PendingMigrationInputs: - """Returns the input questions still awaiting an answer. - - Returns: - A :class:`PendingMigrationInputs` with the unanswered - questions for ``self.phase`` in registration order. - """ - questions = [ - question for question in _QUESTIONS_BY_PHASE[self.phase] if question.question_id not in self._answers - ] - return PendingMigrationInputs(phase=self.phase, questions=questions) - - def answer(self, question_id: str, value: str) -> None: - """Records an answer to one input question. - - Args: - question_id: Stable identifier of the question. - value: Caller-supplied string value. - - Raises: - ValueError: When *question_id* is not a known input for the - session's phase. - """ - if not any(question.question_id == question_id for question in _QUESTIONS_BY_PHASE[self.phase]): - raise ValueError(f"Unknown input question {question_id!r} for phase {self.phase!r}") - self._answers[question_id] = value - - def answer_many(self, answers: dict[str, str]) -> None: - """Records multiple input answers atomically. - - Args: - answers: Mapping of question_id to the caller-supplied value. - - Raises: - ValueError: When any pair references an unknown question. - No answers are recorded when the call raises. - """ - known_ids = {question.question_id for question in _QUESTIONS_BY_PHASE[self.phase]} - unknown = set(answers) - known_ids - if unknown: - raise ValueError(f"Unknown input questions for phase {self.phase!r}: {sorted(unknown)}") - self._answers.update(answers) - - def collected(self) -> dict[str, str]: - """Returns the collected answers merged with each question's default. - - Returns: - A dict keyed by question_id covering every question for the - phase: caller-supplied answers take precedence; otherwise - the question's ``default`` value (which may be the empty - string) is used. Required questions whose answers are - missing are omitted so the caller can detect them. - """ - collected: dict[str, str] = {} - for question in _QUESTIONS_BY_PHASE[self.phase]: - if question.question_id in self._answers: - collected[question.question_id] = self._answers[question.question_id] - elif question.default is not None: - collected[question.question_id] = question.default - return collected diff --git a/src/orchestra/translator/activity_translators/web_activity.py b/src/orchestra/translator/activity_translators/web_activity.py deleted file mode 100644 index 7f33c6b..0000000 --- a/src/orchestra/translator/activity_translators/web_activity.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Translates ADF WebActivity activities to Databricks WebActivity IR.""" - -from __future__ import annotations - -from typing import Any - -from flowx.models.adf_ast import AdfActivity, AdfDefinitions -from flowx.models.ir import Activity, TranslationContext -from flowx.models.ir import WebActivity as WebActivityIR -from flowx.parser.expression_parser import resolve_expression -from flowx.translator.activity_translators.resolve import resolve_dict_values, resolve_field - - -def translate( - activity: AdfActivity, - base_kwargs: dict[str, Any], - context: TranslationContext, - definitions: AdfDefinitions, -) -> Activity: - """Translates a WebActivity. - - Args: - activity: The ADF activity AST node. - base_kwargs: Common fields (name, task_key, timeout, retries, depends_on, cluster). - context: Current translation context. - definitions: Full ADF definitions for cross-referencing. - - Returns: - A :class:`WebActivity` IR node. - """ - type_properties = activity.type_properties or {} - - url = resolve_field(type_properties.get("url", ""), context) - method = type_properties.get("method", "GET") - headers = resolve_dict_values(type_properties.get("headers"), context) or None - body = _resolve_body(type_properties.get("body"), context) - authentication = type_properties.get("authentication") - disable_cert_validation = type_properties.get("disableCertValidation", False) - http_request_timeout = type_properties.get("httpRequestTimeout") - - timeout_seconds: int | None = None - if http_request_timeout and isinstance(http_request_timeout, str): - timeout_seconds = _parse_timeout_to_seconds(http_request_timeout) - - return WebActivityIR( - **base_kwargs, - url=url, - method=method, - body=body, - headers=headers, - authentication=authentication, - disable_cert_validation=disable_cert_validation, - http_request_timeout_seconds=timeout_seconds, - ) - - -def _resolve_body(body: Any, context: TranslationContext) -> Any: - """Pre-resolve ADF expressions in the request body at translate time. - - Args: - body: Raw body from the ADF typeProperties. - context: Current translation context with variable caches. - - Returns: - Resolved body — either a Python code string (for notebook_code), - the original body dict, or ``None``. - """ - if body is None: - return None - - if isinstance(body, dict) and body.get("type") == "Expression" and "value" in body: - result = resolve_expression(body, context) - if result is not None and result.kind == "notebook_code": - return result.value - if result is not None and result.kind == "literal": - return result.value - - return body - - -def _parse_timeout_to_seconds(timeout_str: str) -> int | None: - """Parses an ADF timeout string to seconds. - - Args: - timeout_str: Timeout in ``"d.hh:mm:ss"`` or ``"hh:mm:ss"`` format. - - Returns: - Total seconds, or ``None`` if the format is unrecognised. - """ - try: - parts = timeout_str.split(".") - if len(parts) == 2: - days = int(parts[0]) - time_part = parts[1] - else: - days = 0 - time_part = parts[0] - time_parts = time_part.split(":") - hours = int(time_parts[0]) if len(time_parts) > 0 else 0 - minutes = int(time_parts[1]) if len(time_parts) > 1 else 0 - seconds = int(time_parts[2]) if len(time_parts) > 2 else 0 - return days * 86400 + hours * 3600 + minutes * 60 + seconds - except (ValueError, IndexError): - return None diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py index f7744fb..a89456f 100644 --- a/tests/integration/test_end_to_end.py +++ b/tests/integration/test_end_to_end.py @@ -1,6 +1,6 @@ """End-to-end integration tests for the flowx translation pipeline. -These tests exercise the full ingest -> translate -> prepare -> bundle pipeline +These tests exercise the full profile -> translate -> prepare -> bundle pipeline against realistic ADF fixture files, simulating what happens when a user invokes the flowx skills. """ diff --git a/tests/integration/test_path_equivalence.py b/tests/integration/test_path_equivalence.py new file mode 100644 index 0000000..f4b4c89 --- /dev/null +++ b/tests/integration/test_path_equivalence.py @@ -0,0 +1,59 @@ +"""Guard #1: the in-process bundle path and the report round-trip (CLI) path +must produce identical bundles; and every generated bundle must satisfy the +structural invariants (guard #2).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from flowx.bundler.dab_writer import _pipeline_dict_to_workflow, write_bundle +from flowx.parser.adf_loader import load_adf_definitions +from flowx.preparer.workflow_preparer import prepare_workflow +from flowx.translator.engine import _pipeline_to_dict, translate_pipeline +from flowx.validate.bundle_invariants import check_bundle_dir, format_result + +FIXTURES_DIR = Path(__file__).parent.parent / "resources" / "json" +_DEFS = load_adf_definitions(FIXTURES_DIR) +_PIPELINE_NAMES = sorted(p.name for p in _DEFS.pipelines) + + +def _jobs(bundle_dir: Path) -> dict: + """Merge all `resources.jobs` mappings across the bundle's resource files.""" + jobs: dict = {} + for path in sorted((bundle_dir / "resources").glob("*.yml")): + doc = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + jobs.update(((doc.get("resources") or {}).get("jobs") or {})) + return jobs + + +@pytest.mark.parametrize("name", _PIPELINE_NAMES) +def test_inprocess_and_report_paths_agree(name: str, tmp_path: Path) -> None: + pipeline = next(p for p in _DEFS.pipelines if p.name == name) + report = translate_pipeline(pipeline, _DEFS) + + # Serialize the report BEFORE the in-process write (write_bundle mutates the + # workflow it is given, not the IR, but serialize first to be safe). + report_dict = _pipeline_to_dict(report.pipeline) + + in_process = tmp_path / "in_process" + write_bundle(prepare_workflow(report.pipeline), in_process, catalog="c", schema="s") + + report_path = tmp_path / "report_path" + write_bundle(_pipeline_dict_to_workflow(report_dict), report_path, catalog="c", schema="s") + + assert _jobs(in_process) == _jobs(report_path), ( + f"in-process vs report round-trip bundle diverged for pipeline '{name}'" + ) + + +@pytest.mark.parametrize("name", _PIPELINE_NAMES) +def test_generated_bundle_satisfies_invariants(name: str, tmp_path: Path) -> None: + pipeline = next(p for p in _DEFS.pipelines if p.name == name) + report = translate_pipeline(pipeline, _DEFS) + out = tmp_path / "bundle" + write_bundle(_pipeline_dict_to_workflow(_pipeline_to_dict(report.pipeline)), out, catalog="c", schema="s") + result = check_bundle_dir(out) + assert result.ok, format_result(result) diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 92c2615..f283c1a 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -11,13 +11,13 @@ from flowx.adapter import ( CopyActivityParadigm, NonDatabricksTaskCompute, + TranslationConfiguration, TranslationInputRequired, - TranslationPreferences, - TranslationQuestion, + TranslationOption, TranslationSession, UseLakeflowConnectors, - apply_preferences, - gather_questions, + apply_configuration, + gather_options, validate_answer, ) from flowx.adapter.__main__ import main as adapter_cli_main @@ -27,14 +27,14 @@ COMPUTE_MODE_INHERIT, COMPUTE_MODE_SERVERLESS, LAKEFLOW_CONNECT_REPLACEMENT, - QUESTION_COPY_ACTIVITY_PARADIGM, - QUESTION_LAKEFLOW_CONNECTOR_TYPE, - QUESTION_METADATA_DRIVEN_ACCESS, - QUESTION_METADATA_DRIVEN_CONSOLIDATE, - QUESTION_METADATA_DRIVEN_LOOKUP_TOOL, - QUESTION_METADATA_DRIVEN_SIZE, - QUESTION_NON_DATABRICKS_TASK_COMPUTE, - QUESTION_USE_LAKEFLOW_CONNECTORS, + OPTION_COPY_ACTIVITY_PARADIGM, + OPTION_LAKEFLOW_CONNECTOR_TYPE, + OPTION_METADATA_DRIVEN_ACCESS, + OPTION_METADATA_DRIVEN_CONSOLIDATE, + OPTION_METADATA_DRIVEN_LOOKUP_TOOL, + OPTION_METADATA_DRIVEN_SIZE, + OPTION_NON_DATABRICKS_TASK_COMPUTE, + OPTION_USE_LAKEFLOW_CONNECTORS, ) from flowx.adapter.operations import allowed_values_for, enum_for from flowx.models.ir import ( @@ -121,15 +121,15 @@ def _file_copy(name: str = "copy_files") -> CopyActivity: ) -class TestPreferences: - def test_default_preferences_are_conservative(self): - prefs = TranslationPreferences() +class TestConfiguration: + def test_default_configuration_are_conservative(self): + prefs = TranslationConfiguration() assert prefs.copy_activity_paradigm is CopyActivityParadigm.NOTEBOOK assert prefs.non_databricks_task_compute is NonDatabricksTaskCompute.SERVERLESS assert prefs.use_lakeflow_connectors is UseLakeflowConnectors.EXISTING def test_string_values_coerce_to_enums(self): - prefs = TranslationPreferences( + prefs = TranslationConfiguration( copy_activity_paradigm="sdp", non_databricks_task_compute="classic", ) @@ -138,10 +138,10 @@ def test_string_values_coerce_to_enums(self): def test_invalid_value_raises(self): with pytest.raises(ValueError, match="not a valid CopyActivityParadigm"): - TranslationPreferences(copy_activity_paradigm="bogus") + TranslationConfiguration(copy_activity_paradigm="bogus") def test_per_task_override_takes_precedence(self): - base = TranslationPreferences( + base = TranslationConfiguration( copy_activity_paradigm="notebook", per_task={"copy_a": {"copy_activity_paradigm": "sdp"}}, ) @@ -151,7 +151,7 @@ def test_per_task_override_takes_precedence(self): assert other.copy_activity_paradigm is CopyActivityParadigm.NOTEBOOK def test_effective_for_returns_self_when_no_override(self): - prefs = TranslationPreferences() + prefs = TranslationConfiguration() assert prefs.effective_for("missing") is prefs def test_enum_for_and_allowed_values_for(self): @@ -161,37 +161,37 @@ def test_enum_for_and_allowed_values_for(self): assert allowed_values_for("unknown") == () -class TestGatherQuestions: - def test_no_questions_for_empty_pipeline(self): +class TestGatherOptions: + def test_no_options_for_empty_pipeline(self): pipeline = Pipeline(name="empty", tasks=[]) - pending = gather_questions(pipeline) + pending = gather_options(pipeline) assert pending.pipeline_name == "empty" - assert pending.questions == [] + assert pending.options == [] - def test_copy_paradigm_question_only_when_delta_sink_present(self): + def test_copy_paradigm_option_only_when_delta_sink_present(self): delta_pipeline = Pipeline(name="p", tasks=[_delta_copy()]) - question_ids = {q.question_id for q in gather_questions(delta_pipeline).questions} - assert QUESTION_COPY_ACTIVITY_PARADIGM in question_ids + option_ids = {q.option_id for q in gather_options(delta_pipeline).options} + assert OPTION_COPY_ACTIVITY_PARADIGM in option_ids non_delta = Pipeline(name="p", tasks=[_file_copy()]) - question_ids = {q.question_id for q in gather_questions(non_delta).questions} - assert QUESTION_COPY_ACTIVITY_PARADIGM not in question_ids + option_ids = {q.option_id for q in gather_options(non_delta).options} + assert OPTION_COPY_ACTIVITY_PARADIGM not in option_ids - def test_non_databricks_compute_question_when_any_non_db_task(self): + def test_non_databricks_compute_option_when_any_non_db_task(self): pipeline = Pipeline(name="p", tasks=[WaitActivity(**_make_base("w"), wait_time_seconds=1)]) - ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_NON_DATABRICKS_TASK_COMPUTE in ids + ids = {q.option_id for q in gather_options(pipeline).options} + assert OPTION_NON_DATABRICKS_TASK_COMPUTE in ids - def test_lakeflow_connect_question_only_for_db_to_delta(self): + def test_lakeflow_connect_option_only_for_db_to_delta(self): with_db = Pipeline(name="p", tasks=[_delta_copy()]) - ids = {q.question_id for q in gather_questions(with_db).questions} - assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids + ids = {q.option_id for q in gather_options(with_db).options} + assert OPTION_USE_LAKEFLOW_CONNECTORS in ids without_db = Pipeline(name="p", tasks=[_file_copy()]) - ids = {q.question_id for q in gather_questions(without_db).questions} - assert QUESTION_USE_LAKEFLOW_CONNECTORS not in ids + ids = {q.option_id for q in gather_options(without_db).options} + assert OPTION_USE_LAKEFLOW_CONNECTORS not in ids - def test_lakeflow_connect_question_surfaces_for_database_motif_without_detected_motifs(self): + def test_lakeflow_connect_option_surfaces_for_database_motif_without_detected_motifs(self): """CLI callers don't have DetectedMotif objects; eligibility should derive from the IR alone.""" motif_activity = MotifActivity( **_make_base("motif_incremental_load_watermark", "motif_incremental_load_watermark"), @@ -202,13 +202,13 @@ def test_lakeflow_connect_question_surfaces_for_database_motif_without_detected_ source_type_hint="database", ) pipeline = Pipeline(name="p", tasks=[motif_activity]) - pending = gather_questions(pipeline) - ids = {q.question_id for q in pending.questions} - assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids - question = next(q for q in pending.questions if q.question_id == QUESTION_USE_LAKEFLOW_CONNECTORS) - assert "motif_incremental_load_watermark" in question.affected_task_keys + pending = gather_options(pipeline) + ids = {q.option_id for q in pending.options} + assert OPTION_USE_LAKEFLOW_CONNECTORS in ids + option = next(q for q in pending.options if q.option_id == OPTION_USE_LAKEFLOW_CONNECTORS) + assert "motif_incremental_load_watermark" in option.affected_task_keys - def test_lakeflow_connect_question_surfaces_for_database_motif(self): + def test_lakeflow_connect_option_surfaces_for_database_motif(self): motif_activity = MotifActivity( **_make_base("motif_incremental_load_watermark", "motif_incremental_load_watermark"), motif_id="incremental_load_watermark", @@ -226,35 +226,35 @@ def test_lakeflow_connect_question_surfaces_for_database_motif(self): confidence_notes=[], ) ] - pending = gather_questions(pipeline, motifs) - ids = {q.question_id for q in pending.questions} - assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids - lfc_question = next(q for q in pending.questions if q.question_id == QUESTION_USE_LAKEFLOW_CONNECTORS) - assert "motif_incremental_load_watermark" in lfc_question.affected_task_keys - - def test_no_databricks_task_compute_question_for_notebook(self): - """The serverless-replacement question for Databricks tasks was removed.""" + pending = gather_options(pipeline, motifs) + ids = {q.option_id for q in pending.options} + assert OPTION_USE_LAKEFLOW_CONNECTORS in ids + lfc_option = next(q for q in pending.options if q.option_id == OPTION_USE_LAKEFLOW_CONNECTORS) + assert "motif_incremental_load_watermark" in lfc_option.affected_task_keys + + def test_no_databricks_task_compute_option_for_notebook(self): + """The serverless-replacement option for Databricks tasks was removed.""" pipeline = Pipeline( name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")], ) - ids = {q.question_id for q in gather_questions(pipeline).questions} + ids = {q.option_id for q in gather_options(pipeline).options} assert "databricks_task_compute" not in ids - def test_no_databricks_task_compute_question_for_spark_python(self): - """The serverless-replacement question for Databricks tasks was removed.""" + def test_no_databricks_task_compute_option_for_spark_python(self): + """The serverless-replacement option for Databricks tasks was removed.""" pipeline = Pipeline( name="p", tasks=[SparkPythonActivity(**_make_base("py"), python_file="dbfs:/scripts/etl.py")], ) - ids = {q.question_id for q in gather_questions(pipeline).questions} + ids = {q.option_id for q in gather_options(pipeline).options} assert "databricks_task_compute" not in ids def test_already_answered_filters_pending(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) - pending = gather_questions(pipeline, answers={QUESTION_COPY_ACTIVITY_PARADIGM: "sdp"}) - ids = {q.question_id for q in pending.questions} - assert QUESTION_COPY_ACTIVITY_PARADIGM not in ids + pending = gather_options(pipeline, answers={OPTION_COPY_ACTIVITY_PARADIGM: "sdp"}) + ids = {q.option_id for q in pending.options} + assert OPTION_COPY_ACTIVITY_PARADIGM not in ids def test_walks_into_for_each_inner_activities(self): inner_copy = _delta_copy("inner_copy") @@ -264,15 +264,15 @@ def test_walks_into_for_each_inner_activities(self): inner_activities=[inner_copy], ) pipeline = Pipeline(name="p", tasks=[for_each]) - question = next( - (q for q in gather_questions(pipeline).questions if q.question_id == QUESTION_COPY_ACTIVITY_PARADIGM), + option = next( + (q for q in gather_options(pipeline).options if q.option_id == OPTION_COPY_ACTIVITY_PARADIGM), None, ) - assert question is not None - assert "inner_copy" in question.affected_task_keys + assert option is not None + assert "inner_copy" in option.affected_task_keys - def test_motif_consolidation_question_emitted_per_detected_motif(self): - """Each detected motif produces a ``consolidate_motif:`` question.""" + def test_motif_consolidation_option_emitted_per_detected_motif(self): + """Each detected motif produces a ``consolidate_motif:`` option.""" pipeline = Pipeline(name="p", tasks=[_delta_copy()]) motifs = [ DetectedMotif( @@ -282,20 +282,18 @@ def test_motif_consolidation_question_emitted_per_detected_motif(self): confidence_notes=["Detector matched Lookup→Copy→SP chain"], ) ] - pending = gather_questions(pipeline, motifs) - ids = {q.question_id for q in pending.questions} + pending = gather_options(pipeline, motifs) + ids = {q.option_id for q in pending.options} assert "consolidate_motif:incremental_load_watermark" in ids - motif_question = next( - q for q in pending.questions if q.question_id == "consolidate_motif:incremental_load_watermark" - ) - assert motif_question.default == "keep" - assert {opt.value for opt in motif_question.options} == {"keep", "consolidate"} - assert "WatermarkLookup" in motif_question.affected_task_keys + motif_option = next(q for q in pending.options if q.option_id == "consolidate_motif:incremental_load_watermark") + assert motif_option.default == "keep" + assert {opt.value for opt in motif_option.options} == {"keep", "consolidate"} + assert "WatermarkLookup" in motif_option.affected_task_keys # Confidence note must surface in the rationale so the agent can quote it - assert "Detector matched Lookup→Copy→SP chain" in motif_question.rationale + assert "Detector matched Lookup→Copy→SP chain" in motif_option.rationale - def test_motif_consolidation_question_filtered_by_answer(self): - """Once answered the per-motif question must drop out of pending.""" + def test_motif_consolidation_option_filtered_by_answer(self): + """Once answered the per-motif option must drop out of pending.""" pipeline = Pipeline(name="p", tasks=[_delta_copy()]) motifs = [ DetectedMotif( @@ -305,12 +303,12 @@ def test_motif_consolidation_question_filtered_by_answer(self): confidence_notes=[], ) ] - pending = gather_questions( + pending = gather_options( pipeline, motifs, answers={"consolidate_motif:incremental_load_watermark": "consolidate"}, ) - ids = {q.question_id for q in pending.questions} + ids = {q.option_id for q in pending.options} assert "consolidate_motif:incremental_load_watermark" not in ids def test_motif_consolidation_validate_answer_accepts_keep_or_consolidate(self): @@ -326,19 +324,19 @@ class TestValidateAnswer: def test_accepts_allowed_value(self): assert validate_answer("copy_activity_paradigm", "sdp") == "sdp" - def test_rejects_unknown_question(self): - with pytest.raises(ValueError, match="Unknown question_id"): - validate_answer("not_a_question", "x") + def test_rejects_unknown_option(self): + with pytest.raises(ValueError, match="Unknown option_id"): + validate_answer("not_a_option", "x") def test_rejects_invalid_value(self): with pytest.raises(ValueError, match="Invalid answer"): validate_answer("copy_activity_paradigm", "yaml") -class TestApplyPreferences: +class TestApplyConfiguration: def test_serverless_default_leaves_activities_on_serverless_compute(self): pipeline = Pipeline(name="p", tasks=[_delta_copy(), WaitActivity(**_make_base("w"), wait_time_seconds=1)]) - modified = apply_preferences(pipeline, TranslationPreferences()) + modified = apply_configuration(pipeline, TranslationConfiguration()) copy_task = modified.tasks[0] wait_task = modified.tasks[1] assert copy_task.compute_mode == COMPUTE_MODE_SERVERLESS @@ -346,8 +344,8 @@ def test_serverless_default_leaves_activities_on_serverless_compute(self): def test_classic_compute_routes_copy_to_multi_node_cluster(self): pipeline = Pipeline(name="p", tasks=[_delta_copy(), WaitActivity(**_make_base("w"), wait_time_seconds=1)]) - prefs = TranslationPreferences(non_databricks_task_compute="classic") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(non_databricks_task_compute="classic") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].compute_mode == COMPUTE_MODE_CLASSIC_MULTI_NODE assert modified.tasks[1].compute_mode == COMPUTE_MODE_CLASSIC_SINGLE_NODE @@ -355,7 +353,7 @@ def test_databricks_task_always_inherits_linked_service_cluster(self): """DatabricksNotebook activities always inherit the source linked-service cluster binding; the serverless replacement option was removed.""" pipeline = Pipeline(name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")]) - modified = apply_preferences(pipeline, TranslationPreferences()) + modified = apply_configuration(pipeline, TranslationConfiguration()) assert modified.tasks[0].compute_mode == COMPUTE_MODE_INHERIT def test_spark_python_always_inherits_linked_service_cluster(self): @@ -364,31 +362,31 @@ def test_spark_python_always_inherits_linked_service_cluster(self): pipeline = Pipeline( name="p", tasks=[SparkPythonActivity(**_make_base("py"), python_file="dbfs:/scripts/etl.py")] ) - modified = apply_preferences(pipeline, TranslationPreferences()) + modified = apply_configuration(pipeline, TranslationConfiguration()) assert modified.tasks[0].compute_mode == COMPUTE_MODE_INHERIT def test_copy_paradigm_sdp_stamps_target_format(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) - prefs = TranslationPreferences(copy_activity_paradigm="sdp") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(copy_activity_paradigm="sdp") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].target_format == "sdp" def test_copy_paradigm_does_not_apply_to_non_delta_copy(self): pipeline = Pipeline(name="p", tasks=[_file_copy()]) - prefs = TranslationPreferences(copy_activity_paradigm="sdp") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(copy_activity_paradigm="sdp") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].target_format == "notebook" def test_lakeflow_connect_flag_set_for_eligible_copy(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].use_lakeflow_connector is True def test_lakeflow_connect_skipped_for_non_database_copy(self): pipeline = Pipeline(name="p", tasks=[_file_copy()]) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].use_lakeflow_connector is False def test_motif_replacement_swapped_for_lakeflow_connect_when_database(self): @@ -401,8 +399,8 @@ def test_motif_replacement_swapped_for_lakeflow_connect_when_database(self): source_type_hint="database", ) pipeline = Pipeline(name="p", tasks=[motif]) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].databricks_replacement == LAKEFLOW_CONNECT_REPLACEMENT def test_motif_replacement_unchanged_for_file_source(self): @@ -415,31 +413,31 @@ def test_motif_replacement_unchanged_for_file_source(self): source_type_hint="files", ) pipeline = Pipeline(name="p", tasks=[motif]) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].databricks_replacement == "auto_loader_file_notification" def test_per_task_override_wins(self): pipeline = Pipeline(name="p", tasks=[_delta_copy("c1"), _delta_copy("c2")]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( copy_activity_paradigm="notebook", per_task={"c1": {"copy_activity_paradigm": "sdp"}}, ) - modified = apply_preferences(pipeline, prefs) + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].target_format == "sdp" assert modified.tasks[1].target_format == "notebook" - def test_preferences_attached_to_pipeline(self): + def test_configuration_attached_to_pipeline(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) - prefs = TranslationPreferences(copy_activity_paradigm="sdp") - modified = apply_preferences(pipeline, prefs) - assert modified.translation_preferences is prefs + prefs = TranslationConfiguration(copy_activity_paradigm="sdp") + modified = apply_configuration(pipeline, prefs) + assert modified.translation_configuration is prefs - def test_apply_preferences_does_not_mutate_input(self): + def test_apply_configuration_does_not_mutate_input(self): original = Pipeline(name="p", tasks=[_delta_copy()]) - apply_preferences(original, TranslationPreferences(copy_activity_paradigm="sdp")) + apply_configuration(original, TranslationConfiguration(copy_activity_paradigm="sdp")) assert original.tasks[0].target_format is None - assert original.translation_preferences is None + assert original.translation_configuration is None def test_recurses_into_for_each_inner_activities(self): inner = _delta_copy("inner") @@ -449,74 +447,74 @@ def test_recurses_into_for_each_inner_activities(self): inner_activities=[inner], ) pipeline = Pipeline(name="p", tasks=[for_each]) - prefs = TranslationPreferences(copy_activity_paradigm="sdp") - modified = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(copy_activity_paradigm="sdp") + modified = apply_configuration(pipeline, prefs) inner_after = modified.tasks[0].inner_activities[0] assert inner_after.target_format == "sdp" class TestTranslationSession: - def test_pending_returns_only_outstanding_questions(self): + def test_pending_returns_only_outstanding_options(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) session = TranslationSession(pipeline=pipeline) first = session.pending() - assert len(first.questions) > 0 - session.answer(QUESTION_COPY_ACTIVITY_PARADIGM, "sdp") - ids_after = {q.question_id for q in session.pending().questions} - assert QUESTION_COPY_ACTIVITY_PARADIGM not in ids_after + assert len(first.options) > 0 + session.answer(OPTION_COPY_ACTIVITY_PARADIGM, "sdp") + ids_after = {q.option_id for q in session.pending().options} + assert OPTION_COPY_ACTIVITY_PARADIGM not in ids_after - def test_run_raises_when_questions_outstanding(self): + def test_run_raises_when_options_outstanding(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) session = TranslationSession(pipeline=pipeline) with pytest.raises(TranslationInputRequired) as info: session.run() assert info.value.pending.pipeline_name == "p" - assert any(q.question_id == QUESTION_COPY_ACTIVITY_PARADIGM for q in info.value.pending.questions) + assert any(q.option_id == OPTION_COPY_ACTIVITY_PARADIGM for q in info.value.pending.options) def test_run_returns_modified_pipeline_when_complete(self): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) session = TranslationSession(pipeline=pipeline) pending = session.pending() - answers = {q.question_id: q.default for q in pending.questions} + answers = {q.option_id: q.default for q in pending.options} session.answer_many(answers) modified = session.run() - assert modified.translation_preferences is not None + assert modified.translation_configuration is not None def test_answer_validates(self): session = TranslationSession(pipeline=Pipeline(name="p", tasks=[_delta_copy()])) with pytest.raises(ValueError): - session.answer(QUESTION_COPY_ACTIVITY_PARADIGM, "yaml") + session.answer(OPTION_COPY_ACTIVITY_PARADIGM, "yaml") def test_answer_many_is_atomic(self): session = TranslationSession(pipeline=Pipeline(name="p", tasks=[_delta_copy()])) with pytest.raises(ValueError): - session.answer_many({QUESTION_COPY_ACTIVITY_PARADIGM: "sdp", "bogus": "x"}) - assert QUESTION_COPY_ACTIVITY_PARADIGM not in session._answers + session.answer_many({OPTION_COPY_ACTIVITY_PARADIGM: "sdp", "bogus": "x"}) + assert OPTION_COPY_ACTIVITY_PARADIGM not in session._answers - def test_find_question_returns_pending_question(self): + def test_find_option_returns_pending_option(self): session = TranslationSession(pipeline=Pipeline(name="p", tasks=[_delta_copy()])) - found = session.find_question(QUESTION_COPY_ACTIVITY_PARADIGM) - assert isinstance(found, TranslationQuestion) - session.answer(QUESTION_COPY_ACTIVITY_PARADIGM, "sdp") - assert session.find_question(QUESTION_COPY_ACTIVITY_PARADIGM) is None + found = session.find_option(OPTION_COPY_ACTIVITY_PARADIGM) + assert isinstance(found, TranslationOption) + session.answer(OPTION_COPY_ACTIVITY_PARADIGM, "sdp") + assert session.find_option(OPTION_COPY_ACTIVITY_PARADIGM) is None class TestSerializationRoundtrip: - def test_preferences_survive_json_roundtrip(self): + def test_configuration_survive_json_roundtrip(self): from flowx.bundler.dab_writer import pipeline_dict_to_ir from flowx.translator.engine import _pipeline_to_dict pipeline = Pipeline( name="p", tasks=[_delta_copy(), NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")] ) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( copy_activity_paradigm="sdp", non_databricks_task_compute="classic", use_lakeflow_connectors="lakeflow_connect", ) - stamped = apply_preferences(pipeline, prefs) + stamped = apply_configuration(pipeline, prefs) roundtripped, _ = pipeline_dict_to_ir(json.loads(json.dumps(_pipeline_to_dict(stamped), default=str))) - assert roundtripped.translation_preferences.copy_activity_paradigm is CopyActivityParadigm.SDP + assert roundtripped.translation_configuration.copy_activity_paradigm is CopyActivityParadigm.SDP assert roundtripped.tasks[0].target_format == "sdp" assert roundtripped.tasks[0].use_lakeflow_connector is True assert roundtripped.tasks[0].compute_mode == COMPUTE_MODE_CLASSIC_MULTI_NODE @@ -526,26 +524,26 @@ def test_preferences_survive_json_roundtrip(self): class TestMigrationInputSession: - def test_ingest_session_lists_expected_questions(self): + def test_discover_session_lists_expected_options(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="ingest") - ids = [q.question_id for q in session.pending().questions] + session = MigrationInputSession(phase="discover") + ids = [q.option_id for q in session.pending().options] assert ids == ["adf_source_path", "adf_resource_url", "output_dir"] - def test_translate_session_lists_expected_questions(self): + def test_convert_session_lists_expected_options(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="translate") - ids = [q.question_id for q in session.pending().questions] + session = MigrationInputSession(phase="convert") + ids = [q.option_id for q in session.pending().options] assert "inventory_path" in ids assert "adf_source_path" in ids - def test_prepare_session_lists_expected_questions(self): + def test_package_session_lists_expected_options(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="prepare") - ids = {q.question_id for q in session.pending().questions} + session = MigrationInputSession(phase="package") + ids = {q.option_id for q in session.pending().options} assert {"translation_report_path", "output_bundle_path", "catalog", "schema"} <= ids def test_unknown_phase_raises(self): @@ -557,22 +555,22 @@ def test_unknown_phase_raises(self): def test_answer_records_value_and_drops_from_pending(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="ingest") + session = MigrationInputSession(phase="discover") session.answer("adf_source_path", "/Volumes/main/default/adf") - ids = [q.question_id for q in session.pending().questions] + ids = [q.option_id for q in session.pending().options] assert "adf_source_path" not in ids - def test_answer_rejects_unknown_question(self): + def test_answer_rejects_unknown_option(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="ingest") - with pytest.raises(ValueError, match="Unknown input question"): + session = MigrationInputSession(phase="discover") + with pytest.raises(ValueError, match="Unknown input option"): session.answer("not_a_field", "x") def test_collected_merges_answers_with_defaults(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="prepare") + session = MigrationInputSession(phase="package") session.answer("translation_report_path", "/tmp/report.json") collected = session.collected() assert collected["translation_report_path"] == "/tmp/report.json" @@ -582,10 +580,10 @@ def test_collected_merges_answers_with_defaults(self): def test_collected_omits_required_when_missing(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="ingest") + session = MigrationInputSession(phase="discover") collected = session.collected() assert "adf_source_path" not in collected - assert collected["output_dir"] == "./orchestra_output/ingest" + assert collected["output_dir"] == "./flowx_output" class TestWorkspacePathsCli: @@ -650,26 +648,26 @@ def test_workspace_paths_suggests_host_from_databricks_linked_service(self, tmp_ class TestInputsCli: - def test_inputs_emits_ingest_questions(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): - exit_code = adapter_cli_main(["inputs", "ingest"]) + def test_inputs_emits_discover_options(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): + exit_code = adapter_cli_main(["inputs", "discover"]) assert exit_code == 0 payload = json.loads(capsys.readouterr().out) - assert payload["phase"] == "ingest" - ids = [q["question_id"] for q in payload["questions"]] + assert payload["phase"] == "discover" + ids = [q["option_id"] for q in payload["options"]] assert ids == ["adf_source_path", "adf_resource_url", "output_dir"] def test_inputs_writes_to_file(self, tmp_path: Path): - out = tmp_path / "questions.json" - exit_code = adapter_cli_main(["inputs", "prepare", "--out", str(out)]) + out = tmp_path / "options.json" + exit_code = adapter_cli_main(["inputs", "package", "--out", str(out)]) assert exit_code == 0 payload = json.loads(out.read_text()) - assert payload["phase"] == "prepare" - ids = {q["question_id"] for q in payload["questions"]} + assert payload["phase"] == "package" + ids = {q["option_id"] for q in payload["options"]} assert "output_bundle_path" in ids class TestCli: - def test_inspect_emits_pending_questions(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): + def test_inspect_emits_pending_options(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): from flowx.translator.engine import _pipeline_to_dict pipeline = Pipeline(name="p", tasks=[_delta_copy()]) @@ -679,30 +677,38 @@ def test_inspect_emits_pending_questions(self, tmp_path: Path, capsys: pytest.Ca assert exit_code == 0 payload = json.loads(capsys.readouterr().out) assert payload["pipelines"][0]["pipeline_name"] == "p" - question_ids = {q["question_id"] for q in payload["pipelines"][0]["questions"]} - assert QUESTION_COPY_ACTIVITY_PARADIGM in question_ids + option_ids = {q["option_id"] for q in payload["pipelines"][0]["options"]} + assert OPTION_COPY_ACTIVITY_PARADIGM in option_ids - def test_modify_stamps_preferences(self, tmp_path: Path): + def test_modify_stamps_configuration(self, tmp_path: Path): from flowx.translator.engine import _pipeline_to_dict pipeline = Pipeline(name="p", tasks=[_delta_copy()]) report_path = tmp_path / "report.json" report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) - answers_path = tmp_path / "answers.json" - answers_path.write_text( - json.dumps( - { - "copy_activity_paradigm": "sdp", - "non_databricks_task_compute": "classic", - "use_lakeflow_connectors": "lakeflow_connect", - } - ) - ) out_path = tmp_path / "modified.json" - exit_code = adapter_cli_main(["modify", str(report_path), str(answers_path), "--out", str(out_path)]) + exit_code = adapter_cli_main( + [ + "modify", + str(report_path), + "--answer", + "copy_activity_paradigm=sdp", + "--answer", + "non_databricks_task_compute=classic", + "--answer", + "use_lakeflow_connectors=lakeflow_connect", + "--out", + str(out_path), + "--config-out", + str(tmp_path / "configuration.json"), + ] + ) assert exit_code == 0 modified = json.loads(out_path.read_text()) - assert modified["translation_preferences"]["copy_activity_paradigm"] == "sdp" + # The collected answers are persisted verbatim as configuration.json. + config = json.loads((tmp_path / "configuration.json").read_text()) + assert config["copy_activity_paradigm"] == "sdp" + assert modified["translation_configuration"]["copy_activity_paradigm"] == "sdp" copy_task = next(task for task in modified["tasks"] if task["task_key"] == "copy_to_delta") assert copy_task["target_format"] == "sdp" assert copy_task["compute_mode"] == COMPUTE_MODE_CLASSIC_MULTI_NODE @@ -735,28 +741,23 @@ def test_modify_threads_lookup_values_into_metadata_driven_motif(self, tmp_path: pipeline = Pipeline(name="p", tasks=[motif]) report_path = tmp_path / "report.json" report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) - answers_path = tmp_path / "answers.json" - answers_path.write_text( - json.dumps( - { - "metadata_driven_consolidate": "consolidate", - "metadata_driven_access": "yes", - "metadata_driven_size": "small", - } - ) - ) - lookup_values_path = tmp_path / "lookup_values.json" - lookup_values_path.write_text(json.dumps([{"source_table": "orders"}])) out_path = tmp_path / "modified.json" exit_code = adapter_cli_main( [ "modify", str(report_path), - str(answers_path), - "--lookup-values", - str(lookup_values_path), + "--answer", + "metadata_driven_consolidate=consolidate", + "--answer", + "metadata_driven_access=yes", + "--answer", + "metadata_driven_size=small", + "--lookup-csv", + "source_table\norders", "--out", str(out_path), + "--config-out", + str(tmp_path / "configuration.json"), ] ) assert exit_code == 0 @@ -771,12 +772,112 @@ def test_modify_rejects_invalid_answer(self, tmp_path: Path): pipeline = Pipeline(name="p", tasks=[_delta_copy()]) report_path = tmp_path / "report.json" report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) - answers_path = tmp_path / "answers.json" - answers_path.write_text(json.dumps({"copy_activity_paradigm": "yaml"})) out_path = tmp_path / "modified.json" - exit_code = adapter_cli_main(["modify", str(report_path), str(answers_path), "--out", str(out_path)]) + exit_code = adapter_cli_main( + ["modify", str(report_path), "--answer", "copy_activity_paradigm=yaml", "--out", str(out_path)] + ) + assert exit_code == 2 + + def test_modify_output_dir_convention_writes_work_and_metadata(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + report_path = tmp_path / ".work" / "translation_report.json" + report_path.parent.mkdir(parents=True) + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + exit_code = adapter_cli_main( + ["modify", str(report_path), "--output-dir", str(tmp_path), "--answer", "copy_activity_paradigm=sdp"] + ) + assert exit_code == 0 + # Stamped IR lands in the transient .work/, configuration.json in metadata/. + assert (tmp_path / ".work" / "translation_report.stamped.json").exists() + config = json.loads((tmp_path / "metadata" / "configuration.json").read_text()) + assert config == {"copy_activity_paradigm": "sdp"} + + def test_modify_requires_output_dir_or_out(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + exit_code = adapter_cli_main(["modify", str(report_path), "--answer", "copy_activity_paradigm=sdp"]) assert exit_code == 2 + def test_inspect_emits_full_schema_with_show_when(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): + """inspect returns the whole option tree at once; follow-ups carry a show_when condition + the agent evaluates locally (no per-follow-up round trip).""" + from flowx.models.ir import CopyActivity, Dependency, WebActivity + from flowx.translator.engine import _pipeline_to_dict + + copy = CopyActivity(name="Load", task_key="load") + notify = WebActivity( + name="Notify", + task_key="notify", + url="https://x", + method="POST", + depends_on=[Dependency(task_key="load", outcome="Failed")], + ) + pipeline = Pipeline(name="p", tasks=[copy, notify]) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + + assert adapter_cli_main(["inspect", str(report_path)]) == 0 + options = {o["option_id"]: o for o in json.loads(capsys.readouterr().out)["pipelines"][0]["options"]} + + # The full chain is present up front, not gated behind an answer. + assert "notify_destination" in options + assert "notify_email_recipients" in options + # The destination question is unconditional; the email follow-up is gated by show_when. + assert options["notify_destination"]["show_when"] == [] + assert options["notify_email_recipients"]["show_when"] == [{"option_id": "notify_destination", "in": ["email"]}] + # Free-text follow-up vs. enum option. + assert options["notify_email_recipients"]["free_text"] is True + assert [c["value"] for c in options["notify_destination"]["choices"]][0] == "keep" + + def test_inspect_rejects_malformed_answer(self, tmp_path: Path): + from flowx.translator.engine import _pipeline_to_dict + + pipeline = Pipeline(name="p", tasks=[_delta_copy()]) + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + # Missing '=' -> validation error -> exit 2. + assert adapter_cli_main(["inspect", str(report_path), "--answer", "no_equals_sign"]) == 2 + + def test_record_results_subcommand(self, tmp_path: Path, monkeypatch, capsys: pytest.CaptureFixture[str]): + import flowx.reporting.results as rr + + md = tmp_path / "metadata" + md.mkdir() + (md / "inventory.json").write_text("{}") + monkeypatch.setattr(rr, "write_results", lambda *a, **k: ("run-xyz", 3)) + rc = adapter_cli_main(["record-results", "--output-dir", str(tmp_path), "--results-table", "c.s.t"]) + assert rc == 0 + out = capsys.readouterr().out + assert "run-xyz" in out and "3 pipeline" in out + + def test_record_results_requires_inventory(self, tmp_path: Path): + rc = adapter_cli_main(["record-results", "--output-dir", str(tmp_path), "--results-table", "c.s.t"]) + assert rc == 1 # no metadata/inventory.json + + def test_install_dashboard_subcommand(self, monkeypatch, capsys: pytest.CaptureFixture[str]): + import flowx.reporting.dashboard as dd + + monkeypatch.setattr(dd, "install_dashboard", lambda *a, **k: ("dash-1", "https://x/sql/dashboardsv3/dash-1")) + rc = adapter_cli_main(["install-dashboard", "--results-table", "c.s.t"]) + assert rc == 0 + out = capsys.readouterr().out + assert "dash-1" in out + + def test_install_dashboard_failure_returns_1(self, monkeypatch): + import flowx.reporting.dashboard as dd + + def _boom(*a, **k): + raise RuntimeError("no auth") + + monkeypatch.setattr(dd, "install_dashboard", _boom) + rc = adapter_cli_main(["install-dashboard", "--results-table", "c.s.t"]) + assert rc == 1 + class TestBundleOutput: def test_classic_copy_compute_emits_two_node_multi_node_cluster(self, tmp_path: Path): @@ -786,8 +887,8 @@ def test_classic_copy_compute_emits_two_node_multi_node_cluster(self, tmp_path: from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - prefs = TranslationPreferences(non_databricks_task_compute="classic") - stamped = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(non_databricks_task_compute="classic") + stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) @@ -805,8 +906,8 @@ def test_classic_single_node_cluster_uses_is_single_node_flag(self, tmp_path: Pa from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[WaitActivity(**_make_base("w"), wait_time_seconds=1)]) - prefs = TranslationPreferences(non_databricks_task_compute="classic") - stamped = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(non_databricks_task_compute="classic") + stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) @@ -825,7 +926,7 @@ def test_serverless_default_emits_no_job_clusters(self, tmp_path: Path): from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - stamped = apply_preferences(pipeline, TranslationPreferences()) + stamped = apply_configuration(pipeline, TranslationConfiguration()) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) @@ -836,7 +937,7 @@ def test_sdp_copy_emits_pyspark_pipelines_table_scaffold(self, tmp_path: Path): from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - stamped = apply_preferences(pipeline, TranslationPreferences(copy_activity_paradigm="sdp")) + stamped = apply_configuration(pipeline, TranslationConfiguration(copy_activity_paradigm="sdp")) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) notebook_path = tmp_path / "src" / "notebooks" / "copy_a.py" @@ -852,19 +953,39 @@ def test_lakeflow_connect_emits_pipeline_resource_and_no_notebook(self, tmp_path from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - stamped = apply_preferences(pipeline, TranslationPreferences(use_lakeflow_connectors="lakeflow_connect")) + stamped = apply_configuration(pipeline, TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect")) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) assert not (tmp_path / "src" / "notebooks" / "copy_a.py").exists() - pipeline_yml = tmp_path / "resources" / "pipelines" / "copy_a_lfc.yml" + pipeline_yml = tmp_path / "resources" / "copy_a_lfc.yml" assert pipeline_yml.exists() resource = yaml.safe_load(pipeline_yml.read_text()) lfc = resource["resources"]["pipelines"]["copy_a_lfc"] assert lfc["name"] == "copy_a_lfc" - assert lfc["ingestion_definition"]["connection_name"] == "orchestra_copy_a_connection" + assert lfc["ingestion_definition"]["connection_name"] == "flowx_copy_a_connection" objects = lfc["ingestion_definition"]["objects"] assert objects[0]["table"]["destination_table"] == "raw.events" + def test_lakeflow_connect_declares_referenced_source_variables(self, tmp_path: Path): + import yaml + + from flowx.bundler.dab_writer import write_bundle + from flowx.preparer.workflow_preparer import prepare_workflow + + pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) + stamped = apply_configuration(pipeline, TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect")) + workflow = prepare_workflow(stamped) + write_bundle(workflow, tmp_path, catalog="migration_cat", schema="migration_schema") + + resource = yaml.safe_load((tmp_path / "resources" / "copy_a_lfc.yml").read_text()) + table = resource["resources"]["pipelines"]["copy_a_lfc"]["ingestion_definition"]["objects"][0]["table"] + assert table["source_catalog"] == "${var.source_catalog}" + assert table["source_schema"] == "${var.source_schema}" + + variables = yaml.safe_load((tmp_path / "databricks.yml").read_text())["variables"] + assert variables["source_catalog"]["default"] == "migration_cat" + assert variables["source_schema"]["default"] == "migration_schema" + def test_lakeflow_connect_job_task_references_pipeline(self, tmp_path: Path): import yaml @@ -872,7 +993,7 @@ def test_lakeflow_connect_job_task_references_pipeline(self, tmp_path: Path): from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - stamped = apply_preferences(pipeline, TranslationPreferences(use_lakeflow_connectors="lakeflow_connect")) + stamped = apply_configuration(pipeline, TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect")) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) @@ -880,79 +1001,77 @@ def test_lakeflow_connect_job_task_references_pipeline(self, tmp_path: Path): assert "notebook_task" not in task assert task["pipeline_task"]["pipeline_id"] == "${resources.pipelines.copy_a_lfc.id}" - def test_metadata_driven_consolidate_question_surfaces_for_motif(self): + def test_metadata_driven_consolidate_option_surfaces_for_motif(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) - ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_METADATA_DRIVEN_CONSOLIDATE in ids + ids = {q.option_id for q in gather_options(pipeline).options} + assert OPTION_METADATA_DRIVEN_CONSOLIDATE in ids - def test_metadata_driven_followup_questions_gated_on_consolidate(self): + def test_metadata_driven_followup_options_gated_on_consolidate(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) - first_pass = gather_questions(pipeline).questions - ids = {q.question_id for q in first_pass} - assert QUESTION_METADATA_DRIVEN_CONSOLIDATE in ids - assert QUESTION_METADATA_DRIVEN_ACCESS not in ids - assert QUESTION_METADATA_DRIVEN_SIZE not in ids - assert QUESTION_METADATA_DRIVEN_LOOKUP_TOOL not in ids - - keep_pass = gather_questions(pipeline, answers={QUESTION_METADATA_DRIVEN_CONSOLIDATE: "keep"}).questions - keep_ids = {q.question_id for q in keep_pass} - assert QUESTION_METADATA_DRIVEN_ACCESS not in keep_ids - - consolidate_pass = gather_questions( - pipeline, answers={QUESTION_METADATA_DRIVEN_CONSOLIDATE: "consolidate"} - ).questions - consolidate_ids = {q.question_id for q in consolidate_pass} - assert QUESTION_METADATA_DRIVEN_ACCESS in consolidate_ids - assert QUESTION_METADATA_DRIVEN_SIZE in consolidate_ids - assert QUESTION_METADATA_DRIVEN_LOOKUP_TOOL not in consolidate_ids - - def test_metadata_driven_lookup_tool_question_gated_on_access(self): + first_pass = gather_options(pipeline).options + ids = {q.option_id for q in first_pass} + assert OPTION_METADATA_DRIVEN_CONSOLIDATE in ids + assert OPTION_METADATA_DRIVEN_ACCESS not in ids + assert OPTION_METADATA_DRIVEN_SIZE not in ids + assert OPTION_METADATA_DRIVEN_LOOKUP_TOOL not in ids + + keep_pass = gather_options(pipeline, answers={OPTION_METADATA_DRIVEN_CONSOLIDATE: "keep"}).options + keep_ids = {q.option_id for q in keep_pass} + assert OPTION_METADATA_DRIVEN_ACCESS not in keep_ids + + consolidate_pass = gather_options(pipeline, answers={OPTION_METADATA_DRIVEN_CONSOLIDATE: "consolidate"}).options + consolidate_ids = {q.option_id for q in consolidate_pass} + assert OPTION_METADATA_DRIVEN_ACCESS in consolidate_ids + assert OPTION_METADATA_DRIVEN_SIZE in consolidate_ids + assert OPTION_METADATA_DRIVEN_LOOKUP_TOOL not in consolidate_ids + + def test_metadata_driven_lookup_tool_option_gated_on_access(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) answers = { - QUESTION_METADATA_DRIVEN_CONSOLIDATE: "consolidate", - QUESTION_METADATA_DRIVEN_ACCESS: "yes", + OPTION_METADATA_DRIVEN_CONSOLIDATE: "consolidate", + OPTION_METADATA_DRIVEN_ACCESS: "yes", } - pending = gather_questions(pipeline, answers=answers).questions - ids = {q.question_id for q in pending} - assert QUESTION_METADATA_DRIVEN_LOOKUP_TOOL in ids + pending = gather_options(pipeline, answers=answers).options + ids = {q.option_id for q in pending} + assert OPTION_METADATA_DRIVEN_LOOKUP_TOOL in ids def test_modifier_consolidates_metadata_driven_when_size_is_small(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( metadata_driven_consolidate="consolidate", metadata_driven_access="yes", metadata_driven_size="small", ) - modified = apply_preferences(pipeline, prefs) + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].consolidate_metadata_driven is True def test_modifier_does_not_consolidate_when_size_is_large(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( metadata_driven_consolidate="consolidate", metadata_driven_access="yes", metadata_driven_size="large", ) - modified = apply_preferences(pipeline, prefs) + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].consolidate_metadata_driven is False def test_modifier_does_not_consolidate_when_access_is_no(self): pipeline = Pipeline(name="p", tasks=[_metadata_driven_motif()]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( metadata_driven_consolidate="consolidate", metadata_driven_access="no", metadata_driven_size="small", ) - modified = apply_preferences(pipeline, prefs) + modified = apply_configuration(pipeline, prefs) assert modified.tasks[0].consolidate_metadata_driven is False - def test_lakeflow_connector_type_question_suppressed_when_only_query_copies(self): + def test_lakeflow_connector_type_option_suppressed_when_only_query_copies(self): pipeline = Pipeline(name="p", tasks=[_query_delta_copy("copy_q")]) - ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_USE_LAKEFLOW_CONNECTORS in ids - assert QUESTION_LAKEFLOW_CONNECTOR_TYPE not in ids + ids = {q.option_id for q in gather_options(pipeline).options} + assert OPTION_USE_LAKEFLOW_CONNECTORS in ids + assert OPTION_LAKEFLOW_CONNECTOR_TYPE not in ids - def test_lakeflow_connector_type_question_suppressed_per_copy_eligibility(self): + def test_lakeflow_connector_type_option_suppressed_per_copy_eligibility(self): """Per-Copy eligibility determines connector type with no overlap. Table-based reads can only use CDC (no cursor column) and queries @@ -960,24 +1079,24 @@ def test_lakeflow_connector_type_question_suppressed_per_copy_eligibility(self): eligible connector per Copy and the prompt is suppressed. """ pipeline = Pipeline(name="p", tasks=[_delta_copy("copy_a")]) - ids = {q.question_id for q in gather_questions(pipeline).questions} - assert QUESTION_LAKEFLOW_CONNECTOR_TYPE not in ids + ids = {q.option_id for q in gather_options(pipeline).options} + assert OPTION_LAKEFLOW_CONNECTOR_TYPE not in ids - def test_query_copy_routes_to_query_based_connector_regardless_of_preference(self, tmp_path: Path): + def test_query_copy_routes_to_query_based_connector_regardless_of_configuration(self, tmp_path: Path): import yaml from flowx.bundler.dab_writer import write_bundle from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_query_delta_copy("copy_q")]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( use_lakeflow_connectors="lakeflow_connect", lakeflow_connector_type="cdc", ) - stamped = apply_preferences(pipeline, prefs) + stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) - resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_q_lfc.yml").read_text()) + resource = yaml.safe_load((tmp_path / "resources" / "copy_q_lfc.yml").read_text()) objects = resource["resources"]["pipelines"]["copy_q_lfc"]["ingestion_definition"]["objects"] assert "table_configuration" in objects[0] table_config = objects[0]["table_configuration"] @@ -994,23 +1113,23 @@ def test_table_copy_uses_cdc_connector_by_default(self, tmp_path: Path): from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - stamped = apply_preferences(pipeline, prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) - resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_a_lfc.yml").read_text()) + resource = yaml.safe_load((tmp_path / "resources" / "copy_a_lfc.yml").read_text()) objects = resource["resources"]["pipelines"]["copy_a_lfc"]["ingestion_definition"]["objects"] assert "table" in objects[0] assert "table_configuration" not in objects[0] - def test_table_copy_with_query_based_preference_routes_to_cdc(self, tmp_path: Path): + def test_table_copy_with_query_based_configuration_routes_to_cdc(self, tmp_path: Path): """LFC query-based requires a cursor column. Table-based Copies have none. Per the Lakeflow Connect query-based-overview docs, the connector requires a cursor column to drive incremental ingestion. When the user prefers query_based but the Copy is table-based (no query, no cursor candidate), the modifier honours the - eligibility rules over the preference and routes to CDC. + eligibility rules over the configuration and routes to CDC. """ import yaml @@ -1018,14 +1137,14 @@ def test_table_copy_with_query_based_preference_routes_to_cdc(self, tmp_path: Pa from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( use_lakeflow_connectors="lakeflow_connect", lakeflow_connector_type="query_based", ) - stamped = apply_preferences(pipeline, prefs) + stamped = apply_configuration(pipeline, prefs) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) - resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_a_lfc.yml").read_text()) + resource = yaml.safe_load((tmp_path / "resources" / "copy_a_lfc.yml").read_text()) objects = resource["resources"]["pipelines"]["copy_a_lfc"]["ingestion_definition"]["objects"] assert "table" in objects[0] assert "table_configuration" not in objects[0] @@ -1040,12 +1159,12 @@ def test_consolidated_metadata_driven_motif_emits_single_pipeline(self, tmp_path motif = _metadata_driven_motif() pipeline = Pipeline(name="job", tasks=[motif]) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( metadata_driven_consolidate="consolidate", metadata_driven_access="yes", metadata_driven_size="medium", ) - stamped = apply_preferences(pipeline, prefs) + stamped = apply_configuration(pipeline, prefs) consolidated_motif = dataclasses.replace( stamped.tasks[0], lookup_values=[ @@ -1056,7 +1175,7 @@ def test_consolidated_metadata_driven_motif_emits_single_pipeline(self, tmp_path stamped = dataclasses.replace(stamped, tasks=[consolidated_motif]) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) - resource_path = tmp_path / "resources" / "pipelines" / "motif_metadata_driven_bulk_copy_consolidated.yml" + resource_path = tmp_path / "resources" / "motif_metadata_driven_bulk_copy_consolidated.yml" assert resource_path.exists() resource = yaml.safe_load(resource_path.read_text()) pipeline_def = resource["resources"]["pipelines"]["motif_metadata_driven_bulk_copy_consolidated"] @@ -1065,7 +1184,7 @@ def test_consolidated_metadata_driven_motif_emits_single_pipeline(self, tmp_path assert objects[0]["table"]["source_table"] == "orders" assert objects[1]["table"]["source_table"] == "customers" - def test_table_based_copy_with_query_based_preference_falls_back_to_cdc(self, tmp_path: Path): + def test_table_based_copy_with_query_based_configuration_falls_back_to_cdc(self, tmp_path: Path): """Table-based reads have no cursor column, so query-based isn't eligible. When the user prefers query_based but the only eligible LFC connector @@ -1090,13 +1209,13 @@ def test_table_based_copy_with_query_based_preference_falls_back_to_cdc(self, tm "connection": {"host": "flowx-test-sql.database.windows.net", "port": 1433}, }, ) - prefs = TranslationPreferences( + prefs = TranslationConfiguration( use_lakeflow_connectors="lakeflow_connect", lakeflow_connector_type="query_based", ) - stamped = apply_preferences(Pipeline(name="job", tasks=[copy]), prefs) + stamped = apply_configuration(Pipeline(name="job", tasks=[copy]), prefs) write_bundle(prepare_workflow(stamped), tmp_path) - resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / "copy_customers_lfc.yml").read_text()) + resource = yaml.safe_load((tmp_path / "resources" / "copy_customers_lfc.yml").read_text()) obj = resource["resources"]["pipelines"]["copy_customers_lfc"]["ingestion_definition"]["objects"][0] assert "table" in obj assert obj["table"]["destination_table"] == "customers" @@ -1116,8 +1235,8 @@ def test_lakeflow_connect_uses_resolved_host_from_linked_service(self, tmp_path: "connection": {"host": "flowx-test-sql.database.windows.net", "port": 1433}, }, ) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - stamped = apply_preferences(Pipeline(name="job", tasks=[copy]), prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + stamped = apply_configuration(Pipeline(name="job", tasks=[copy]), prefs) write_bundle(prepare_workflow(stamped), tmp_path) body = (tmp_path / "src" / "setup" / "create_connections.py").read_text() assert "flowx-test-sql.database.windows.net" in body @@ -1151,17 +1270,17 @@ def test_lakeflow_connect_dedupes_connection_across_copies(self, tmp_path: Path) sink_properties={"table": "orders"}, source_properties={**shared_source, "source_table": "orders"}, ) - prefs = TranslationPreferences(use_lakeflow_connectors="lakeflow_connect") - stamped = apply_preferences(Pipeline(name="job", tasks=[copy_a, copy_b]), prefs) + prefs = TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect") + stamped = apply_configuration(Pipeline(name="job", tasks=[copy_a, copy_b]), prefs) write_bundle(prepare_workflow(stamped), tmp_path) body = (tmp_path / "src" / "setup" / "create_connections.py").read_text() assert body.count("CREATE CONNECTION IF NOT EXISTS") == 1 - assert body.count("orchestra_LS_AzureSqlDb_connection") >= 1 + assert body.count("flowx_LS_AzureSqlDb_connection") >= 1 for pipeline_file in ("copy_a_lfc.yml", "copy_b_lfc.yml"): - resource = yaml.safe_load((tmp_path / "resources" / "pipelines" / pipeline_file).read_text()) + resource = yaml.safe_load((tmp_path / "resources" / pipeline_file).read_text()) key = pipeline_file.replace(".yml", "") assert resource["resources"]["pipelines"][key]["ingestion_definition"]["connection_name"] == ( - "orchestra_LS_AzureSqlDb_connection" + "flowx_LS_AzureSqlDb_connection" ) def test_lakeflow_connect_emits_connection_setup_notebook(self, tmp_path: Path): @@ -1169,13 +1288,13 @@ def test_lakeflow_connect_emits_connection_setup_notebook(self, tmp_path: Path): from flowx.preparer.workflow_preparer import prepare_workflow pipeline = Pipeline(name="job", tasks=[_delta_copy("copy_a")]) - stamped = apply_preferences(pipeline, TranslationPreferences(use_lakeflow_connectors="lakeflow_connect")) + stamped = apply_configuration(pipeline, TranslationConfiguration(use_lakeflow_connectors="lakeflow_connect")) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) setup_notebook = tmp_path / "src" / "setup" / "create_connections.py" assert setup_notebook.exists() body = setup_notebook.read_text() - assert "orchestra_copy_a_connection" in body + assert "flowx_copy_a_connection" in body assert "SQLSERVER" in body def test_existing_default_binds_to_default_cluster(self, tmp_path: Path): @@ -1188,7 +1307,7 @@ def test_existing_default_binds_to_default_cluster(self, tmp_path: Path): name="job", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/existing")], ) - stamped = apply_preferences(pipeline, TranslationPreferences()) + stamped = apply_configuration(pipeline, TranslationConfiguration()) workflow = prepare_workflow(stamped) write_bundle(workflow, tmp_path) job_yml = yaml.safe_load((tmp_path / "resources" / "job.yml").read_text()) diff --git a/tests/unit/test_adf_loader.py b/tests/unit/test_adf_loader.py index 678c24a..f56b4d8 100644 --- a/tests/unit/test_adf_loader.py +++ b/tests/unit/test_adf_loader.py @@ -13,6 +13,7 @@ _parse_pipeline_json, build_inventory, classify_activity, + clear_stale_outputs, load_adf_definitions, ) @@ -302,3 +303,52 @@ def test_normalize_arm_passthrough(self): data = {"name": "simple", "properties": {"activities": []}} result = _normalize_arm(data) assert result is data + + +# --------------------------------------------------------------------------- +# clear_stale_outputs +# --------------------------------------------------------------------------- + + +class TestClearStaleOutputs: + """Discover must reset a reused output_dir so prior runs don't leak into the bundle.""" + + def test_removes_prior_run_artifacts(self, tmp_path): + """Stale per-pipeline metadata and a prior generated bundle are removed.""" + (tmp_path / "metadata").mkdir() + (tmp_path / "metadata" / "OldPipeline.arm.json").write_text("{}", encoding="utf-8") + (tmp_path / "resources").mkdir() + (tmp_path / "resources" / "old_pipeline.yml").write_text("name: old", encoding="utf-8") + (tmp_path / "src" / "notebooks").mkdir(parents=True) + (tmp_path / "src" / "notebooks" / "old.py").write_text("print('old')", encoding="utf-8") + (tmp_path / ".work").mkdir() + (tmp_path / ".work" / "translation_report.json").write_text("{}", encoding="utf-8") + (tmp_path / "databricks.yml").write_text("bundle: old", encoding="utf-8") + (tmp_path / "SETUP.md").write_text("# old", encoding="utf-8") + (tmp_path / "WARNINGS.md").write_text("# old", encoding="utf-8") + + clear_stale_outputs(tmp_path) + + assert not (tmp_path / "metadata").exists() + assert not (tmp_path / "resources").exists() + assert not (tmp_path / "src").exists() + assert not (tmp_path / ".work").exists() + assert not (tmp_path / "databricks.yml").exists() + assert not (tmp_path / "SETUP.md").exists() + assert not (tmp_path / "WARNINGS.md").exists() + + def test_preserves_unrelated_files(self, tmp_path): + """Only flowx-managed entries are removed; unrelated files stay put.""" + (tmp_path / "notes.txt").write_text("keep me", encoding="utf-8") + (tmp_path / "user_data").mkdir() + (tmp_path / "user_data" / "keep.csv").write_text("a,b", encoding="utf-8") + + clear_stale_outputs(tmp_path) + + assert (tmp_path / "notes.txt").read_text(encoding="utf-8") == "keep me" + assert (tmp_path / "user_data" / "keep.csv").exists() + + def test_idempotent_on_empty_dir(self, tmp_path): + """Clearing a directory with no flowx artifacts is a no-op (no error).""" + clear_stale_outputs(tmp_path) + assert list(tmp_path.iterdir()) == [] diff --git a/tests/unit/test_bundle_invariants.py b/tests/unit/test_bundle_invariants.py new file mode 100644 index 0000000..aed8ebc --- /dev/null +++ b/tests/unit/test_bundle_invariants.py @@ -0,0 +1,61 @@ +"""Unit tests for bundle structural-invariant checks (guard #2).""" + +from __future__ import annotations + +from flowx.validate.bundle_invariants import check_job, check_resource_text + + +def _codes(findings) -> set[str]: + return {f.code for f in findings} + + +def test_clean_job_has_no_findings(): + job = { + "name": "p", + "parameters": [{"name": "region", "default": "us"}], + "tasks": [ + { + "task_key": "a", + "notebook_task": {"notebook_path": "/n", "base_parameters": {"region": "{{job.parameters.region}}"}}, + }, + {"task_key": "b", "depends_on": [{"task_key": "a"}], "notebook_task": {"notebook_path": "/n"}}, + ], + } + assert check_job("p", job) == [] + + +def test_duplicate_job_parameter_flagged(): + job = {"parameters": [{"name": "region", "default": "us"}, {"name": "region", "default": "us"}], "tasks": []} + assert "duplicate_job_parameter" in _codes(check_job("p", job)) + + +def test_duplicate_task_key_flagged(): + job = {"tasks": [{"task_key": "a"}, {"task_key": "a"}]} + assert "duplicate_task_key" in _codes(check_job("p", job)) + + +def test_undeclared_job_parameter_reference_flagged(): + job = { + "parameters": [{"name": "region"}], + "tasks": [{"task_key": "a", "notebook_task": {"base_parameters": {"env": "{{job.parameters.env}}"}}}], + } + codes = _codes(check_job("p", job)) + assert "undeclared_job_parameter" in codes # env is referenced but not declared + + +def test_dangling_depends_on_flagged(): + job = {"tasks": [{"task_key": "a", "depends_on": [{"task_key": "ghost"}]}]} + assert "dangling_depends_on" in _codes(check_job("p", job)) + + +def test_yaml_anchor_smell_flagged(): + # The exact shape PyYAML emits when the same object is in a list twice. + text = ( + "resources:\n jobs:\n p:\n name: p\n tasks: []\n" + " parameters:\n - &id001\n name: region\n default: us\n - *id001\n" + ) + findings = check_resource_text(text, filename="p.yml") + codes = _codes(findings) + assert "yaml_anchor" in codes + # and the parsed structure also trips the duplicate-parameter invariant + assert "duplicate_job_parameter" in codes diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index 4cfa416..90ecb02 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -951,6 +951,22 @@ def test_connection_setup_notebook(self): nb = notebooks[0] assert nb.relative_path == "setup/create_connections.py" assert "sql_conn" in nb.content + assert "user 'PLACEHOLDER_USER'" in nb.content + assert "password 'PLACEHOLDER_PASSWORD'" in nb.content + + def test_connection_setup_notebook_non_sqlserver_omits_credentials(self): + from flowx.bundler.setup_generator import generate_setup_tasks + + setup_tasks = [ + SetupTask( + type="connection", + config={"connection_name": "my_conn", "connection_type": "MYSQL", "host": "mysql.example.com"}, + ), + ] + notebooks = generate_setup_tasks(secrets=[], setup_tasks=setup_tasks, catalog="main", schema="default") + content = notebooks[0].content + assert "user '" not in content + assert "password '" not in content def test_no_setup_when_empty(self): from flowx.bundler.setup_generator import generate_setup_tasks diff --git a/tests/unit/test_code_generator.py b/tests/unit/test_code_generator.py index 4e4b237..eb98fc1 100644 --- a/tests/unit/test_code_generator.py +++ b/tests/unit/test_code_generator.py @@ -284,7 +284,7 @@ def test_file_source_lookup_emits_spark_read(self): # File-source branch: no spark.sql(''), uses spark.read.format().load(). assert "spark.sql" not in content assert "spark.read.format('json')" in content - assert ".option(\"multiline\", \"true\")" in content + assert '.option("multiline", "true")' in content assert "source_path" in content @@ -351,9 +351,7 @@ def test_auth_block_msi_raises_not_implemented(self): method="GET", authentication={"type": auth_type, "resource": "https://management.azure.com"}, ) - content = generate_web_activity_notebook( - activity, scope=f"msi_api_{auth_type.lower()}" - ) + content = generate_web_activity_notebook(activity, scope=f"msi_api_{auth_type.lower()}") _assert_valid_python(content, f"msi_api ({auth_type})") assert "auth-credential" not in content assert "NotImplementedError" in content diff --git a/tests/unit/test_dag_equivalence.py b/tests/unit/test_dag_equivalence.py new file mode 100644 index 0000000..4957c9b --- /dev/null +++ b/tests/unit/test_dag_equivalence.py @@ -0,0 +1,220 @@ +"""Tests for the motif-aware Tier-0 DAG equivalence check.""" + +from __future__ import annotations + +from flowx.models.adf_ast import AdfActivity, AdfDependency, AdfPipeline +from flowx.models.ir import Activity, Dependency, Pipeline +from flowx.models.motifs import ( + MOTIF_ACTIVITY_AND_NOTIFY, + MOTIF_METADATA_DRIVEN_BULK_COPY, + DetectedMotif, +) +from flowx.motifs.collapser import collapse_motifs +from flowx.utils import normalize_task_key +from flowx.validate import check_dag_equivalence, format_result + + +def _adf(name: str, deps: dict[str, list[str]] | None = None, adf_type: str = "Copy") -> AdfActivity: + """ADF activity; *deps* maps upstream name -> dependency conditions.""" + depends = [AdfDependency(activity=u, dependency_conditions=c) for u, c in (deps or {}).items()] + return AdfActivity(name=name, type=adf_type, depends_on=depends or None) + + +def _task(name: str, deps: list[tuple[str, str]] | None = None) -> Activity: + """IR leaf task; *deps* is a list of (upstream_name, outcome).""" + edges = [Dependency(task_key=normalize_task_key(u), outcome=o) for u, o in (deps or [])] + return Activity(name=name, task_key=normalize_task_key(name), depends_on=edges or None) + + +def _codes(result) -> set[str]: + return {f.code for f in result.findings} + + +# --------------------------------------------------------------------------- +# Identity (no motifs) +# --------------------------------------------------------------------------- + + +def test_identity_dag_is_equivalent(): + adf = AdfPipeline( + name="p", + activities=[_adf("A"), _adf("B", {"A": ["Succeeded"]}), _adf("C", {"B": ["Succeeded"]})], + ) + ir = Pipeline( + name="p", + tasks=[_task("A"), _task("B", [("A", "Succeeded")]), _task("C", [("B", "Succeeded")])], + ) + result = check_dag_equivalence(adf, ir) + assert result.equivalent + assert not result.violations + assert not result.warnings + + +# --------------------------------------------------------------------------- +# Motif collapse (convex) -- differences tolerated +# --------------------------------------------------------------------------- + + +def test_convex_motif_collapse_is_tolerated(): + # ADF: Lookup -> ForEach -> Sink. Motif collapses {Lookup, ForEach}. + adf = AdfPipeline( + name="p", + activities=[ + _adf("Lookup", adf_type="Lookup"), + _adf("ForEach", {"Lookup": ["Succeeded"]}, adf_type="ForEach"), + _adf("Sink", {"ForEach": ["Succeeded"]}), + ], + ) + pre = Pipeline( + name="p", + tasks=[_task("Lookup"), _task("ForEach", [("Lookup", "Succeeded")]), _task("Sink", [("ForEach", "Succeeded")])], + ) + motif = DetectedMotif(definition=MOTIF_METADATA_DRIVEN_BULK_COPY, matched_activities=["Lookup", "ForEach"]) + collapsed = collapse_motifs(pre, [motif]) + + result = check_dag_equivalence(adf, collapsed) + assert result.equivalent + assert not result.violations + # The Lookup -> ForEach internal edge was absorbed, not reported as a loss. + assert "collapsed_internal_edges" in _codes(result) + assert "missing_edge" not in _codes(result) + + +def test_activity_and_notify_collapse_is_equivalent(): + # ADF: Copy -> {NotifySuccess, NotifyFailure}. All three collapse. + adf = AdfPipeline( + name="p", + activities=[ + _adf("Copy"), + _adf("NotifySuccess", {"Copy": ["Succeeded"]}, adf_type="WebActivity"), + _adf("NotifyFailure", {"Copy": ["Failed"]}, adf_type="WebActivity"), + ], + ) + pre = Pipeline( + name="p", + tasks=[ + _task("Copy"), + _task("NotifySuccess", [("Copy", "Succeeded")]), + _task("NotifyFailure", [("Copy", "Failed")]), + ], + ) + motif = DetectedMotif( + definition=MOTIF_ACTIVITY_AND_NOTIFY, + matched_activities=["Copy", "NotifySuccess", "NotifyFailure"], + ) + collapsed = collapse_motifs(pre, [motif]) + + result = check_dag_equivalence(adf, collapsed) + assert result.equivalent + assert not result.violations + + +# --------------------------------------------------------------------------- +# Non-convex collapse -- the invariant violation +# --------------------------------------------------------------------------- + + +def test_non_convex_motif_is_a_violation(): + # ADF: a1 -> w -> a2, but the motif tries to collapse {a1, a2} with the + # external w sandwiched between them. Collapsing would reorder w. + adf = AdfPipeline( + name="p", + activities=[ + _adf("a1"), + _adf("w", {"a1": ["Succeeded"]}), + _adf("a2", {"w": ["Succeeded"]}), + ], + ) + pre = Pipeline( + name="p", + tasks=[_task("a1"), _task("w", [("a1", "Succeeded")]), _task("a2", [("w", "Succeeded")])], + ) + motif = DetectedMotif(definition=MOTIF_METADATA_DRIVEN_BULK_COPY, matched_activities=["a1", "a2"]) + collapsed = collapse_motifs(pre, [motif]) + + result = check_dag_equivalence(adf, collapsed) + assert not result.equivalent + assert "non_convex_motif" in _codes(result) + non_convex = next(f for f in result.violations if f.code == "non_convex_motif") + assert "w" in non_convex.nodes + + +# --------------------------------------------------------------------------- +# Dropped cross-boundary edge -- ordering constraint lost +# --------------------------------------------------------------------------- + + +def test_dropped_ordering_edge_is_a_violation(): + adf = AdfPipeline( + name="p", + activities=[_adf("A"), _adf("B", {"A": ["Succeeded"]}), _adf("C", {"B": ["Succeeded"]})], + ) + # IR forgot the B -> C edge. + ir = Pipeline(name="p", tasks=[_task("A"), _task("B", [("A", "Succeeded")]), _task("C")]) + result = check_dag_equivalence(adf, ir) + assert not result.equivalent + assert "missing_edge" in _codes(result) + + +# --------------------------------------------------------------------------- +# Synthesised init task -- IR-only, tolerated +# --------------------------------------------------------------------------- + + +def test_synthesised_init_task_is_tolerated(): + adf = AdfPipeline(name="p", activities=[_adf("A"), _adf("B", {"A": ["Succeeded"]})]) + init = Activity(name="_init_flag", task_key="_init_flag") + ir = Pipeline(name="p", tasks=[init, _task("A"), _task("B", [("A", "Succeeded")])]) + result = check_dag_equivalence(adf, ir) + assert result.equivalent + assert "synthesised_task" in _codes(result) + assert "unmapped_ir_task" not in _codes(result) + + +# --------------------------------------------------------------------------- +# Merged dependency outcomes -- lossy collapse, warned (not blocking) +# --------------------------------------------------------------------------- + + +def test_merged_outcome_is_warned_not_blocking(): + # Both motif members depend on E, but with different conditions; the + # collapser keeps only one, which we surface as a warning. + adf = AdfPipeline( + name="p", + activities=[ + _adf("E"), + _adf("a1", {"E": ["Succeeded"]}), + _adf("a2", {"E": ["Failed"]}), + ], + ) + pre = Pipeline( + name="p", + tasks=[_task("E"), _task("a1", [("E", "Succeeded")]), _task("a2", [("E", "Failed")])], + ) + motif = DetectedMotif(definition=MOTIF_ACTIVITY_AND_NOTIFY, matched_activities=["a1", "a2"]) + collapsed = collapse_motifs(pre, [motif]) + + result = check_dag_equivalence(adf, collapsed) + assert result.equivalent # warning, not violation + assert "merged_outcome" in _codes(result) + + +# --------------------------------------------------------------------------- +# Extra ordering edge -- over-constraining, warned +# --------------------------------------------------------------------------- + + +def test_extra_edge_is_warned(): + adf = AdfPipeline(name="p", activities=[_adf("A"), _adf("B")]) # A and B independent + ir = Pipeline(name="p", tasks=[_task("A"), _task("B", [("A", "Succeeded")])]) # IR adds A -> B + result = check_dag_equivalence(adf, ir) + assert result.equivalent + assert "extra_edge" in _codes(result) + + +def test_format_result_reports_status(): + adf = AdfPipeline(name="p", activities=[_adf("A"), _adf("B", {"A": ["Succeeded"]})]) + ir = Pipeline(name="p", tasks=[_task("A"), _task("B")]) # missing edge + rendered = format_result(check_dag_equivalence(adf, ir)) + assert "NOT EQUIVALENT" in rendered + assert "missing_edge" in rendered diff --git a/tests/unit/test_expression_parser.py b/tests/unit/test_expression_parser.py index cd5e769..5df2cc4 100644 --- a/tests/unit/test_expression_parser.py +++ b/tests/unit/test_expression_parser.py @@ -248,9 +248,7 @@ def test_concat_collapses_when_all_parts_resolve_to_literals(self): # C-01: factory globals collapse @concat parts to literal kinds, # so the whole concat should likewise be a literal string. ctx = TranslationContext( - global_parameters=MappingProxyType( - {"env_variable": "t", "deequLibFileName": "deequ-3.5.6.jar"} - ), + global_parameters=MappingProxyType({"env_variable": "t", "deequLibFileName": "deequ-3.5.6.jar"}), ) result = resolve_expression( "@concat('/Volumes/datahub01', pipeline().globalParameters.env_variable, " @@ -413,9 +411,7 @@ def test_equals_bool_literal_emits_lowercase_string(self): def test_substring_two_arg_form(self): """C-33 (VAREX4-001): ADF accepts substring(text, start) without an explicit length argument.""" - result = resolve_expression( - "@substring(string(pipeline().parameters.params), 1)", _context() - ) + result = resolve_expression("@substring(string(pipeline().parameters.params), 1)", _context()) assert result is not None assert result.kind == "notebook_code" assert "[int(" in result.value @@ -425,9 +421,7 @@ def test_split_with_subscript(self): """C-33 (VAREX4-001): a trailing ``[N]`` on a function call lowers to notebook_code Python source so SetVariable activities wrapping ``@split(...)[0]`` actually resolve.""" - result = resolve_expression( - "@split(pipeline().parameters.referenceDate,'/')[0]", _context() - ) + result = resolve_expression("@split(pipeline().parameters.referenceDate,'/')[0]", _context()) assert result is not None assert result.kind == "notebook_code" assert ".split(str('/'))" in result.value diff --git a/tests/unit/test_for_each_inner_job_params.py b/tests/unit/test_for_each_inner_job_params.py index 8c20140..932f1c6 100644 --- a/tests/unit/test_for_each_inner_job_params.py +++ b/tests/unit/test_for_each_inner_job_params.py @@ -91,9 +91,7 @@ def _base(name: str, key: str) -> dict[str, object]: def _spy(tasks, *, raw_ir_tasks=None, variable_task_keys=None): captured.append(variable_task_keys) - return original( - tasks, raw_ir_tasks=raw_ir_tasks, variable_task_keys=variable_task_keys - ) + return original(tasks, raw_ir_tasks=raw_ir_tasks, variable_task_keys=variable_task_keys) monkeypatch.setattr(for_each_module, "collect_inner_job_params", _spy) @@ -118,9 +116,7 @@ def _spy(tasks, *, raw_ir_tasks=None, variable_task_keys=None): variable_task_keys={"continue": "_init_continue"}, ) # Multi-child escalation -> exactly one collect call from for_each preparer. - for_each_call = next( - (m for m in captured if m and "continue" in m), None - ) + for_each_call = next((m for m in captured if m and "continue" in m), None) assert for_each_call is not None, "variable_task_keys must be forwarded" assert for_each_call["continue"] == "_init_continue" # Multi-child path -> inner_workflows populated. diff --git a/tests/unit/test_mcp_migrate.py b/tests/unit/test_mcp_migrate.py new file mode 100644 index 0000000..cf38589 --- /dev/null +++ b/tests/unit/test_mcp_migrate.py @@ -0,0 +1,117 @@ +"""Tests for the MCP server's agent-driven interactive ``migrate`` flow. + +The server returns the full option schema once (``needs_input``); the agent walks the chain locally +and re-calls ``migrate`` once with the complete answers, which applies and packages. These tests +guard that one-shot contract. Skipped where the optional ``mcp`` dependency is absent. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +pytest.importorskip("mcp") + +from flowx.mcp import runner, server # noqa: E402 + + +class _FakeResult: + def __init__(self, *, ok: bool = True, stdout: str = "", stderr: str = "") -> None: + self.ok = ok + self.stdout = stdout + self.stderr = stderr + self.returncode = 0 if ok else 1 + + def as_dict(self) -> dict[str, object]: + return {"returncode": self.returncode, "stdout": self.stdout, "stderr": self.stderr} + + +_SCHEMA = { + "pipelines": [ + { + "pipeline_name": "p", + "options": [ + { + "option_id": "notify_destination", + "prompt": "Route notifications?", + "rationale": "...", + "choices": [{"value": "keep", "label": "Keep", "description": ""}], + "free_text": False, + "default": "keep", + "affected_task_keys": ["load"], + "show_when": [], + }, + { + "option_id": "notify_slack_url", + "prompt": "Slack URL?", + "rationale": "...", + "choices": [], + "free_text": True, + "default": "", + "affected_task_keys": ["load"], + "show_when": [{"option_id": "notify_destination", "in": ["slack"]}], + }, + ], + } + ] +} + + +@pytest.fixture +def stub_adapter(monkeypatch): + """Stubs the adapter subprocess + artifact readers; records which subcommands ran.""" + calls: list[str] = [] + + def fake_run_adapter(args): + calls.append(args[0]) + if args[0] == "inspect": + return _FakeResult(stdout=json.dumps(_SCHEMA)) + return _FakeResult() + + monkeypatch.setattr(runner, "run_adapter", fake_run_adapter) + monkeypatch.setattr(runner, "summarize_inventory", lambda out: {"pipeline_count": 1}) + monkeypatch.setattr(runner, "summarize_translation", lambda out: {"translated": 1}) + monkeypatch.setattr(runner, "list_tree", lambda out: ["databricks.yml"]) + monkeypatch.setattr(runner, "read_tree", lambda out: {"files": {}, "truncated": []}) + return calls + + +def test_first_call_returns_full_schema_without_packaging(stub_adapter, tmp_path: Path): + result = server._cmd_migrate({"adf_source_path": str(tmp_path / "adf"), "output_dir": str(tmp_path / "out")}) + assert result["status"] == "needs_input" + # The whole tree (including the conditional slack follow-up) is returned up front. + option_ids = {o["option_id"] for pipe in result["pending_options"] for o in pipe["options"]} + assert {"notify_destination", "notify_slack_url"} <= option_ids + all_options = [o for pipe in result["pending_options"] for o in pipe["options"]] + slack = next(o for o in all_options if o["option_id"] == "notify_slack_url") + assert slack["show_when"] == [{"option_id": "notify_destination", "in": ["slack"]}] + # discover + convert ran, but NOT package (we paused for input). + assert stub_adapter == ["discover", "convert", "inspect"] + + +def test_resume_with_answers_applies_and_packages_once(stub_adapter, tmp_path: Path): + out = tmp_path / "out" + (out / ".work").mkdir(parents=True) + (out / ".work" / "translation_report.json").write_text("{}") # prior convert output -> resume path + + result = server._cmd_migrate( + { + "adf_source_path": str(tmp_path / "adf"), + "output_dir": str(out), + "answers": ["notify_destination=slack", "notify_slack_url=https://hooks.slack.com/x"], + } + ) + assert result["status"] == "completed" + # Resume skips discover/convert and does not re-inspect; it applies the answers then packages. + assert stub_adapter == ["modify", "package"] + assert "apply_answers" in result["steps"] and "package" in result["steps"] + + +def test_interactive_false_skips_prompt_and_packages(stub_adapter, tmp_path: Path): + result = server._cmd_migrate( + {"adf_source_path": str(tmp_path / "adf"), "output_dir": str(tmp_path / "out"), "interactive": False} + ) + assert result["status"] == "completed" + assert stub_adapter == ["discover", "convert", "package"] # no inspect, no pause diff --git a/tests/unit/test_merge_agentic.py b/tests/unit/test_merge_agentic.py new file mode 100644 index 0000000..9723cbe --- /dev/null +++ b/tests/unit/test_merge_agentic.py @@ -0,0 +1,100 @@ +"""Tests for engine --merge-agentic (folding agent results into a translation report).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from flowx.translator.engine import merge_agentic_results + + +def _write(path: Path, obj: object) -> None: + path.write_text(json.dumps(obj), encoding="utf-8") + + +def test_merge_replaces_nested_placeholder_and_preserves_edges(tmp_path: Path): + report = tmp_path / "translation_report.json" + _write( + report, + { + "name": "p", + "tasks": [ + { + "name": "Gate", + "type": "IfConditionActivity", + "task_key": "gate", + "if_true_activities": [ + { + "name": "Wait", + "type": "PlaceholderActivity", + "task_key": "wait", + "original_type": "Until", + "depends_on": [{"task_key": "upstream", "outcome": "Succeeded"}], + }, + ], + }, + ], + }, + ) + results = tmp_path / "agentic_results" + results.mkdir() + _write( + results / "wait.json", + { + "activity_name": "Wait", + "task": { + "type": "NotebookActivity", + "name": "Wait", + "task_key": "wait", + "notebook_path": "/Workspace/Shared/until_wait", + }, + }, + ) + + merged, unmatched = merge_agentic_results(report, results) + assert (merged, unmatched) == (1, 0) + out = json.loads(report.read_text()) + task = out["tasks"][0]["if_true_activities"][0] + assert task["type"] == "NotebookActivity" + assert task["notebook_path"] == "/Workspace/Shared/until_wait" + # depends_on carried over from the placeholder + assert task["depends_on"] == [{"task_key": "upstream", "outcome": "Succeeded"}] + + +def test_merge_unmatched_when_activity_absent(tmp_path: Path): + report = tmp_path / "r.json" + _write(report, {"name": "p", "tasks": [{"name": "A", "type": "NotebookActivity", "task_key": "a"}]}) + results = tmp_path / "res" + results.mkdir() + _write(results / "x.json", {"activity_name": "Nope", "task": {"type": "NotebookActivity", "name": "Nope"}}) + + merged, unmatched = merge_agentic_results(report, results) + assert (merged, unmatched) == (0, 1) + + +def test_merge_multi_pipeline_disambiguates_by_name(tmp_path: Path): + report = tmp_path / "r.json" + _write( + report, + { + "pipelines": [ + {"name": "p1", "tasks": [{"name": "U", "type": "PlaceholderActivity", "task_key": "u1"}]}, + {"name": "p2", "tasks": [{"name": "U", "type": "PlaceholderActivity", "task_key": "u2"}]}, + ] + }, + ) + results = tmp_path / "res" + results.mkdir() + _write( + results / "u.json", + { + "pipeline": "p2", + "activity_name": "U", + "task": {"type": "NotebookActivity", "name": "U", "task_key": "u2", "notebook_path": "/x"}, + }, + ) + merged, unmatched = merge_agentic_results(report, results) + assert (merged, unmatched) == (1, 0) + out = json.loads(report.read_text()) + assert out["pipelines"][0]["tasks"][0]["type"] == "PlaceholderActivity" # p1 untouched + assert out["pipelines"][1]["tasks"][0]["type"] == "NotebookActivity" # p2 merged diff --git a/tests/unit/test_motifs.py b/tests/unit/test_motifs.py index 1be2e9b..0ca9639 100644 --- a/tests/unit/test_motifs.py +++ b/tests/unit/test_motifs.py @@ -110,7 +110,7 @@ def test_detects_copy_then_web_notification(self): ) motifs = detect_motifs(pipeline, _EMPTY_DEFS) assert len(motifs) == 1 - assert motifs[0].definition.motif_id == "copy_and_notify" + assert motifs[0].definition.motif_id == "activity_and_notify" class TestDetectorParentChild: diff --git a/tests/unit/test_notify.py b/tests/unit/test_notify.py new file mode 100644 index 0000000..ea8cc88 --- /dev/null +++ b/tests/unit/test_notify.py @@ -0,0 +1,397 @@ +"""Tests for the activity_and_notify -> Databricks notification destination feature.""" + +from __future__ import annotations + +from flowx.adapter.models import TranslationConfiguration +from flowx.adapter.operations import ( + apply_configuration, + collect_notify_args, + gather_options, + provision_notification_destinations, +) +from flowx.models.ir import ( + CopyActivity, + Dependency, + LookupActivity, + NotebookActivity, + Pipeline, + WebActivity, +) +from flowx.preparer.notifications import resolve_task_notifications + + +def _pipeline_with_upstream_notify(upstream) -> Pipeline: + """A non-Copy upstream activity followed by success/failure notify Web activities.""" + notify_ok = WebActivity( + name="Notify Success", + task_key="notify_success", + url="https://x", + method="POST", + depends_on=[Dependency(task_key=upstream.task_key, outcome="Succeeded")], + ) + notify_fail = WebActivity( + name="Notify Failure", + task_key="notify_failure", + url="https://x", + method="POST", + depends_on=[Dependency(task_key=upstream.task_key, outcome="Failed")], + ) + return Pipeline(name="p", tasks=[upstream, notify_ok, notify_fail]) + + +def test_notebook_upstream_surfaces_and_collapses(): + """A Notebook (not a Copy) followed by notify Web activities is offered and collapses.""" + p = _pipeline_with_upstream_notify(NotebookActivity(name="Transform", task_key="transform", notebook_path="/t")) + assert "notify_destination" in {o.option_id for o in gather_options(p, []).options} + + cfg = TranslationConfiguration(notify_destination="email", notify_args={"addresses": "a@x.com"}) + out = apply_configuration(p, cfg) + names = {t.name for t in out.tasks} + assert "Notify Success" not in names and "Notify Failure" not in names + transform = next(t for t in out.tasks if t.task_key == "transform") + assert transform.notifications["destination"] == "email" + assert set(transform.notifications["events"]) == {"on_success", "on_failure"} + + +def test_lookup_upstream_collapses(): + """A Lookup followed by a failure-notify Web collapses onto the Lookup task.""" + lookup = LookupActivity(name="Read Control", task_key="read_control") + notify = WebActivity( + name="Alert", + task_key="alert", + url="https://x", + method="POST", + depends_on=[Dependency(task_key="read_control", outcome="Failed")], + ) + p = Pipeline(name="p", tasks=[lookup, notify]) + out = apply_configuration(p, TranslationConfiguration(notify_destination="email", notify_args={"addresses": "a@x"})) + assert "Alert" not in {t.name for t in out.tasks} + read = next(t for t in out.tasks if t.task_key == "read_control") + assert read.notifications["destination"] == "email" + assert read.notifications["events"] == ["on_failure"] + + +def test_generic_preparer_wires_notifications_on_non_copy_task(): + """prepare_activity wires a stamped notification spec into the task for any type, not just Copy.""" + from flowx.preparer.workflow_preparer import prepare_activity + + notebook = NotebookActivity( + name="Transform", + task_key="transform", + notebook_path="/t", + notifications={"destination": "email", "args": {"addresses": ["a@x.com"]}, "events": ["on_failure"]}, + ) + prepared = prepare_activity(notebook) + assert prepared.task["email_notifications"] == {"on_failure": ["a@x.com"]} + + +def test_web_upstream_is_not_a_notify_target(): + """A WebActivity following another WebActivity is not treated as a notify group (web->web).""" + work = WebActivity(name="Call API", task_key="call_api", url="https://api", method="POST") + notify = WebActivity( + name="Notify", + task_key="notify", + url="https://x", + method="POST", + depends_on=[Dependency(task_key="call_api", outcome="Succeeded")], + ) + p = Pipeline(name="p", tasks=[work, notify]) + assert "notify_destination" not in {o.option_id for o in gather_options(p, []).options} + + +def _pipeline_with_notify() -> Pipeline: + copy = CopyActivity(name="Load Curated", task_key="load_curated") + notify_ok = WebActivity( + name="Notify Success", + task_key="notify_success", + url="https://x", + method="POST", + depends_on=[Dependency(task_key="load_curated", outcome="Succeeded")], + ) + notify_fail = WebActivity( + name="Notify Failure", + task_key="notify_failure", + url="https://x", + method="POST", + depends_on=[Dependency(task_key="load_curated", outcome="Failed")], + ) + downstream = NotebookActivity( + name="After", + task_key="after", + notebook_path="/n", + depends_on=[Dependency(task_key="notify_success", outcome="Succeeded")], + ) + return Pipeline(name="p", tasks=[copy, notify_ok, notify_fail, downstream]) + + +def test_option_surfaces_and_followups_are_gated_by_answer(): + p = _pipeline_with_notify() + ids = {o.option_id for o in gather_options(p, []).options} + assert "notify_destination" in ids + # follow-ups not shown until a destination is chosen + assert "notify_email_recipients" not in ids + assert "notify_slack_url" not in ids + + email_ids = {o.option_id for o in gather_options(p, [], answers={"notify_destination": "email"}).options} + assert "notify_email_recipients" in email_ids + assert "notify_slack_url" not in email_ids + + slack_ids = {o.option_id for o in gather_options(p, [], answers={"notify_destination": "slack"}).options} + assert "notify_slack_url" in slack_ids + assert "notify_destination_name" in slack_ids + assert "notify_email_recipients" not in slack_ids + + +def test_chain_surfaces_every_sdk_field_for_destination(): + """Each SDK field of the chosen destination becomes its own follow-up option, + in registry order (required first), so the agent can prompt sequentially.""" + p = _pipeline_with_notify() + webhook_ids = [o.option_id for o in gather_options(p, [], answers={"notify_destination": "webhook"}).options] + # SDK fields surface in registry order (required url first), then name + events + webhook_fields = [i for i in webhook_ids if i.startswith("notify_webhook")] + assert webhook_fields == [ + "notify_webhook_url", + "notify_webhook_username", + "notify_webhook_password", + ] + assert "notify_destination_name" in webhook_ids + assert "notify_events" in webhook_ids + + slack_ids = [o.option_id for o in gather_options(p, [], answers={"notify_destination": "slack"}).options] + slack_fields = [i for i in slack_ids if i.startswith("notify_slack")] + assert slack_fields == [ + "notify_slack_url", + "notify_slack_channel_id", + "notify_slack_oauth_token", + ] + + +def test_answered_field_drops_out_of_the_chain(): + """Already-answered follow-ups are filtered, so the chain advances field by field.""" + p = _pipeline_with_notify() + answers = {"notify_destination": "slack", "notify_slack_url": "https://hooks.slack.com/x"} + ids = {o.option_id for o in gather_options(p, [], answers=answers).options} + assert "notify_slack_url" not in ids # answered -> gone + assert "notify_slack_channel_id" in ids # still pending + + +def test_collect_notify_args_reads_only_chosen_destination_fields(): + answers = { + "notify_destination": "webhook", + "notify_webhook_url": "https://hooks.example.com", + "notify_webhook_username": "svc", + "notify_webhook_password": "", # blank -> omitted + "notify_slack_url": "https://leftover.slack", # belongs to a different dest -> ignored + } + args = collect_notify_args(answers) + assert args == {"url": "https://hooks.example.com", "username": "svc"} + + +def test_keep_default_does_not_collapse(): + p = _pipeline_with_notify() + out = apply_configuration(p, TranslationConfiguration()) # default = keep + names = {t.name for t in out.tasks} + assert {"Notify Success", "Notify Failure"} <= names # still present + + +def test_email_collapse_drops_notifies_and_stamps_copy(): + p = _pipeline_with_notify() + cfg = TranslationConfiguration( + notify_destination="email", + notify_args={"addresses": "a@x.com, b@x.com"}, + notify_events="both", + ) + out = apply_configuration(p, cfg) + names = {t.name for t in out.tasks} + assert "Notify Success" not in names and "Notify Failure" not in names + copy = next(t for t in out.tasks if t.task_key == "load_curated") + assert copy.notifications["destination"] == "email" + assert copy.notifications["args"]["addresses"] == ["a@x.com", "b@x.com"] + assert set(copy.notifications["events"]) == {"on_success", "on_failure"} + # downstream task rewired off the dropped notify onto the copy + after = next(t for t in out.tasks if t.task_key == "after") + assert any(d.task_key == "load_curated" for d in (after.depends_on or [])) + + +def test_webhook_collapse_stamps_resolved_args(): + p = _pipeline_with_notify() + cfg = TranslationConfiguration( + notify_destination="webhook", + notify_args={"url": "https://hooks.example.com", "username": "svc"}, + notify_events="both", + ) + out = apply_configuration(p, cfg) + copy = next(t for t in out.tasks if t.task_key == "load_curated") + assert copy.notifications["destination"] == "webhook" + assert copy.notifications["args"] == {"url": "https://hooks.example.com", "username": "svc"} + + +def test_events_restriction_to_failure_only(): + p = _pipeline_with_notify() + cfg = TranslationConfiguration( + notify_destination="email", + notify_args={"addresses": "a@x.com"}, + notify_events="on_failure", + ) + out = apply_configuration(p, cfg) + copy = next(t for t in out.tasks if t.task_key == "load_curated") + assert copy.notifications["events"] == ["on_failure"] + + +def test_resolve_email_notifications(): + keys, setup = resolve_task_notifications( + {"destination": "email", "args": {"addresses": ["a@x.com"]}, "events": ["on_failure", "on_success"]} + ) + assert keys == {"email_notifications": {"on_failure": ["a@x.com"], "on_success": ["a@x.com"]}} + assert setup == [] + + +def test_resolve_webhook_without_workspace_falls_back_to_setup_task(monkeypatch): + # Force the SDK create path to fail -> graceful fallback to a setup task. + import flowx.preparer.notifications as nm + + monkeypatch.setattr(nm, "_ensure_destination", lambda *a, **k: None) + keys, setup = resolve_task_notifications( + { + "destination": "slack", + "args": {"url": "https://hooks"}, + "destination_name": "flowx-slack", + "events": ["on_failure"], + } + ) + assert keys == {} + assert len(setup) == 1 and setup[0].type == "notification_destination" + assert setup[0].config["url"] == "https://hooks" + + +def test_build_destination_config_passes_only_supplied_optional_fields(): + """Optional SDK kwargs are omitted when blank so the SDK applies its defaults.""" + import flowx.preparer.notifications as nm + + class _FakeSlackConfig: + def __init__(self, **kwargs): + self.kwargs = kwargs + + class _FakeConfig: + def __init__(self, slack=None): + self.slack = slack + + class _FakeSettings: + Config = _FakeConfig + SlackConfig = _FakeSlackConfig + + cfg = nm._build_destination_config(_FakeSettings, "slack", {"url": "https://hooks", "channel_id": ""}) + assert cfg.slack.kwargs == {"url": "https://hooks"} # blank channel_id dropped + + +def test_validate_answer_accepts_free_text_notify_options(): + """Regression: free-text notify follow-ups must validate (were rejected + as 'Unknown option_id', so email recipients never reached the config).""" + import pytest + + from flowx.adapter.operations import validate_answer + + # free-text options accept any value + assert validate_answer("notify_email_recipients", "a@x.com, b@x.com") == "a@x.com, b@x.com" + assert validate_answer("notify_webhook_url", "https://hooks.example.com") == "https://hooks.example.com" + assert validate_answer("notify_slack_oauth_token", "xoxb-123") == "xoxb-123" + assert validate_answer("notify_pagerduty_integration_key", "abc123") == "abc123" + assert validate_answer("notify_destination_name", "flowx-oncall") == "flowx-oncall" + # enum-backed options still validate against their enum + assert validate_answer("notify_destination", "email") == "email" + with pytest.raises(ValueError): + validate_answer("notify_destination", "carrier_pigeon") + # genuinely unknown ids are still rejected + with pytest.raises(ValueError): + validate_answer("totally_unknown_option", "x") + + +def test_provision_destination_creates_at_prompt_time(monkeypatch): + """Non-email destinations are created (SDK) at prompt time and the resolved + id is stamped onto the spec.""" + import flowx.preparer.notifications as nm + + monkeypatch.setattr(nm, "_ensure_destination", lambda dest, name, args: "dest-abc-1") + spec = {"destination": "slack", "destination_name": "flowx-slack", "args": {"url": "https://h"}} + new_spec, message = nm.provision_destination(spec) + assert new_spec["destination_id"] == "dest-abc-1" + assert "Created" in message or "reused" in message.lower() + + +def test_provision_destination_email_is_passthrough(monkeypatch): + """Email needs no destination -- provision is a no-op and never calls the SDK.""" + import flowx.preparer.notifications as nm + + def _boom(*a, **k): + raise AssertionError("SDK must not be called for email") + + monkeypatch.setattr(nm, "_ensure_destination", _boom) + spec = {"destination": "email", "args": {"addresses": ["a@x.com"]}} + new_spec, message = nm.provision_destination(spec) + assert new_spec is spec + assert message == "" + + +def test_provision_destination_failure_keeps_spec(monkeypatch): + """When creation fails at prompt time the spec is unchanged (args retained) so + prepare can retry / emit a setup task, and a warning is surfaced.""" + import flowx.preparer.notifications as nm + + monkeypatch.setattr(nm, "_ensure_destination", lambda *a, **k: None) + spec = {"destination": "webhook", "args": {"url": "https://h"}} + new_spec, message = nm.provision_destination(spec) + assert "destination_id" not in new_spec + assert new_spec is spec + assert message.startswith("WARNING") + + +def test_provision_notification_destinations_walk(monkeypatch): + """The adapter modify-phase walk stamps resolved ids onto non-email copy tasks.""" + import flowx.preparer.notifications as nm + + monkeypatch.setattr(nm, "_ensure_destination", lambda dest, name, args: "dest-xyz-9") + p = _pipeline_with_notify() + cfg = TranslationConfiguration( + notify_destination="slack", + notify_args={"url": "https://hooks.slack.com/x"}, + notify_events="both", + ) + stamped = apply_configuration(p, cfg) + provisioned, messages = provision_notification_destinations(stamped) + copy = next(t for t in provisioned.tasks if t.task_key == "load_curated") + assert copy.notifications["destination_id"] == "dest-xyz-9" + assert len(messages) == 1 + + +def test_provision_walk_skips_email_and_keep(monkeypatch): + """Email collapse produces no destination; the walk makes no SDK call and stamps no id.""" + import flowx.preparer.notifications as nm + + def _boom(*a, **k): + raise AssertionError("SDK must not be called for email") + + monkeypatch.setattr(nm, "_ensure_destination", _boom) + p = _pipeline_with_notify() + cfg = TranslationConfiguration( + notify_destination="email", notify_args={"addresses": "a@x.com"}, notify_events="both" + ) + provisioned, messages = provision_notification_destinations(apply_configuration(p, cfg)) + copy = next(t for t in provisioned.tasks if t.task_key == "load_curated") + assert "destination_id" not in copy.notifications + assert messages == [] + + +def test_resolve_uses_pre_resolved_id_without_sdk(monkeypatch): + """At prepare time a spec carrying a prompt-time destination_id wires directly, + with no further SDK call.""" + import flowx.preparer.notifications as nm + + def _boom(*a, **k): + raise AssertionError("prepare must reuse the prompt-time id, not call the SDK") + + monkeypatch.setattr(nm, "_ensure_destination", _boom) + keys, setup = resolve_task_notifications( + {"destination": "slack", "destination_id": "dest-pre-7", "events": ["on_failure"]} + ) + assert keys == {"webhook_notifications": {"on_failure": [{"id": "dest-pre-7"}]}} + assert setup == [] diff --git a/tests/unit/test_param_dedup.py b/tests/unit/test_param_dedup.py new file mode 100644 index 0000000..2c5680c --- /dev/null +++ b/tests/unit/test_param_dedup.py @@ -0,0 +1,36 @@ +"""Regression: job parameters must not be duplicated through the CLI report path.""" + +from __future__ import annotations + +from flowx.bundler.dab_writer import _build_job_resource, _pipeline_dict_to_workflow + + +def _report(default="us"): + return { + "name": "pipeline_simple", + "parameters": [{"name": "region", "type": "String", "default": default}], + "tasks": [ + { + "name": "Ingest Bronze", + "type": "NotebookActivity", + "task_key": "ingest_bronze", + "notebook_path": "/Shared/ETL/01_ingest_bronze", + "base_parameters": {"region": "@pipeline().parameters.region"}, + }, + ], + } + + +def test_report_path_does_not_duplicate_parameters(): + wf = _pipeline_dict_to_workflow(_report()) + names = [p.get("name") for p in wf.parameters] + assert names == ["region"], f"expected one region parameter, got {names}" + + +def test_build_job_resource_dedupes_parameters(): + wf = _pipeline_dict_to_workflow(_report()) + # even if a caller double-added, the emitted job declares region once + wf.parameters = wf.parameters + wf.parameters + job = _build_job_resource(wf, "pipeline_simple")["resources"]["jobs"]["pipeline_simple"] + names = [p["name"] for p in job["parameters"]] + assert names == ["region"] diff --git a/tests/unit/test_preparers.py b/tests/unit/test_preparers.py index 3b9a8fc..f206c73 100644 --- a/tests/unit/test_preparers.py +++ b/tests/unit/test_preparers.py @@ -99,9 +99,7 @@ def test_prepare_notebook_dispatch_stub_for_unresolved_path(self): # SETUP.md SetupTask is emitted. kinds = [st.type for st in prepared.setup_tasks] assert "dynamic_notebook_dispatch" in kinds - config = next( - st.config for st in prepared.setup_tasks if st.type == "dynamic_notebook_dispatch" - ) + config = next(st.config for st in prepared.setup_tasks if st.type == "dynamic_notebook_dispatch") assert config["task_key"] == "dispatch" assert "@trim" in config["expression"] # The notebook_path widget is registered with an empty default. @@ -124,9 +122,7 @@ def test_prepare_notebook_emits_unresolved_library_setup_task(self): prepared = prepare_activity(activity) kinds = [st.type for st in prepared.setup_tasks] assert "unresolved_library" in kinds - config = next( - st.config for st in prepared.setup_tasks if st.type == "unresolved_library" - ) + config = next(st.config for st in prepared.setup_tasks if st.type == "unresolved_library") assert config["task_key"] == "run_nb" assert config["library_type"] == "jar" assert "proj4jLibFileName" in config["missing"] @@ -142,9 +138,9 @@ def test_prepare_notebook_no_params(self): # No placeholder for an absolute workspace path. assert prepared.notebooks == [] - def test_prepare_notebook_vendors_downloaded_workspace_notebook(self, monkeypatch): + def test_prepare_notebook_downloads_downloaded_workspace_notebook(self, monkeypatch): """When downloads are enabled and the SDK returns content, the workspace notebook - is vendored into src/notebooks/ under the workspace basename, and the task is + is downloaded into src/notebooks/ under the workspace basename, and the task is bound to the default cluster.""" monkeypatch.setattr(workspace_downloader, "_downloads_enabled", True) monkeypatch.setattr( @@ -164,7 +160,7 @@ def test_prepare_notebook_vendors_downloaded_workspace_notebook(self, monkeypatc # skips ../src/ paths because flowx-generated notebooks are # serverless-only; downloaded notebooks need classic compute). assert prepared.task["job_cluster_key"] == "default_cluster" - # Notebook vendored under the workspace basename + # Notebook downloaded under the workspace basename assert len(prepared.notebooks) == 1 assert prepared.notebooks[0].relative_path == "notebooks/transform.py" assert "from /Shared/ETL/transform" in prepared.notebooks[0].content @@ -188,7 +184,7 @@ def test_prepare_notebook_preserves_workspace_basename_verbatim(self, monkeypatc def test_prepare_notebook_falls_back_to_in_place_when_download_fails(self, monkeypatch): """If downloads are enabled but the SDK returns None, behavior matches the - legacy in-place reference (no vendor, no cluster bind in the preparer).""" + legacy in-place reference (no download, no cluster bind in the preparer).""" monkeypatch.setattr(workspace_downloader, "_downloads_enabled", True) monkeypatch.setattr( "flowx.preparer.activity_preparers.notebook.download_notebook", @@ -415,8 +411,7 @@ def test_prepare_web_activity_key_vault_secret_uses_vault_scope_and_secret_name( ) prepared = prepare_activity(activity) assert any( - s.scope == "lakeh_ls_keyvault" and s.key == "adapp-auccommonutilssp-secret" - for s in prepared.secrets + s.scope == "lakeh_ls_keyvault" and s.key == "adapp-auccommonutilssp-secret" for s in prepared.secrets ) # The generic auth-credential placeholder is suppressed when a real # secret reference is available. @@ -628,9 +623,7 @@ def test_prepare_for_each_bridges_split_items_via_seed_task(self): # inputs now reference the bridge task value, not the @split string. assert prepared.task["for_each_task"]["inputs"] == "{{tasks.loop_inputs_bridge.values.items}}" # ForEach depends on the bridge so the value is materialised first. - assert any( - dep.get("task_key") == "loop_inputs_bridge" for dep in prepared.task.get("depends_on") or [] - ) + assert any(dep.get("task_key") == "loop_inputs_bridge" for dep in prepared.task.get("depends_on") or []) def test_for_each_with_inner_if_condition_carries_branches(self): """Change foreach-inner-extra-tasks (P0): CF-001. @@ -794,9 +787,7 @@ def test_set_var_in_foreach_read_by_sibling_emits_setup_task(self): ) pipeline = Pipeline(name="cross_foreach_pipe", tasks=[loop, sibling]) wf = prepare_workflow(pipeline) - rollups = [ - st for st in wf.setup_tasks if st.type == "manual_variable_rollup" - ] + rollups = [st for st in wf.setup_tasks if st.type == "manual_variable_rollup"] assert len(rollups) == 1 config = rollups[0].config assert config["variable_name"] == "continue" @@ -833,9 +824,7 @@ def test_set_var_with_parent_scope_setter_emits_no_warning(self): tasks=[parent_setter, loop, sibling], ) wf = prepare_workflow(pipeline) - rollups = [ - st for st in wf.setup_tasks if st.type == "manual_variable_rollup" - ] + rollups = [st for st in wf.setup_tasks if st.type == "manual_variable_rollup"] assert rollups == [] @@ -954,10 +943,50 @@ def test_motif_preparer_registered(self): motif_config={"sink_table": "raw.{schema_name}_{table_name}"}, ) prepared = prepare_activity(activity) - assert "notebook_task" in prepared.task - assert prepared.task["notebook_task"]["notebook_path"].endswith("bulk_ingest.py") - assert len(prepared.notebooks) == 1 - assert "metadata_driven_bulk_copy" in prepared.notebooks[0].content + # Default (non-consolidated) metadata-driven bulk copy now becomes a for_each_task that runs + # one Spark JDBC read per source table. With no resolved lookup_values, a runtime + # control-table lookup task seeds the iteration inputs. + assert "for_each_task" in prepared.task + for_each = prepared.task["for_each_task"] + assert for_each["inputs"] == "{{tasks.bulk_ingest_control_lookup.values.items}}" + assert for_each["task"]["notebook_task"]["base_parameters"]["item"] == "{{input}}" + assert prepared.task["depends_on"] == [{"task_key": "bulk_ingest_control_lookup"}] + assert [t["task_key"] for t in prepared.extra_tasks] == ["bulk_ingest_control_lookup"] + # inner per-item read notebook + the control-lookup seed notebook + assert {nb.relative_path for nb in prepared.notebooks} == { + "notebooks/bulk_ingest_ingest.py", + "notebooks/bulk_ingest_control_lookup.py", + } + + def test_metadata_driven_for_each_uses_static_inputs_when_lookup_values_resolved(self): + """Resolved control rows are inlined as the for_each_task inputs -- no runtime lookup task.""" + import json + + from flowx.models.ir import MotifActivity + + rows = [{"schema_name": "dbo", "table_name": "orders"}, {"schema_name": "dbo", "table_name": "customers"}] + activity = MotifActivity( + **_make_base("Bulk Ingest", "bulk_ingest"), + motif_id="metadata_driven_bulk_copy", + display_name="Metadata-driven bulk copy", + databricks_replacement="for_each_ingestion", + matched_activity_names=["GetTableList", "ForEachTable", "CopyTable"], + source_type_hint="database", + motif_config={"sink_table": "raw.{schema_name}_{table_name}", "copy_scope": "src_db"}, + lookup_values=rows, + ) + prepared = prepare_activity(activity) + for_each = prepared.task["for_each_task"] + assert json.loads(for_each["inputs"]) == rows # inlined literal JSON array + assert prepared.extra_tasks == [] # no control-lookup seed task + assert "depends_on" not in prepared.task or all( + d["task_key"] != "bulk_ingest_control_lookup" for d in prepared.task.get("depends_on", []) + ) + assert [nb.relative_path for nb in prepared.notebooks] == ["notebooks/bulk_ingest_ingest.py"] + # inner read notebook targets the configured sink + secret scope + body = prepared.notebooks[0].content + assert 'dbutils.secrets.get(scope="src_db"' in body + assert "raw.{schema_name}_{table_name}" in body class TestSwitchPreparer: diff --git a/tests/unit/test_prereqs_writer.py b/tests/unit/test_prereqs_writer.py index a04fe0a..f053618 100644 --- a/tests/unit/test_prereqs_writer.py +++ b/tests/unit/test_prereqs_writer.py @@ -18,7 +18,7 @@ def test_workflow_secrets_union_with_notebook_scanned_scopes(self): content=( "# Databricks notebook source\n" "auth_token = dbutils.secrets.get(" - "scope=\"lakeh_a_pl_operational_sendMail\", key=\"auth-credential\")\n" + 'scope="lakeh_a_pl_operational_sendMail", key="auth-credential")\n' ), ) ] @@ -52,10 +52,7 @@ def test_setup_md_lists_unioned_secrets(self): notebooks = [ DabNotebook( relative_path="notebooks/x.py", - content=( - "auth_token = dbutils.secrets.get(" - "scope=\"scope_from_notebook\", key=\"key_from_notebook\")\n" - ), + content=('auth_token = dbutils.secrets.get(scope="scope_from_notebook", key="key_from_notebook")\n'), ) ] secret_instructions = [ diff --git a/tests/unit/test_profile_report.py b/tests/unit/test_profile_report.py new file mode 100644 index 0000000..e7cc7ef --- /dev/null +++ b/tests/unit/test_profile_report.py @@ -0,0 +1,107 @@ +"""Tests for the profile-phase complexity report (CSV + T-shirt sizing + ARM export).""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +from flowx.models.adf_ast import ( + AdfActivity, + AdfDataset, + AdfDatasetReference, + AdfDefinitions, + AdfLinkedService, + AdfLinkedServiceReference, + AdfPipeline, +) +from flowx.parser.adf_loader import ( + _activity_category, + _complexity_score, + _tshirt_size, + build_profile_rows, + write_pipeline_arm, + write_profile_csv, +) + + +def test_activity_category_ordering(): + assert _activity_category("DatabricksNotebook") == "databricks" + assert _activity_category("ForEach") == "control" + assert _activity_category("SetVariable") == "control" + assert _activity_category("Copy") == "other" + assert _activity_category("WebActivity") == "other" + + +def test_complexity_score_weights_other_highest(): + # 1 native vs 1 other: other must score higher. + native = _complexity_score({"databricks": 1, "control": 0, "other": 0}, 0, 0, 0) + control = _complexity_score({"databricks": 0, "control": 1, "other": 0}, 0, 0, 0) + other = _complexity_score({"databricks": 0, "control": 0, "other": 1}, 0, 0, 0) + assert native < control < other + + +def test_tshirt_size_buckets(): + assert _tshirt_size(2) == "S" + assert _tshirt_size(10) == "M" + assert _tshirt_size(25) == "L" + assert _tshirt_size(40) == "XL" + + +def _pipeline_with_copy() -> AdfDefinitions: + copy = AdfActivity( + name="Load", + type="Copy", + inputs=[AdfDatasetReference(reference_name="src_ds")], + outputs=[AdfDatasetReference(reference_name="dst_ds")], + ) + notebook = AdfActivity( + name="Run", + type="DatabricksNotebook", + linked_service_name=AdfLinkedServiceReference(reference_name="adb_ls"), + ) + pipeline = AdfPipeline(name="p1", activities=[copy, notebook], raw={"name": "p1", "properties": {}}) + return AdfDefinitions( + pipelines=[pipeline], + datasets={ + "src_ds": AdfDataset(name="src_ds", type="AzureSqlTable", properties={}, linked_service_name="sql_ls"), + "dst_ds": AdfDataset(name="dst_ds", type="DelimitedText", properties={}, linked_service_name="adls_ls"), + }, + linked_services={ + "sql_ls": AdfLinkedService(name="sql_ls", type="AzureSqlDatabase", properties={}), + "adls_ls": AdfLinkedService(name="adls_ls", type="AzureBlobFS", properties={}), + "adb_ls": AdfLinkedService(name="adb_ls", type="AzureDatabricks", properties={}), + }, + ) + + +def test_build_profile_rows_counts_datasets_and_linked_services(): + rows = build_profile_rows(_pipeline_with_copy()) + assert len(rows) == 1 + row = rows[0] + assert row["pipeline"] == "p1" + assert row["activities"] == 2 + assert row["datasets"] == 2 # src_ds + dst_ds + # 2 from datasets (sql_ls, adls_ls) + 1 activity-level (adb_ls) + assert row["linked_services"] == 3 + assert row["databricks_native_activities"] == 1 + assert row["other_activities"] == 1 + assert row["complexity_size"] in {"S", "M", "L", "XL"} + + +def test_write_profile_csv_roundtrip(tmp_path: Path): + rows = build_profile_rows(_pipeline_with_copy()) + csv_path = tmp_path / "profile_report.csv" + write_profile_csv(rows, csv_path) + with csv_path.open() as handle: + read_rows = list(csv.DictReader(handle)) + assert read_rows[0]["pipeline"] == "p1" + assert read_rows[0]["activities"] == "2" + + +def test_write_pipeline_arm_emits_verbatim_source(tmp_path: Path): + definitions = _pipeline_with_copy() + written = write_pipeline_arm(definitions, tmp_path) + assert len(written) == 1 + arm = json.loads(written[0].read_text()) + assert arm == {"name": "p1", "properties": {}} diff --git a/tests/unit/test_reporting_coverage.py b/tests/unit/test_reporting_coverage.py new file mode 100644 index 0000000..5c1a8f9 --- /dev/null +++ b/tests/unit/test_reporting_coverage.py @@ -0,0 +1,90 @@ +"""Tests for building per-pipeline coverage rows from migration metadata.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +from flowx.reporting.coverage import build_coverage_rows + + +def _write_metadata(tmp_path: Path) -> Path: + md = tmp_path / "metadata" + md.mkdir() + inventory = { + "pipelines": [ + { + "name": "p_alpha", + "activities": [ + {"name": "a1", "type": "DatabricksNotebook", "strategy": "deterministic"}, + {"name": "a2", "type": "Copy", "strategy": "deterministic"}, + {"name": "a3", "type": "ExecuteDataFlow", "strategy": "agentic"}, + {"name": "a4", "type": "Custom", "strategy": "unsupported"}, + ], + }, + { + "name": "p_beta", + "activities": [ + {"name": "b1", "type": "DatabricksNotebook", "strategy": "deterministic"}, + ], + }, + ], + "summary": {"pipeline_count": 2}, + } + (md / "inventory.json").write_text(json.dumps(inventory), encoding="utf-8") + rows = [ + { + "pipeline": "p_alpha", + "activities": 4, + "datasets": 2, + "linked_services": 1, + "collapsible_patterns": 1, + "databricks_native_activities": 1, + "control_flow_activities": 0, + "other_activities": 3, + "complexity_score": 12, + "complexity_size": "M", + }, + { + "pipeline": "p_beta", + "activities": 1, + "datasets": 0, + "linked_services": 1, + "collapsible_patterns": 0, + "databricks_native_activities": 1, + "control_flow_activities": 0, + "other_activities": 0, + "complexity_score": 2, + "complexity_size": "S", + }, + ] + with (md / "profile_report.csv").open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + return md + + +def test_build_coverage_rows_joins_inventory_and_csv(tmp_path: Path): + rows = build_coverage_rows(_write_metadata(tmp_path)) + assert [r["pipeline"] for r in rows] == ["p_alpha", "p_beta"] # sorted by name + alpha = rows[0] + assert alpha["activities"] == 4 + assert alpha["deterministic_activities"] == 2 + assert alpha["agentic_activities"] == 1 + assert alpha["unsupported_activities"] == 1 + # coverage = (det + agentic) / total = 3/4 = 75.0 + assert alpha["coverage_pct"] == 75.0 + # complexity columns come from the CSV + assert alpha["datasets"] == 2 and alpha["linked_services"] == 1 + assert alpha["collapsible_patterns"] == 1 and alpha["complexity_size"] == "M" + + +def test_build_coverage_rows_full_coverage_and_missing_csv(tmp_path: Path): + md = _write_metadata(tmp_path) + (md / "profile_report.csv").unlink() # CSV optional -> complexity columns default + rows = {r["pipeline"]: r for r in build_coverage_rows(md)} + beta = rows["p_beta"] + assert beta["coverage_pct"] == 100.0 # 1/1 deterministic + assert beta["datasets"] == 0 and beta["complexity_size"] == "" # defaulted, no CSV diff --git a/tests/unit/test_reporting_dashboard.py b/tests/unit/test_reporting_dashboard.py new file mode 100644 index 0000000..319e75d --- /dev/null +++ b/tests/unit/test_reporting_dashboard.py @@ -0,0 +1,101 @@ +"""Tests for the coverage dashboard builder + installer.""" + +from __future__ import annotations + +import json + +import pytest + +from flowx.reporting import dashboard as D + + +def test_build_serialized_dashboard_injects_table_and_is_valid_json(): + serialized = D.build_serialized_dashboard("cat.sch.results") + spec = json.loads(serialized) + assert "{{RESULTS_TABLE}}" not in serialized + # every dataset query references the fully-qualified table + joined = " ".join(line for ds in spec["datasets"] for line in ds["queryLines"]) + assert "cat.sch.results" in joined + assert spec["pages"][0]["pageType"] == "PAGE_TYPE_CANVAS" + # widget field names match their dataset fields (counter references a real column) + widget_names = {w["widget"]["name"] for w in spec["pages"][0]["layout"]} + assert {"kpi-coverage", "by-size", "coverage-trend", "pipeline-table"} <= widget_names + + +def test_build_serialized_dashboard_requires_table(): + with pytest.raises(ValueError): + D.build_serialized_dashboard("") + + +class _Created: + dashboard_id = "dash-123" + + +class _FakeLakeview: + def __init__(self): + self.created = None + self.published = None + + def create(self, dashboard): + self.created = dashboard + return _Created() + + def publish(self, dashboard_id, warehouse_id): + self.published = (dashboard_id, warehouse_id) + + +class _FakeWarehousesAPI: + def list(self): + class _W: + id = "wh1" + name = "wh1" + state = "RUNNING" + enable_serverless_compute = True + warehouse_type = "PRO" + + return [_W()] + + +class _Me: + user_name = "greg@databricks.com" + + +class _FakeCurrentUser: + def me(self): + return _Me() + + +class _FakeConfig: + host = "https://example.cloud.databricks.com" + + +class _FakeClient: + def __init__(self): + self.lakeview = _FakeLakeview() + self.warehouses = _FakeWarehousesAPI() + self.current_user = _FakeCurrentUser() + self.config = _FakeConfig() + + +def test_install_dashboard_creates_and_publishes(): + client = _FakeClient() + dashboard_id, url = D.install_dashboard("cat.sch.results", client=client) + assert dashboard_id == "dash-123" + assert url == "https://example.cloud.databricks.com/sql/dashboardsv3/dash-123" + # created with resolved warehouse, table-bound spec, and default parent path = user home + created = client.lakeview.created + assert created.warehouse_id == "wh1" + assert created.parent_path == "/Workspace/Users/greg@databricks.com" + assert "cat.sch.results" in created.serialized_dashboard + assert client.lakeview.published == ("dash-123", "wh1") + + +def test_install_dashboard_respects_overrides(): + client = _FakeClient() + D.install_dashboard( + "cat.sch.results", warehouse_id="whX", display_name="My Dash", parent_path="/Workspace/Shared", client=client + ) + created = client.lakeview.created + assert created.warehouse_id == "whX" + assert created.display_name == "My Dash" + assert created.parent_path == "/Workspace/Shared" diff --git a/tests/unit/test_reporting_results.py b/tests/unit/test_reporting_results.py new file mode 100644 index 0000000..6351ecf --- /dev/null +++ b/tests/unit/test_reporting_results.py @@ -0,0 +1,180 @@ +"""Tests for the UC-table results writer (SQL builders, warehouse resolution, write).""" + +from __future__ import annotations + +import csv +import json +import uuid +from pathlib import Path + +import pytest + +from flowx.reporting import results as R + + +def test_create_table_sql_has_run_metadata_and_all_columns(): + sql = R.build_create_table_sql("cat.sch.tbl") + assert sql.startswith("CREATE TABLE IF NOT EXISTS cat.sch.tbl") + assert "run_id STRING" in sql + assert "run_date TIMESTAMP" in sql + assert "run_by STRING" in sql + assert "coverage_pct DOUBLE" in sql + assert "complexity_size STRING" in sql + + +def test_insert_sql_stamps_run_metadata_and_escapes(): + rows = [ + { + "pipeline": "p1", + "activities": 3, + "datasets": 1, + "linked_services": 0, + "collapsible_patterns": 0, + "databricks_native_activities": 1, + "control_flow_activities": 0, + "other_activities": 2, + "deterministic_activities": 2, + "agentic_activities": 1, + "unsupported_activities": 0, + "coverage_pct": 100.0, + "complexity_score": 7, + "complexity_size": "M", + }, + { + "pipeline": "O'Brien's pipe", + "activities": 1, + "datasets": 0, + "linked_services": 0, + "collapsible_patterns": 0, + "databricks_native_activities": 0, + "control_flow_activities": 0, + "other_activities": 1, + "deterministic_activities": 0, + "agentic_activities": 0, + "unsupported_activities": 1, + "coverage_pct": 0.0, + "complexity_score": 3, + "complexity_size": "S", + }, + ] + run_id = "abc-123" + sql = R.build_insert_sql("cat.sch.tbl", rows, run_id) + assert "INSERT INTO cat.sch.tbl (run_id, run_date, run_by," in sql + # run metadata: literal run_id + SQL functions on every row + assert sql.count("'abc-123'") == 2 + assert sql.count("CURRENT_TIMESTAMP()") == 2 + assert sql.count("CURRENT_USER()") == 2 + # apostrophe escaped by doubling + assert "'O''Brien''s pipe'" in sql + # numeric + float rendered unquoted + assert "100.0" in sql + + +class _FakeWarehouse: + def __init__(self, id, state, serverless=False): + self.id = id + self.name = id + self.state = state + self.enable_serverless_compute = serverless + self.warehouse_type = "PRO" + + +class _FakeWarehousesAPI: + def __init__(self, items): + self._items = items + + def list(self): + return list(self._items) + + +class _FakeStmtAPI: + def __init__(self): + self.statements = [] + + def execute_statement(self, statement, warehouse_id, wait_timeout=None): + self.statements.append((warehouse_id, statement)) + + class _Resp: + class status: + state = "SUCCEEDED" + + return _Resp() + + +class _FakeClient: + def __init__(self, warehouses): + self.warehouses = _FakeWarehousesAPI(warehouses) + self.statement_execution = _FakeStmtAPI() + + +def test_resolve_warehouse_prefers_running_serverless(): + client = _FakeClient( + [ + _FakeWarehouse("w_stopped", "STOPPED", serverless=True), + _FakeWarehouse("w_running_classic", "RUNNING", serverless=False), + _FakeWarehouse("w_running_serverless", "RUNNING", serverless=True), + ] + ) + assert R.resolve_warehouse_id(client) == "w_running_serverless" + # explicit id passes through + assert R.resolve_warehouse_id(client, "explicit") == "explicit" + + +def test_resolve_warehouse_none_raises(): + with pytest.raises(RuntimeError): + R.resolve_warehouse_id(_FakeClient([])) + + +def _metadata(tmp_path: Path) -> Path: + md = tmp_path / "metadata" + md.mkdir() + inv = { + "pipelines": [ + {"name": "p1", "activities": [{"name": "a", "type": "DatabricksNotebook", "strategy": "deterministic"}]} + ] + } + (md / "inventory.json").write_text(json.dumps(inv)) + with (md / "profile_report.csv").open("w", newline="") as fh: + w = csv.DictWriter( + fh, + fieldnames=[ + "pipeline", + "activities", + "datasets", + "linked_services", + "collapsible_patterns", + "databricks_native_activities", + "control_flow_activities", + "other_activities", + "complexity_score", + "complexity_size", + ], + ) + w.writeheader() + w.writerow( + { + "pipeline": "p1", + "activities": 1, + "datasets": 0, + "linked_services": 1, + "collapsible_patterns": 0, + "databricks_native_activities": 1, + "control_flow_activities": 0, + "other_activities": 0, + "complexity_score": 2, + "complexity_size": "S", + } + ) + return md + + +def test_write_results_executes_create_then_insert(tmp_path: Path): + client = _FakeClient([_FakeWarehouse("wh1", "RUNNING", serverless=True)]) + run_id, rows = R.write_results(_metadata(tmp_path), "cat.sch.tbl", client=client) + assert rows == 1 + uuid.UUID(run_id) # valid uuid + stmts = client.statement_execution.statements + assert len(stmts) == 2 + assert stmts[0][0] == "wh1" and stmts[0][1].startswith("CREATE TABLE IF NOT EXISTS") + assert stmts[1][1].startswith("INSERT INTO cat.sch.tbl") + assert run_id in stmts[1][1] diff --git a/tests/unit/test_translators.py b/tests/unit/test_translators.py index a4ffa7d..92d52d4 100644 --- a/tests/unit/test_translators.py +++ b/tests/unit/test_translators.py @@ -187,14 +187,7 @@ def test_translate_notebook_resolves_library_with_globals(self): "DatabricksNotebook", { "notebookPath": "/Shared/x", - "libraries": [ - { - "jar": ( - "@concat('/Volumes/x/', " - "pipeline().globalParameters.libFileName)" - ) - } - ], + "libraries": [{"jar": ("@concat('/Volumes/x/', pipeline().globalParameters.libFileName)")}], }, ) # Context with global parameter so the @concat resolves. @@ -281,14 +274,7 @@ def test_translate_notebook_unresolved_library_captured(self): "DatabricksNotebook", { "notebookPath": "/Shared/x", - "libraries": [ - { - "jar": ( - "@concat('/Volumes/x/', " - "pipeline().globalParameters.proj4jLibFileName)" - ) - } - ], + "libraries": [{"jar": ("@concat('/Volumes/x/', pipeline().globalParameters.proj4jLibFileName)")}], }, ) ctx = TranslationContext() @@ -815,7 +801,6 @@ def test_translate_lookup_resolves_json_file_dataset(self): assert result.source_properties["file_name"] == "tables.json" assert result.source_properties.get("multiLineJson") is True - def test_translate_lookup_substitutes_dataset_parameter_refs(self): """C-47 (LSC5-001): a file Lookup whose dataset folderPath references ``dataset().X`` substitutes the dataset reference's parameter bindings @@ -932,9 +917,7 @@ def test_lookup_resolves_dataset_case_insensitively(self): assert result.source_properties["dataset_type"] == "Json" assert result.source_properties["container"] == "configext" # Linked-service URL surfaces so the code generator can build abfss://... - assert result.source_properties["linked_service_url"] == ( - "abfss://configext@myacct.dfs.core.windows.net" - ) + assert result.source_properties["linked_service_url"] == ("abfss://configext@myacct.dfs.core.windows.net") def test_generated_file_lookup_notebook_uses_abfss_path(self): """LSC3-005 end-to-end: generated file-lookup notebook ships a real @@ -1362,9 +1345,7 @@ def test_translate_if_condition_boolean_variable_bridges_when_default_literal_kn from flowx.translator.activity_translators.if_condition import translate # Declared Boolean type AND a seeded literal default -> bridge. - ctx = _context().with_variable_types( - {"continue": "Boolean"}, default_literals={"continue": "true"} - ) + ctx = _context().with_variable_types({"continue": "Boolean"}, default_literals={"continue": "true"}) activity = _make_activity( "Branch", "IfCondition", @@ -1656,7 +1637,6 @@ def _mock_translate(activities, context, definitions): assert result.cases[1].value == "incremental" assert len(result.default_activities) == 1 - def test_translate_switch_function_call_routes_through_bridge(self): """C-07 (CF-iter2-001 / CF-iter2-003): @toUpper(coalesce(...)) on the Switch on-expression lowers to a bridge SetVariable task rather than @@ -1706,9 +1686,7 @@ def test_default_valued_variable_yields_init_task(self): ], variables={"uuid": AdfVariable(type="String", default_value="seed-value")}, ) - definitions = AdfDefinitions( - pipelines=[pipeline], datasets={}, linked_services={}, triggers=[] - ) + definitions = AdfDefinitions(pipelines=[pipeline], datasets={}, linked_services={}, triggers=[]) report = translate_pipeline(pipeline, definitions) # An init task is prepended before the regular activities. task_keys = [t.task_key for t in report.pipeline.tasks] @@ -1736,9 +1714,7 @@ def test_default_valued_boolean_variable_renders_lowercase(self): "continue_f": AdfVariable(type="Boolean", default_value=False), }, ) - definitions = AdfDefinitions( - pipelines=[pipeline], datasets={}, linked_services={}, triggers=[] - ) + definitions = AdfDefinitions(pipelines=[pipeline], datasets={}, linked_services={}, triggers=[]) report = translate_pipeline(pipeline, definitions) init_true = next(t for t in report.pipeline.tasks if t.task_key == "_init_continue_t") init_false = next(t for t in report.pipeline.tasks if t.task_key == "_init_continue_f") @@ -1763,9 +1739,7 @@ def test_set_variable_with_raw_bool_value_renders_lowercase(self): ), ], ) - definitions = AdfDefinitions( - pipelines=[pipeline], datasets={}, linked_services={}, triggers=[] - ) + definitions = AdfDefinitions(pipelines=[pipeline], datasets={}, linked_services={}, triggers=[]) report = translate_pipeline(pipeline, definitions) set_var = next(t for t in report.pipeline.tasks if t.name == "Reset") assert isinstance(set_var, SetVariableActivity) @@ -1815,9 +1789,7 @@ def _build_definitions(self, trigger_props, *, runtime_state="Started", trigger_ properties=props, pipelines=[{"pipelineReference": {"referenceName": "pl_with_trigger"}}], ) - definitions = AdfDefinitions( - pipelines=[pipeline], datasets={}, linked_services={}, triggers=[trigger] - ) + definitions = AdfDefinitions(pipelines=[pipeline], datasets={}, linked_services={}, triggers=[trigger]) return pipeline, definitions def test_schedule_trigger_daily_at_specific_time(self): @@ -1959,9 +1931,7 @@ def test_trigger_carries_per_pipeline_parameter_overrides(self): } ], ) - definitions = AdfDefinitions( - pipelines=[pipeline], datasets={}, linked_services={}, triggers=[trigger] - ) + definitions = AdfDefinitions(pipelines=[pipeline], datasets={}, linked_services={}, triggers=[trigger]) report = translate_pipeline(pipeline, definitions) assert report.pipeline.schedule is not None overrides = report.pipeline.schedule.get("parameter_overrides") or {} diff --git a/tests/unit/test_until_agentic_handler.py b/tests/unit/test_until_agentic_handler.py new file mode 100644 index 0000000..aa97a05 --- /dev/null +++ b/tests/unit/test_until_agentic_handler.py @@ -0,0 +1,71 @@ +"""#1: Until (incl. nested) must surface as an agentic gap carrying the full ARM JSON.""" + +from __future__ import annotations + +from flowx.models.adf_ast import AdfDefinitions +from flowx.models.ir import PlaceholderActivity +from flowx.parser.adf_loader import _parse_pipeline_json +from flowx.translator.engine import translate_pipeline + +_DEFS = AdfDefinitions(pipelines=[], datasets={}, linked_services={}, triggers=[]) + +_PIPELINE = { + "name": "p", + "properties": { + "activities": [ + { + "name": "Gate", + "type": "IfCondition", + "typeProperties": { + "expression": {"value": "@equals(1, 1)", "type": "Expression"}, + "ifTrueActivities": [ + { + "name": "Poll Until Ready", + "type": "Until", + "typeProperties": { + "expression": {"value": "@equals(variables('s'), 'done')", "type": "Expression"}, + "timeout": "0.01:00:00", + "activities": [ + {"name": "Wait A Bit", "type": "Wait", "typeProperties": {"waitTimeInSeconds": 5}} + ], + }, + } + ], + }, + } + ] + }, +} + + +def test_nested_until_gap_carries_full_arm_json(): + adf = _parse_pipeline_json(_PIPELINE, fallback_name="p") + report = translate_pipeline(adf, _DEFS) + until_gaps = [g for g in report.gaps if g.activity_type == "Until"] + assert len(until_gaps) == 1, "nested Until must be reported as a gap" + raw = until_gaps[0].raw_definition + assert raw is not None and raw.get("type") == "Until" + # full ARM JSON, not just typeProperties: name + nested loop body present + assert raw.get("name") == "Poll Until Ready" + assert raw["typeProperties"]["activities"][0]["name"] == "Wait A Bit" + assert until_gaps[0].recommended_skill == "adf-to-databricks:adf-pipeline-converter" + + +def test_until_placeholder_ir_node_carries_arm_json(): + adf = _parse_pipeline_json(_PIPELINE, fallback_name="p") + report = translate_pipeline(adf, _DEFS) + + def _find(tasks): + for t in tasks: + if isinstance(t, PlaceholderActivity) and t.original_type == "Until": + return t + for attr in ("inner_activities", "if_true_activities", "if_false_activities"): + found = _find(getattr(t, attr, []) or []) + if found: + return found + return None + + ph = _find(report.pipeline.tasks) + assert ph is not None and ph.raw_definition is not None + assert ph.raw_definition.get("type") == "Until" + assert ph.agentic_skill == "adf-to-databricks:adf-pipeline-converter" diff --git a/tests/unit/test_web_body_and_param_defaults.py b/tests/unit/test_web_body_and_param_defaults.py new file mode 100644 index 0000000..48e1e48 --- /dev/null +++ b/tests/unit/test_web_body_and_param_defaults.py @@ -0,0 +1,71 @@ +"""Regression tests for #2 parsing fixes: web-activity body expressions and +@utcNow pipeline-parameter defaults.""" + +from __future__ import annotations + +from flowx.models.adf_ast import AdfActivity, AdfDefinitions, AdfParameter, AdfPipeline +from flowx.models.ir import TranslationContext +from flowx.preparer.code_generator import generate_web_activity_notebook +from flowx.translator.activity_translators import web_activity +from flowx.translator.engine import translate_pipeline + +_DEFS = AdfDefinitions(pipelines=[], datasets={}, linked_services={}, triggers=[]) + + +def _web(type_properties: dict, context: TranslationContext): + activity = AdfActivity(name="Notify", type="WebActivity", type_properties=type_properties) + return web_activity.translate(activity, {"name": "Notify", "task_key": "notify"}, context, _DEFS) + + +def test_nested_concat_variables_body_is_lowered_to_python(): + ctx = TranslationContext().with_variable("batchId", "_init_batchId") + ir = _web( + { + "method": "POST", + "url": "https://example.com/hook", + "body": {"text": {"value": "@concat('batch ', variables('batchId'))", "type": "Expression"}}, + }, + ctx, + ) + assert ir.body_code is not None + nb = generate_web_activity_notebook(ir) + assert "@concat" not in nb and "@variables" not in nb + assert "dbutils.widgets.get('batchId')" in nb + assert "'batch ' +" in nb # concatenation, not raw token + + +def test_bare_variables_body_reads_from_widget_and_binds_it(): + ctx = TranslationContext().with_variable("statusMessage", "set_msg") + ir = _web( + { + "method": "POST", + "url": "https://example.com/hook", + "body": {"text": {"value": "@variables('statusMessage')", "type": "Expression"}}, + }, + ctx, + ) + nb = generate_web_activity_notebook(ir) + assert 'dbutils.widgets.get("statusMessage")' in nb + # the dab ref is threaded so the preparer can bind it in base_parameters + assert ir.body_required_parameters.get("statusMessage") == "{{tasks.set_msg.values.statusMessage}}" + + +def test_literal_body_unchanged(): + ir = _web( + {"method": "POST", "url": "https://x", "body": {"status": "completed"}}, + TranslationContext(), + ) + assert ir.body_code is None # pure literal -> generator renders directly + nb = generate_web_activity_notebook(ir) + assert "completed" in nb + + +def test_utcnow_parameter_default_resolves_to_dab_ref(): + pipeline = AdfPipeline( + name="p", + activities=[AdfActivity(name="W", type="Wait", type_properties={"waitTimeInSeconds": 1})], + parameters={"runDate": AdfParameter(type="String", default_value="@utcNow('yyyy-MM-dd')")}, + ) + report = translate_pipeline(pipeline, _DEFS) + run_date = next(p for p in report.pipeline.parameters if p["name"] == "runDate") + assert run_date["default"] == "{{job.start_time.iso_date}}" diff --git a/tests/unit/test_workspace_downloader.py b/tests/unit/test_workspace_downloader.py index da35d7f..69d3483 100644 --- a/tests/unit/test_workspace_downloader.py +++ b/tests/unit/test_workspace_downloader.py @@ -90,10 +90,38 @@ def test_returns_true_when_host_and_token_set(self, monkeypatch): monkeypatch.setenv("DATABRICKS_TOKEN", "dapi-abc") assert auth_available() is True + def test_returns_true_when_oauth_m2m_env_set(self, monkeypatch): + # The MCP path: flowx hosted as a Databricks App injects the service + # principal's OAuth client id/secret (no PAT, no profile). + monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False) + monkeypatch.delenv("DATABRICKS_TOKEN", raising=False) + monkeypatch.setenv("DATABRICKS_HOST", "https://example.cloud.databricks.com") + monkeypatch.setenv("DATABRICKS_CLIENT_ID", "sp-client-id") + monkeypatch.setenv("DATABRICKS_CLIENT_SECRET", "sp-secret") + monkeypatch.setattr(workspace_downloader, "_local_workspace_accessible", lambda: False) + monkeypatch.setattr(workspace_downloader, "_list_profiles", lambda: []) + assert auth_available() is True + def test_returns_false_when_no_env_and_no_profiles(self, monkeypatch): monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False) monkeypatch.delenv("DATABRICKS_HOST", raising=False) monkeypatch.delenv("DATABRICKS_TOKEN", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + monkeypatch.setattr(workspace_downloader, "_local_workspace_accessible", lambda: False) + monkeypatch.setattr(workspace_downloader, "_is_databricks_runtime", lambda: False) + monkeypatch.setattr(workspace_downloader, "_list_profiles", lambda: []) + assert auth_available() is False + + def test_returns_false_when_host_set_without_token_or_client_creds(self, monkeypatch): + # HOST alone must not satisfy the check (incomplete credentials). + monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False) + monkeypatch.delenv("DATABRICKS_TOKEN", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + monkeypatch.setenv("DATABRICKS_HOST", "https://example.cloud.databricks.com") + monkeypatch.setattr(workspace_downloader, "_local_workspace_accessible", lambda: False) + monkeypatch.setattr(workspace_downloader, "_is_databricks_runtime", lambda: False) monkeypatch.setattr(workspace_downloader, "_list_profiles", lambda: []) assert auth_available() is False diff --git a/uv.lock b/uv.lock index 6104941..1d45f14 100644 --- a/uv.lock +++ b/uv.lock @@ -2,22 +2,53 @@ version = 1 revision = 3 requires-python = ">=3.12" +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", upload-time = "2026-03-24T12:59:08.246Z" }, +] + [[package]] name = "argcomplete" version = "3.6.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/38/61/0b9ae6399dd4a58d8c1b1dc5a27d6f2808023d0b5dd3104bb99f45a33ff6/argcomplete-3.6.3.tar.gz", hash = "sha256:62e8ed4fd6a45864acc8235409461b72c9a28ee785a2011cc5eb78318786c89c", size = 73754, upload-time = "2025-10-20T03:33:34.741Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/61/0b9ae6399dd4a58d8c1b1dc5a27d6f2808023d0b5dd3104bb99f45a33ff6/argcomplete-3.6.3.tar.gz", hash = "sha256:62e8ed4fd6a45864acc8235409461b72c9a28ee785a2011cc5eb78318786c89c", upload-time = "2025-10-20T03:33:34.741Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/74/f5/9373290775639cb67a2fce7f629a1c240dce9f12fe927bc32b2736e16dfc/argcomplete-3.6.3-py3-none-any.whl", hash = "sha256:f5007b3a600ccac5d25bbce33089211dfd49eab4a7718da3f10e3082525a92ce", size = 43846, upload-time = "2025-10-20T03:33:33.021Z" }, + { url = "https://files.pythonhosted.org/packages/74/f5/9373290775639cb67a2fce7f629a1c240dce9f12fe927bc32b2736e16dfc/argcomplete-3.6.3-py3-none-any.whl", hash = "sha256:f5007b3a600ccac5d25bbce33089211dfd49eab4a7718da3f10e3082525a92ce", upload-time = "2025-10-20T03:33:33.021Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", upload-time = "2026-03-19T14:22:23.645Z" }, ] [[package]] name = "certifi" version = "2026.5.20" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", upload-time = "2026-05-20T11:46:50.073Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", upload-time = "2026-05-20T11:46:48.578Z" }, ] [[package]] @@ -27,220 +58,232 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, - { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, - { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, - { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, - { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, - { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, - { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, - { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, - { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, - { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, - { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, - { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, - { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, - { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, - { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, - { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, - { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, - { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, - { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, - { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, - { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, - { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, - { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, - { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, - { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, - { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, - { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, - { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, - { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, - { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, - { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, - { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, - { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, - { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, - { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, - { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, - { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, - { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, - { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, - { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, - { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, - { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, - { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", upload-time = "2025-09-08T23:22:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", upload-time = "2025-09-08T23:23:02.231Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", upload-time = "2025-09-08T23:23:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", upload-time = "2025-09-08T23:23:04.792Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", upload-time = "2025-09-08T23:23:06.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", upload-time = "2025-09-08T23:23:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", upload-time = "2025-09-08T23:23:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", upload-time = "2025-09-08T23:23:10.928Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", upload-time = "2025-09-08T23:23:12.42Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", upload-time = "2025-09-08T23:23:18.087Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", upload-time = "2025-09-08T23:23:19.622Z" }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", upload-time = "2025-09-08T23:23:20.853Z" }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", upload-time = "2025-09-08T23:23:22.08Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", upload-time = "2025-09-08T23:23:23.314Z" }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", upload-time = "2025-09-08T23:23:24.541Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", upload-time = "2025-09-08T23:23:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", upload-time = "2025-09-08T23:23:27.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", upload-time = "2025-09-08T23:23:29.347Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", upload-time = "2025-09-08T23:23:30.63Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", upload-time = "2025-09-08T23:23:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", upload-time = "2025-09-08T23:23:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", upload-time = "2025-09-08T23:23:34.495Z" }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", upload-time = "2025-09-08T23:23:36.096Z" }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", upload-time = "2025-09-08T23:23:37.328Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", upload-time = "2025-09-08T23:23:38.945Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", upload-time = "2025-09-08T23:23:43.004Z" }, ] [[package]] name = "charset-normalizer" version = "3.4.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, - { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, - { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, - { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, - { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, - { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, - { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, - { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, - { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, - { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, - { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, - { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, - { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, - { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, - { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, - { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, - { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, - { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, - { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, - { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, - { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, - { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, - { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, - { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, - { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, - { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, - { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, - { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, - { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, - { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, - { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, - { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, - { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, - { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, - { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, - { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, - { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, - { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, - { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, - { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, - { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, - { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, - { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, - { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, - { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, - { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, - { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, - { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, - { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, - { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, - { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, - { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, - { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, - { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, - { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, - { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, - { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, - { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, - { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, - { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, - { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", upload-time = "2026-04-02T09:26:24.331Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", upload-time = "2026-04-02T09:26:25.568Z" }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", upload-time = "2026-04-02T09:26:26.865Z" }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", upload-time = "2026-04-02T09:26:28.044Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", upload-time = "2026-04-02T09:26:29.239Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", upload-time = "2026-04-02T09:26:30.5Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", upload-time = "2026-04-02T09:26:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", upload-time = "2026-04-02T09:26:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", upload-time = "2026-04-02T09:26:34.845Z" }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", upload-time = "2026-04-02T09:26:36.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", upload-time = "2026-04-02T09:26:37.672Z" }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", upload-time = "2026-04-02T09:26:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", upload-time = "2026-04-02T09:26:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", upload-time = "2026-04-02T09:26:41.416Z" }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", upload-time = "2026-04-02T09:26:42.554Z" }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", upload-time = "2026-04-02T09:26:44.075Z" }, + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", upload-time = "2026-04-02T09:26:45.198Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", upload-time = "2026-04-02T09:26:46.824Z" }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", upload-time = "2026-04-02T09:26:48.397Z" }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", upload-time = "2026-04-02T09:26:49.684Z" }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", upload-time = "2026-04-02T09:26:50.915Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", upload-time = "2026-04-02T09:26:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", upload-time = "2026-04-02T09:26:53.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", upload-time = "2026-04-02T09:26:54.975Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", upload-time = "2026-04-02T09:26:56.303Z" }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", upload-time = "2026-04-02T09:26:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", upload-time = "2026-04-02T09:26:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", upload-time = "2026-04-02T09:27:00.437Z" }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", upload-time = "2026-04-02T09:27:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", upload-time = "2026-04-02T09:27:03.192Z" }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", upload-time = "2026-04-02T09:27:04.454Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", upload-time = "2026-04-02T09:27:05.971Z" }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", upload-time = "2026-04-02T09:27:07.194Z" }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", upload-time = "2026-04-02T09:27:08.749Z" }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", upload-time = "2026-04-02T09:27:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", upload-time = "2026-04-02T09:27:11.175Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", upload-time = "2026-04-02T09:27:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", upload-time = "2026-04-02T09:27:13.721Z" }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", upload-time = "2026-04-02T09:27:15.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", upload-time = "2026-04-02T09:27:16.834Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", upload-time = "2026-04-02T09:27:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", upload-time = "2026-04-02T09:27:19.445Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", upload-time = "2026-04-02T09:27:20.79Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", upload-time = "2026-04-02T09:27:22.063Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", upload-time = "2026-04-02T09:27:23.486Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", upload-time = "2026-04-02T09:27:25.146Z" }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", upload-time = "2026-04-02T09:27:26.642Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", upload-time = "2026-04-02T09:27:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", upload-time = "2026-04-02T09:27:29.474Z" }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", upload-time = "2026-04-02T09:27:30.793Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", upload-time = "2026-04-02T09:27:32.44Z" }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", upload-time = "2026-04-02T09:27:34.03Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", upload-time = "2026-04-02T09:27:35.369Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", upload-time = "2026-04-02T09:27:36.661Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", upload-time = "2026-04-02T09:27:38.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", upload-time = "2026-04-02T09:27:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", upload-time = "2026-04-02T09:27:40.722Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", upload-time = "2026-04-02T09:27:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", upload-time = "2026-04-02T09:27:43.924Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", upload-time = "2026-04-02T09:27:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", upload-time = "2026-04-02T09:27:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", upload-time = "2026-04-02T09:27:48.053Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", upload-time = "2026-04-02T09:27:49.795Z" }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", upload-time = "2026-04-02T09:27:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", upload-time = "2026-04-02T09:28:37.794Z" }, +] + +[[package]] +name = "click" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", upload-time = "2026-05-22T04:08:37.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", upload-time = "2026-05-22T04:08:35.26Z" }, ] [[package]] name = "colorama" version = "0.4.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", upload-time = "2022-10-25T02:36:20.889Z" }, ] [[package]] name = "coverage" version = "7.13.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", size = 219554, upload-time = "2026-03-17T10:30:42.208Z" }, - { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", size = 219908, upload-time = "2026-03-17T10:30:43.906Z" }, - { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", size = 251419, upload-time = "2026-03-17T10:30:45.545Z" }, - { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", size = 254159, upload-time = "2026-03-17T10:30:47.204Z" }, - { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", size = 255270, upload-time = "2026-03-17T10:30:48.812Z" }, - { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", size = 257538, upload-time = "2026-03-17T10:30:50.77Z" }, - { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", size = 251821, upload-time = "2026-03-17T10:30:52.5Z" }, - { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", size = 253191, upload-time = "2026-03-17T10:30:54.543Z" }, - { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", size = 251337, upload-time = "2026-03-17T10:30:56.663Z" }, - { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", size = 255404, upload-time = "2026-03-17T10:30:58.427Z" }, - { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", size = 250903, upload-time = "2026-03-17T10:31:00.093Z" }, - { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", size = 252780, upload-time = "2026-03-17T10:31:01.916Z" }, - { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", size = 222093, upload-time = "2026-03-17T10:31:03.642Z" }, - { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", size = 222900, upload-time = "2026-03-17T10:31:05.651Z" }, - { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", size = 221515, upload-time = "2026-03-17T10:31:07.293Z" }, - { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" }, - { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" }, - { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" }, - { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" }, - { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" }, - { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" }, - { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" }, - { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" }, - { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" }, - { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" }, - { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" }, - { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" }, - { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" }, - { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" }, - { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" }, - { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" }, - { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" }, - { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" }, - { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" }, - { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" }, - { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" }, - { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" }, - { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" }, - { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" }, - { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" }, - { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" }, - { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" }, - { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" }, - { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" }, - { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" }, - { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" }, - { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" }, - { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" }, - { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" }, - { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" }, - { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" }, - { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" }, - { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" }, - { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" }, - { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" }, - { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" }, - { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" }, - { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" }, - { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" }, - { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" }, - { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" }, - { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" }, - { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" }, - { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" }, - { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" }, - { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" }, - { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" }, - { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" }, - { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" }, - { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" }, - { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" }, - { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", upload-time = "2026-03-17T10:33:18.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", upload-time = "2026-03-17T10:30:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", upload-time = "2026-03-17T10:30:43.906Z" }, + { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", upload-time = "2026-03-17T10:30:45.545Z" }, + { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", upload-time = "2026-03-17T10:30:47.204Z" }, + { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", upload-time = "2026-03-17T10:30:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", upload-time = "2026-03-17T10:30:50.77Z" }, + { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", upload-time = "2026-03-17T10:30:52.5Z" }, + { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", upload-time = "2026-03-17T10:30:54.543Z" }, + { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", upload-time = "2026-03-17T10:30:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", upload-time = "2026-03-17T10:30:58.427Z" }, + { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", upload-time = "2026-03-17T10:31:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", upload-time = "2026-03-17T10:31:01.916Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", upload-time = "2026-03-17T10:31:03.642Z" }, + { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", upload-time = "2026-03-17T10:31:05.651Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", upload-time = "2026-03-17T10:31:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", upload-time = "2026-03-17T10:31:09.045Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", upload-time = "2026-03-17T10:31:10.708Z" }, + { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", upload-time = "2026-03-17T10:31:12.392Z" }, + { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", upload-time = "2026-03-17T10:31:14.247Z" }, + { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", upload-time = "2026-03-17T10:31:16.193Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", upload-time = "2026-03-17T10:31:17.89Z" }, + { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", upload-time = "2026-03-17T10:31:19.605Z" }, + { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", upload-time = "2026-03-17T10:31:21.312Z" }, + { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", upload-time = "2026-03-17T10:31:23.565Z" }, + { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", upload-time = "2026-03-17T10:31:25.58Z" }, + { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", upload-time = "2026-03-17T10:31:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", upload-time = "2026-03-17T10:31:29.472Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", upload-time = "2026-03-17T10:31:31.526Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", upload-time = "2026-03-17T10:31:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", upload-time = "2026-03-17T10:31:35.445Z" }, + { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", upload-time = "2026-03-17T10:31:37.184Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", upload-time = "2026-03-17T10:31:39.245Z" }, + { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", upload-time = "2026-03-17T10:31:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", upload-time = "2026-03-17T10:31:43.724Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", upload-time = "2026-03-17T10:31:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", upload-time = "2026-03-17T10:31:48.293Z" }, + { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", upload-time = "2026-03-17T10:31:50.125Z" }, + { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", upload-time = "2026-03-17T10:31:51.961Z" }, + { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", upload-time = "2026-03-17T10:31:53.872Z" }, + { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", upload-time = "2026-03-17T10:31:55.997Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", upload-time = "2026-03-17T10:31:58.308Z" }, + { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", upload-time = "2026-03-17T10:32:00.141Z" }, + { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", upload-time = "2026-03-17T10:32:02.246Z" }, + { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", upload-time = "2026-03-17T10:32:04.416Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", upload-time = "2026-03-17T10:32:06.65Z" }, + { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", upload-time = "2026-03-17T10:32:08.589Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", upload-time = "2026-03-17T10:32:10.507Z" }, + { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", upload-time = "2026-03-17T10:32:12.41Z" }, + { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", upload-time = "2026-03-17T10:32:14.449Z" }, + { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", upload-time = "2026-03-17T10:32:16.56Z" }, + { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", upload-time = "2026-03-17T10:32:19.004Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", upload-time = "2026-03-17T10:32:21.344Z" }, + { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", upload-time = "2026-03-17T10:32:23.506Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", upload-time = "2026-03-17T10:32:25.516Z" }, + { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", upload-time = "2026-03-17T10:32:27.944Z" }, + { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", upload-time = "2026-03-17T10:32:29.914Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", upload-time = "2026-03-17T10:32:31.981Z" }, + { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", upload-time = "2026-03-17T10:32:34.233Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", upload-time = "2026-03-17T10:32:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", upload-time = "2026-03-17T10:32:38.736Z" }, + { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", upload-time = "2026-03-17T10:32:41.196Z" }, + { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", upload-time = "2026-03-17T10:32:43.204Z" }, + { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", upload-time = "2026-03-17T10:32:45.514Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", upload-time = "2026-03-17T10:32:47.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", upload-time = "2026-03-17T10:32:50.1Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", upload-time = "2026-03-17T10:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", upload-time = "2026-03-17T10:32:54.544Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", upload-time = "2026-03-17T10:32:56.803Z" }, + { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", upload-time = "2026-03-17T10:32:59.466Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", upload-time = "2026-03-17T10:33:01.847Z" }, + { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", upload-time = "2026-03-17T10:33:03.945Z" }, + { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", upload-time = "2026-03-17T10:33:06.285Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", upload-time = "2026-03-17T10:33:08.756Z" }, + { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", upload-time = "2026-03-17T10:33:11.174Z" }, + { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", upload-time = "2026-03-17T10:33:13.466Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", upload-time = "2026-03-17T10:33:15.691Z" }, ] [[package]] @@ -250,52 +293,102 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", size = 832984, upload-time = "2026-05-04T22:59:38.133Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", size = 8001587, upload-time = "2026-05-04T22:57:36.803Z" }, - { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", size = 4690433, upload-time = "2026-05-04T22:57:40.373Z" }, - { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", size = 4710620, upload-time = "2026-05-04T22:57:42.935Z" }, - { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", size = 4696283, upload-time = "2026-05-04T22:57:45.294Z" }, - { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", size = 5296573, upload-time = "2026-05-04T22:57:47.933Z" }, - { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", size = 4743677, upload-time = "2026-05-04T22:57:50.067Z" }, - { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", size = 4330808, upload-time = "2026-05-04T22:57:52.301Z" }, - { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", size = 4695941, upload-time = "2026-05-04T22:57:54.603Z" }, - { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", size = 5252579, upload-time = "2026-05-04T22:57:57.207Z" }, - { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", size = 4743326, upload-time = "2026-05-04T22:57:59.535Z" }, - { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", size = 4826672, upload-time = "2026-05-04T22:58:01.996Z" }, - { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", size = 4972574, upload-time = "2026-05-04T22:58:03.968Z" }, - { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", size = 3294868, upload-time = "2026-05-04T22:58:06.467Z" }, - { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", size = 3817107, upload-time = "2026-05-04T22:58:08.845Z" }, - { url = "https://files.pythonhosted.org/packages/6b/84/70e3feea9feea87fd7cbe77efb2712ae1e3e6edf10749dc6e95f4e60e455/cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec", size = 7986556, upload-time = "2026-05-04T22:58:11.172Z" }, - { url = "https://files.pythonhosted.org/packages/89/6e/18e07a618bb5442ba10cf4df16e99c071365528aa570dfcb8c02e25a303b/cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18", size = 4684776, upload-time = "2026-05-04T22:58:13.712Z" }, - { url = "https://files.pythonhosted.org/packages/be/6a/4ea3b4c6c6759794d5ee2103c304a5076dc4b19ae1f9fe47dba439e159e9/cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20", size = 4698121, upload-time = "2026-05-04T22:58:16.448Z" }, - { url = "https://files.pythonhosted.org/packages/2f/59/6ff6ad6cae03bb887da2a5860b2c9805f8dac969ef01ce563336c49bd1d1/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff", size = 4690042, upload-time = "2026-05-04T22:58:18.544Z" }, - { url = "https://files.pythonhosted.org/packages/ca/b4/fc334ed8cfd705aca282fe4d8f5ae64a8e0f74932e9feecb344610cf6e4d/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c", size = 5282526, upload-time = "2026-05-04T22:58:20.75Z" }, - { url = "https://files.pythonhosted.org/packages/11/08/9f8c5386cc4cd90d8255c7cdd0f5baf459a08502a09de30dc51f553d38dc/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db", size = 4733116, upload-time = "2026-05-04T22:58:23.627Z" }, - { url = "https://files.pythonhosted.org/packages/b8/77/99307d7574045699f8805aa500fa0fb83422d115b5400a064ddd306d7750/cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741", size = 4316030, upload-time = "2026-05-04T22:58:25.581Z" }, - { url = "https://files.pythonhosted.org/packages/fd/36/a608b98337af3cb2aff4818e406649d30572b7031918b04c87d979495348/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166", size = 4689640, upload-time = "2026-05-04T22:58:27.747Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a6/825010a291b4438aecc1f568bc428189fc1175515223632477c07dc0a6df/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336", size = 5237657, upload-time = "2026-05-04T22:58:29.848Z" }, - { url = "https://files.pythonhosted.org/packages/b9/09/4e76a09b4caa29aad535ddc806f5d4c5d01885bd978bd984fbc6ca032cae/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057", size = 4732362, upload-time = "2026-05-04T22:58:32.009Z" }, - { url = "https://files.pythonhosted.org/packages/18/78/444fa04a77d0cb95f417dda20d450e13c56ba8e5220fc892a1658f44f882/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae", size = 4819580, upload-time = "2026-05-04T22:58:34.254Z" }, - { url = "https://files.pythonhosted.org/packages/38/85/ea67067c70a1fd4be2c63d35eeed82658023021affccc7b17705f8527dd2/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c", size = 4963283, upload-time = "2026-05-04T22:58:36.376Z" }, - { url = "https://files.pythonhosted.org/packages/75/54/cc6d0f3deac3e81c7f847e8a189a12b6cdd65059b43dad25d4316abd849a/cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f", size = 3270954, upload-time = "2026-05-04T22:58:38.791Z" }, - { url = "https://files.pythonhosted.org/packages/49/67/cc947e288c0758a4e5473d1dcb743037ab7785541265a969240b8885441a/cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12", size = 3797313, upload-time = "2026-05-04T22:58:40.746Z" }, - { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", size = 7983482, upload-time = "2026-05-04T22:58:43.769Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", size = 4683266, upload-time = "2026-05-04T22:58:46.064Z" }, - { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", size = 4696228, upload-time = "2026-05-04T22:58:48.22Z" }, - { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", size = 4689097, upload-time = "2026-05-04T22:58:50.9Z" }, - { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", size = 5283582, upload-time = "2026-05-04T22:58:53.017Z" }, - { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", size = 4730479, upload-time = "2026-05-04T22:58:55.611Z" }, - { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", size = 4326481, upload-time = "2026-05-04T22:58:57.607Z" }, - { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", size = 4688713, upload-time = "2026-05-04T22:59:00.077Z" }, - { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", size = 5238165, upload-time = "2026-05-04T22:59:02.317Z" }, - { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", size = 4729947, upload-time = "2026-05-04T22:59:05.255Z" }, - { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", size = 4822059, upload-time = "2026-05-04T22:59:07.802Z" }, - { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", size = 4960575, upload-time = "2026-05-04T22:59:09.851Z" }, - { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", size = 3279117, upload-time = "2026-05-04T22:59:12.019Z" }, - { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", size = 3792100, upload-time = "2026-05-04T22:59:14.884Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", upload-time = "2026-05-04T22:59:38.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", upload-time = "2026-05-04T22:57:36.803Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", upload-time = "2026-05-04T22:57:40.373Z" }, + { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", upload-time = "2026-05-04T22:57:42.935Z" }, + { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", upload-time = "2026-05-04T22:57:45.294Z" }, + { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", upload-time = "2026-05-04T22:57:47.933Z" }, + { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", upload-time = "2026-05-04T22:57:50.067Z" }, + { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", upload-time = "2026-05-04T22:57:52.301Z" }, + { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", upload-time = "2026-05-04T22:57:54.603Z" }, + { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", upload-time = "2026-05-04T22:57:57.207Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", upload-time = "2026-05-04T22:57:59.535Z" }, + { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", upload-time = "2026-05-04T22:58:01.996Z" }, + { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", upload-time = "2026-05-04T22:58:03.968Z" }, + { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", upload-time = "2026-05-04T22:58:06.467Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", upload-time = "2026-05-04T22:58:08.845Z" }, + { url = "https://files.pythonhosted.org/packages/6b/84/70e3feea9feea87fd7cbe77efb2712ae1e3e6edf10749dc6e95f4e60e455/cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec", upload-time = "2026-05-04T22:58:11.172Z" }, + { url = "https://files.pythonhosted.org/packages/89/6e/18e07a618bb5442ba10cf4df16e99c071365528aa570dfcb8c02e25a303b/cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18", upload-time = "2026-05-04T22:58:13.712Z" }, + { url = "https://files.pythonhosted.org/packages/be/6a/4ea3b4c6c6759794d5ee2103c304a5076dc4b19ae1f9fe47dba439e159e9/cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20", upload-time = "2026-05-04T22:58:16.448Z" }, + { url = "https://files.pythonhosted.org/packages/2f/59/6ff6ad6cae03bb887da2a5860b2c9805f8dac969ef01ce563336c49bd1d1/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff", upload-time = "2026-05-04T22:58:18.544Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b4/fc334ed8cfd705aca282fe4d8f5ae64a8e0f74932e9feecb344610cf6e4d/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c", upload-time = "2026-05-04T22:58:20.75Z" }, + { url = "https://files.pythonhosted.org/packages/11/08/9f8c5386cc4cd90d8255c7cdd0f5baf459a08502a09de30dc51f553d38dc/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db", upload-time = "2026-05-04T22:58:23.627Z" }, + { url = "https://files.pythonhosted.org/packages/b8/77/99307d7574045699f8805aa500fa0fb83422d115b5400a064ddd306d7750/cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741", upload-time = "2026-05-04T22:58:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/fd/36/a608b98337af3cb2aff4818e406649d30572b7031918b04c87d979495348/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166", upload-time = "2026-05-04T22:58:27.747Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a6/825010a291b4438aecc1f568bc428189fc1175515223632477c07dc0a6df/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336", upload-time = "2026-05-04T22:58:29.848Z" }, + { url = "https://files.pythonhosted.org/packages/b9/09/4e76a09b4caa29aad535ddc806f5d4c5d01885bd978bd984fbc6ca032cae/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057", upload-time = "2026-05-04T22:58:32.009Z" }, + { url = "https://files.pythonhosted.org/packages/18/78/444fa04a77d0cb95f417dda20d450e13c56ba8e5220fc892a1658f44f882/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae", upload-time = "2026-05-04T22:58:34.254Z" }, + { url = "https://files.pythonhosted.org/packages/38/85/ea67067c70a1fd4be2c63d35eeed82658023021affccc7b17705f8527dd2/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c", upload-time = "2026-05-04T22:58:36.376Z" }, + { url = "https://files.pythonhosted.org/packages/75/54/cc6d0f3deac3e81c7f847e8a189a12b6cdd65059b43dad25d4316abd849a/cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f", upload-time = "2026-05-04T22:58:38.791Z" }, + { url = "https://files.pythonhosted.org/packages/49/67/cc947e288c0758a4e5473d1dcb743037ab7785541265a969240b8885441a/cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12", upload-time = "2026-05-04T22:58:40.746Z" }, + { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", upload-time = "2026-05-04T22:58:43.769Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", upload-time = "2026-05-04T22:58:46.064Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", upload-time = "2026-05-04T22:58:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", upload-time = "2026-05-04T22:58:50.9Z" }, + { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", upload-time = "2026-05-04T22:58:53.017Z" }, + { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", upload-time = "2026-05-04T22:58:55.611Z" }, + { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", upload-time = "2026-05-04T22:58:57.607Z" }, + { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", upload-time = "2026-05-04T22:59:00.077Z" }, + { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", upload-time = "2026-05-04T22:59:02.317Z" }, + { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", upload-time = "2026-05-04T22:59:05.255Z" }, + { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", upload-time = "2026-05-04T22:59:07.802Z" }, + { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", upload-time = "2026-05-04T22:59:09.851Z" }, + { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", upload-time = "2026-05-04T22:59:12.019Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", upload-time = "2026-05-04T22:59:14.884Z" }, ] +[[package]] +name = "databricks-flowx" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "databricks-sdk" }, + { name = "pyyaml" }, + { name = "sqlglot" }, +] + +[package.optional-dependencies] +mcp = [ + { name = "mcp" }, + { name = "starlette" }, + { name = "uvicorn" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage" }, + { name = "mypy" }, + { name = "pytest" }, + { name = "ruff" }, + { name = "types-pyyaml" }, +] +yq = [ + { name = "yq" }, +] + +[package.metadata] +requires-dist = [ + { name = "databricks-sdk", specifier = ">=0.40" }, + { name = "mcp", marker = "extra == 'mcp'", specifier = ">=1.12" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=25.0" }, + { name = "starlette", marker = "extra == 'mcp'", specifier = ">=0.40" }, + { name = "uvicorn", marker = "extra == 'mcp'", specifier = ">=0.30" }, +] +provides-extras = ["mcp"] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", specifier = ">=7.6.1,<8" }, + { name = "mypy", specifier = ">=1.18.2,<2" }, + { name = "pytest", specifier = ">=8.3.3,<9" }, + { name = "ruff", specifier = ">=0.14.0,<1" }, + { name = "types-pyyaml", specifier = ">=6.0.12.20250915,<7" }, +] +yq = [{ name = "yq", specifier = "~=3.4.3" }] + [[package]] name = "databricks-sdk" version = "0.110.0" @@ -305,9 +398,9 @@ dependencies = [ { name = "protobuf" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/0f/488d61ece084f70a6d4d0ab8b5e38b0902e0b9029d0b72cde99e3f2c6b4a/databricks_sdk-0.110.0.tar.gz", hash = "sha256:b62d806982b37f8160f700d657c37b3bd586c649eb5c8c4c1216090d888c5820", size = 945261, upload-time = "2026-05-19T09:18:46.23Z" } +sdist = { url = "https://files.pythonhosted.org/packages/07/0f/488d61ece084f70a6d4d0ab8b5e38b0902e0b9029d0b72cde99e3f2c6b4a/databricks_sdk-0.110.0.tar.gz", hash = "sha256:b62d806982b37f8160f700d657c37b3bd586c649eb5c8c4c1216090d888c5820", upload-time = "2026-05-19T09:18:46.23Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/23/7c2a827890ab120ac349847ec17ab5a37eb4e3bf8f1d0989fd9eec0c1e6a/databricks_sdk-0.110.0-py3-none-any.whl", hash = "sha256:8a23db05be7a304bea43b4fa78b437051ed0f3755b19594429c649ee4159b546", size = 892096, upload-time = "2026-05-19T09:18:44.313Z" }, + { url = "https://files.pythonhosted.org/packages/9d/23/7c2a827890ab120ac349847ec17ab5a37eb4e3bf8f1d0989fd9eec0c1e6a/databricks_sdk-0.110.0-py3-none-any.whl", hash = "sha256:8a23db05be7a304bea43b4fa78b437051ed0f3755b19594429c649ee4159b546", upload-time = "2026-05-19T09:18:44.313Z" }, ] [[package]] @@ -318,87 +411,185 @@ dependencies = [ { name = "cryptography" }, { name = "pyasn1-modules" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c6/ad/ff781329bbbdc0974a098d996e89c9e1f7024262f9e3eec442fbb9ad1ac6/google_auth-2.53.0.tar.gz", hash = "sha256:e7e6aa16f6bee7b2b264830fd04f08087a1d5a836df516251a5d15327b246c9c", size = 335844, upload-time = "2026-05-15T20:53:07.928Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/ad/ff781329bbbdc0974a098d996e89c9e1f7024262f9e3eec442fbb9ad1ac6/google_auth-2.53.0.tar.gz", hash = "sha256:e7e6aa16f6bee7b2b264830fd04f08087a1d5a836df516251a5d15327b246c9c", upload-time = "2026-05-15T20:53:07.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/c9/db44165ba7c581268c6d46017ef63339110378305062830104fc7fa144cb/google_auth-2.53.0-py3-none-any.whl", hash = "sha256:6e7449917c599b35126a99ec268ec6880301f2fea41dce198fe8fd83ff642b68", upload-time = "2026-05-15T20:53:05.609Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", upload-time = "2025-04-24T03:35:25.427Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/c9/db44165ba7c581268c6d46017ef63339110378305062830104fc7fa144cb/google_auth-2.53.0-py3-none-any.whl", hash = "sha256:6e7449917c599b35126a99ec268ec6880301f2fea41dce198fe8fd83ff642b68", size = 246071, upload-time = "2026-05-15T20:53:05.609Z" }, + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "httpx-sse" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", upload-time = "2025-10-10T21:48:22.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", upload-time = "2025-10-10T21:48:21.158Z" }, ] [[package]] name = "idna" version = "3.15" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", upload-time = "2026-05-12T22:45:57.011Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, + { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", upload-time = "2026-05-12T22:45:55.733Z" }, ] [[package]] name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", upload-time = "2025-09-08T01:34:59.186Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", upload-time = "2025-09-08T01:34:57.871Z" }, ] [[package]] name = "librt" version = "0.8.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/56/9c/b4b0c54d84da4a94b37bd44151e46d5e583c9534c7e02250b961b1b6d8a8/librt-0.8.1.tar.gz", hash = "sha256:be46a14693955b3bd96014ccbdb8339ee8c9346fbe11c1b78901b55125f14c73", size = 177471, upload-time = "2026-02-17T16:13:06.101Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/21/d39b0a87ac52fc98f621fb6f8060efb017a767ebbbac2f99fbcbc9ddc0d7/librt-0.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a28f2612ab566b17f3698b0da021ff9960610301607c9a5e8eaca62f5e1c350a", size = 66516, upload-time = "2026-02-17T16:11:41.604Z" }, - { url = "https://files.pythonhosted.org/packages/69/f1/46375e71441c43e8ae335905e069f1c54febee63a146278bcee8782c84fd/librt-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:60a78b694c9aee2a0f1aaeaa7d101cf713e92e8423a941d2897f4fa37908dab9", size = 68634, upload-time = "2026-02-17T16:11:43.268Z" }, - { url = "https://files.pythonhosted.org/packages/0a/33/c510de7f93bf1fa19e13423a606d8189a02624a800710f6e6a0a0f0784b3/librt-0.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:758509ea3f1eba2a57558e7e98f4659d0ea7670bff49673b0dde18a3c7e6c0eb", size = 198941, upload-time = "2026-02-17T16:11:44.28Z" }, - { url = "https://files.pythonhosted.org/packages/dd/36/e725903416409a533d92398e88ce665476f275081d0d7d42f9c4951999e5/librt-0.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:039b9f2c506bd0ab0f8725aa5ba339c6f0cd19d3b514b50d134789809c24285d", size = 209991, upload-time = "2026-02-17T16:11:45.462Z" }, - { url = "https://files.pythonhosted.org/packages/30/7a/8d908a152e1875c9f8eac96c97a480df425e657cdb47854b9efaa4998889/librt-0.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bb54f1205a3a6ab41a6fd71dfcdcbd278670d3a90ca502a30d9da583105b6f7", size = 224476, upload-time = "2026-02-17T16:11:46.542Z" }, - { url = "https://files.pythonhosted.org/packages/a8/b8/a22c34f2c485b8903a06f3fe3315341fe6876ef3599792344669db98fcff/librt-0.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:05bd41cdee35b0c59c259f870f6da532a2c5ca57db95b5f23689fcb5c9e42440", size = 217518, upload-time = "2026-02-17T16:11:47.746Z" }, - { url = "https://files.pythonhosted.org/packages/79/6f/5c6fea00357e4f82ba44f81dbfb027921f1ab10e320d4a64e1c408d035d9/librt-0.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adfab487facf03f0d0857b8710cf82d0704a309d8ffc33b03d9302b4c64e91a9", size = 225116, upload-time = "2026-02-17T16:11:49.298Z" }, - { url = "https://files.pythonhosted.org/packages/f2/a0/95ced4e7b1267fe1e2720a111685bcddf0e781f7e9e0ce59d751c44dcfe5/librt-0.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:153188fe98a72f206042be10a2c6026139852805215ed9539186312d50a8e972", size = 217751, upload-time = "2026-02-17T16:11:50.49Z" }, - { url = "https://files.pythonhosted.org/packages/93/c2/0517281cb4d4101c27ab59472924e67f55e375bc46bedae94ac6dc6e1902/librt-0.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:dd3c41254ee98604b08bd5b3af5bf0a89740d4ee0711de95b65166bf44091921", size = 218378, upload-time = "2026-02-17T16:11:51.783Z" }, - { url = "https://files.pythonhosted.org/packages/43/e8/37b3ac108e8976888e559a7b227d0ceac03c384cfd3e7a1c2ee248dbae79/librt-0.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e0d138c7ae532908cbb342162b2611dbd4d90c941cd25ab82084aaf71d2c0bd0", size = 241199, upload-time = "2026-02-17T16:11:53.561Z" }, - { url = "https://files.pythonhosted.org/packages/4b/5b/35812d041c53967fedf551a39399271bbe4257e681236a2cf1a69c8e7fa1/librt-0.8.1-cp312-cp312-win32.whl", hash = "sha256:43353b943613c5d9c49a25aaffdba46f888ec354e71e3529a00cca3f04d66a7a", size = 54917, upload-time = "2026-02-17T16:11:54.758Z" }, - { url = "https://files.pythonhosted.org/packages/de/d1/fa5d5331b862b9775aaf2a100f5ef86854e5d4407f71bddf102f4421e034/librt-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:ff8baf1f8d3f4b6b7257fcb75a501f2a5499d0dda57645baa09d4d0d34b19444", size = 62017, upload-time = "2026-02-17T16:11:55.748Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7c/c614252f9acda59b01a66e2ddfd243ed1c7e1deab0293332dfbccf862808/librt-0.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:0f2ae3725904f7377e11cc37722d5d401e8b3d5851fb9273d7f4fe04f6b3d37d", size = 52441, upload-time = "2026-02-17T16:11:56.801Z" }, - { url = "https://files.pythonhosted.org/packages/c5/3c/f614c8e4eaac7cbf2bbdf9528790b21d89e277ee20d57dc6e559c626105f/librt-0.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e6bad1cd94f6764e1e21950542f818a09316645337fd5ab9a7acc45d99a8f35", size = 66529, upload-time = "2026-02-17T16:11:57.809Z" }, - { url = "https://files.pythonhosted.org/packages/ab/96/5836544a45100ae411eda07d29e3d99448e5258b6e9c8059deb92945f5c2/librt-0.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cf450f498c30af55551ba4f66b9123b7185362ec8b625a773b3d39aa1a717583", size = 68669, upload-time = "2026-02-17T16:11:58.843Z" }, - { url = "https://files.pythonhosted.org/packages/06/53/f0b992b57af6d5531bf4677d75c44f095f2366a1741fb695ee462ae04b05/librt-0.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:eca45e982fa074090057132e30585a7e8674e9e885d402eae85633e9f449ce6c", size = 199279, upload-time = "2026-02-17T16:11:59.862Z" }, - { url = "https://files.pythonhosted.org/packages/f3/ad/4848cc16e268d14280d8168aee4f31cea92bbd2b79ce33d3e166f2b4e4fc/librt-0.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c3811485fccfda840861905b8c70bba5ec094e02825598bb9d4ca3936857a04", size = 210288, upload-time = "2026-02-17T16:12:00.954Z" }, - { url = "https://files.pythonhosted.org/packages/52/05/27fdc2e95de26273d83b96742d8d3b7345f2ea2bdbd2405cc504644f2096/librt-0.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e4af413908f77294605e28cfd98063f54b2c790561383971d2f52d113d9c363", size = 224809, upload-time = "2026-02-17T16:12:02.108Z" }, - { url = "https://files.pythonhosted.org/packages/7a/d0/78200a45ba3240cb042bc597d6f2accba9193a2c57d0356268cbbe2d0925/librt-0.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5212a5bd7fae98dae95710032902edcd2ec4dc994e883294f75c857b83f9aba0", size = 218075, upload-time = "2026-02-17T16:12:03.631Z" }, - { url = "https://files.pythonhosted.org/packages/af/72/a210839fa74c90474897124c064ffca07f8d4b347b6574d309686aae7ca6/librt-0.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e692aa2d1d604e6ca12d35e51fdc36f4cda6345e28e36374579f7ef3611b3012", size = 225486, upload-time = "2026-02-17T16:12:04.725Z" }, - { url = "https://files.pythonhosted.org/packages/a3/c1/a03cc63722339ddbf087485f253493e2b013039f5b707e8e6016141130fa/librt-0.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4be2a5c926b9770c9e08e717f05737a269b9d0ebc5d2f0060f0fe3fe9ce47acb", size = 218219, upload-time = "2026-02-17T16:12:05.828Z" }, - { url = "https://files.pythonhosted.org/packages/58/f5/fff6108af0acf941c6f274a946aea0e484bd10cd2dc37610287ce49388c5/librt-0.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fd1a720332ea335ceb544cf0a03f81df92abd4bb887679fd1e460976b0e6214b", size = 218750, upload-time = "2026-02-17T16:12:07.09Z" }, - { url = "https://files.pythonhosted.org/packages/71/67/5a387bfef30ec1e4b4f30562c8586566faf87e47d696768c19feb49e3646/librt-0.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2af9e01e0ef80d95ae3c720be101227edae5f2fe7e3dc63d8857fadfc5a1d", size = 241624, upload-time = "2026-02-17T16:12:08.43Z" }, - { url = "https://files.pythonhosted.org/packages/d4/be/24f8502db11d405232ac1162eb98069ca49c3306c1d75c6ccc61d9af8789/librt-0.8.1-cp313-cp313-win32.whl", hash = "sha256:086a32dbb71336627e78cc1d6ee305a68d038ef7d4c39aaff41ae8c9aa46e91a", size = 54969, upload-time = "2026-02-17T16:12:09.633Z" }, - { url = "https://files.pythonhosted.org/packages/5c/73/c9fdf6cb2a529c1a092ce769a12d88c8cca991194dfe641b6af12fa964d2/librt-0.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:e11769a1dbda4da7b00a76cfffa67aa47cfa66921d2724539eee4b9ede780b79", size = 62000, upload-time = "2026-02-17T16:12:10.632Z" }, - { url = "https://files.pythonhosted.org/packages/d3/97/68f80ca3ac4924f250cdfa6e20142a803e5e50fca96ef5148c52ee8c10ea/librt-0.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:924817ab3141aca17893386ee13261f1d100d1ef410d70afe4389f2359fea4f0", size = 52495, upload-time = "2026-02-17T16:12:11.633Z" }, - { url = "https://files.pythonhosted.org/packages/c9/6a/907ef6800f7bca71b525a05f1839b21f708c09043b1c6aa77b6b827b3996/librt-0.8.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6cfa7fe54fd4d1f47130017351a959fe5804bda7a0bc7e07a2cdbc3fdd28d34f", size = 66081, upload-time = "2026-02-17T16:12:12.766Z" }, - { url = "https://files.pythonhosted.org/packages/1b/18/25e991cd5640c9fb0f8d91b18797b29066b792f17bf8493da183bf5caabe/librt-0.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:228c2409c079f8c11fb2e5d7b277077f694cb93443eb760e00b3b83cb8b3176c", size = 68309, upload-time = "2026-02-17T16:12:13.756Z" }, - { url = "https://files.pythonhosted.org/packages/a4/36/46820d03f058cfb5a9de5940640ba03165ed8aded69e0733c417bb04df34/librt-0.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7aae78ab5e3206181780e56912d1b9bb9f90a7249ce12f0e8bf531d0462dd0fc", size = 196804, upload-time = "2026-02-17T16:12:14.818Z" }, - { url = "https://files.pythonhosted.org/packages/59/18/5dd0d3b87b8ff9c061849fbdb347758d1f724b9a82241aa908e0ec54ccd0/librt-0.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:172d57ec04346b047ca6af181e1ea4858086c80bdf455f61994c4aa6fc3f866c", size = 206907, upload-time = "2026-02-17T16:12:16.513Z" }, - { url = "https://files.pythonhosted.org/packages/d1/96/ef04902aad1424fd7299b62d1890e803e6ab4018c3044dca5922319c4b97/librt-0.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b1977c4ea97ce5eb7755a78fae68d87e4102e4aaf54985e8b56806849cc06a3", size = 221217, upload-time = "2026-02-17T16:12:17.906Z" }, - { url = "https://files.pythonhosted.org/packages/6d/ff/7e01f2dda84a8f5d280637a2e5827210a8acca9a567a54507ef1c75b342d/librt-0.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10c42e1f6fd06733ef65ae7bebce2872bcafd8d6e6b0a08fe0a05a23b044fb14", size = 214622, upload-time = "2026-02-17T16:12:19.108Z" }, - { url = "https://files.pythonhosted.org/packages/1e/8c/5b093d08a13946034fed57619742f790faf77058558b14ca36a6e331161e/librt-0.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4c8dfa264b9193c4ee19113c985c95f876fae5e51f731494fc4e0cf594990ba7", size = 221987, upload-time = "2026-02-17T16:12:20.331Z" }, - { url = "https://files.pythonhosted.org/packages/d3/cc/86b0b3b151d40920ad45a94ce0171dec1aebba8a9d72bb3fa00c73ab25dd/librt-0.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:01170b6729a438f0dedc4a26ed342e3dc4f02d1000b4b19f980e1877f0c297e6", size = 215132, upload-time = "2026-02-17T16:12:21.54Z" }, - { url = "https://files.pythonhosted.org/packages/fc/be/8588164a46edf1e69858d952654e216a9a91174688eeefb9efbb38a9c799/librt-0.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7b02679a0d783bdae30d443025b94465d8c3dc512f32f5b5031f93f57ac32071", size = 215195, upload-time = "2026-02-17T16:12:23.073Z" }, - { url = "https://files.pythonhosted.org/packages/f5/f2/0b9279bea735c734d69344ecfe056c1ba211694a72df10f568745c899c76/librt-0.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:190b109bb69592a3401fe1ffdea41a2e73370ace2ffdc4a0e8e2b39cdea81b78", size = 237946, upload-time = "2026-02-17T16:12:24.275Z" }, - { url = "https://files.pythonhosted.org/packages/e9/cc/5f2a34fbc8aeb35314a3641f9956fa9051a947424652fad9882be7a97949/librt-0.8.1-cp314-cp314-win32.whl", hash = "sha256:e70a57ecf89a0f64c24e37f38d3fe217a58169d2fe6ed6d70554964042474023", size = 50689, upload-time = "2026-02-17T16:12:25.766Z" }, - { url = "https://files.pythonhosted.org/packages/a0/76/cd4d010ab2147339ca2b93e959c3686e964edc6de66ddacc935c325883d7/librt-0.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:7e2f3edca35664499fbb36e4770650c4bd4a08abc1f4458eab9df4ec56389730", size = 57875, upload-time = "2026-02-17T16:12:27.465Z" }, - { url = "https://files.pythonhosted.org/packages/84/0f/2143cb3c3ca48bd3379dcd11817163ca50781927c4537345d608b5045998/librt-0.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:0d2f82168e55ddefd27c01c654ce52379c0750ddc31ee86b4b266bcf4d65f2a3", size = 48058, upload-time = "2026-02-17T16:12:28.556Z" }, - { url = "https://files.pythonhosted.org/packages/d2/0e/9b23a87e37baf00311c3efe6b48d6b6c168c29902dfc3f04c338372fd7db/librt-0.8.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2c74a2da57a094bd48d03fa5d196da83d2815678385d2978657499063709abe1", size = 68313, upload-time = "2026-02-17T16:12:29.659Z" }, - { url = "https://files.pythonhosted.org/packages/db/9a/859c41e5a4f1c84200a7d2b92f586aa27133c8243b6cac9926f6e54d01b9/librt-0.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a355d99c4c0d8e5b770313b8b247411ed40949ca44e33e46a4789b9293a907ee", size = 70994, upload-time = "2026-02-17T16:12:31.516Z" }, - { url = "https://files.pythonhosted.org/packages/4c/28/10605366ee599ed34223ac2bf66404c6fb59399f47108215d16d5ad751a8/librt-0.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2eb345e8b33fb748227409c9f1233d4df354d6e54091f0e8fc53acdb2ffedeb7", size = 220770, upload-time = "2026-02-17T16:12:33.294Z" }, - { url = "https://files.pythonhosted.org/packages/af/8d/16ed8fd452dafae9c48d17a6bc1ee3e818fd40ef718d149a8eff2c9f4ea2/librt-0.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9be2f15e53ce4e83cc08adc29b26fb5978db62ef2a366fbdf716c8a6c8901040", size = 235409, upload-time = "2026-02-17T16:12:35.443Z" }, - { url = "https://files.pythonhosted.org/packages/89/1b/7bdf3e49349c134b25db816e4a3db6b94a47ac69d7d46b1e682c2c4949be/librt-0.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:785ae29c1f5c6e7c2cde2c7c0e148147f4503da3abc5d44d482068da5322fd9e", size = 246473, upload-time = "2026-02-17T16:12:36.656Z" }, - { url = "https://files.pythonhosted.org/packages/4e/8a/91fab8e4fd2a24930a17188c7af5380eb27b203d72101c9cc000dbdfd95a/librt-0.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d3a7da44baf692f0c6aeb5b2a09c5e6fc7a703bca9ffa337ddd2e2da53f7732", size = 238866, upload-time = "2026-02-17T16:12:37.849Z" }, - { url = "https://files.pythonhosted.org/packages/b9/e0/c45a098843fc7c07e18a7f8a24ca8496aecbf7bdcd54980c6ca1aaa79a8e/librt-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5fc48998000cbc39ec0d5311312dda93ecf92b39aaf184c5e817d5d440b29624", size = 250248, upload-time = "2026-02-17T16:12:39.445Z" }, - { url = "https://files.pythonhosted.org/packages/82/30/07627de23036640c952cce0c1fe78972e77d7d2f8fd54fa5ef4554ff4a56/librt-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e96baa6820280077a78244b2e06e416480ed859bbd8e5d641cf5742919d8beb4", size = 240629, upload-time = "2026-02-17T16:12:40.889Z" }, - { url = "https://files.pythonhosted.org/packages/fb/c1/55bfe1ee3542eba055616f9098eaf6eddb966efb0ca0f44eaa4aba327307/librt-0.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31362dbfe297b23590530007062c32c6f6176f6099646bb2c95ab1b00a57c382", size = 239615, upload-time = "2026-02-17T16:12:42.446Z" }, - { url = "https://files.pythonhosted.org/packages/2b/39/191d3d28abc26c9099b19852e6c99f7f6d400b82fa5a4e80291bd3803e19/librt-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc3656283d11540ab0ea01978378e73e10002145117055e03722417aeab30994", size = 263001, upload-time = "2026-02-17T16:12:43.627Z" }, - { url = "https://files.pythonhosted.org/packages/b9/eb/7697f60fbe7042ab4e88f4ee6af496b7f222fffb0a4e3593ef1f29f81652/librt-0.8.1-cp314-cp314t-win32.whl", hash = "sha256:738f08021b3142c2918c03692608baed43bc51144c29e35807682f8070ee2a3a", size = 51328, upload-time = "2026-02-17T16:12:45.148Z" }, - { url = "https://files.pythonhosted.org/packages/7c/72/34bf2eb7a15414a23e5e70ecb9440c1d3179f393d9349338a91e2781c0fb/librt-0.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:89815a22daf9c51884fb5dbe4f1ef65ee6a146e0b6a8df05f753e2e4a9359bf4", size = 58722, upload-time = "2026-02-17T16:12:46.85Z" }, - { url = "https://files.pythonhosted.org/packages/b2/c8/d148e041732d631fc76036f8b30fae4e77b027a1e95b7a84bb522481a940/librt-0.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:bf512a71a23504ed08103a13c941f763db13fb11177beb3d9244c98c29fb4a61", size = 48755, upload-time = "2026-02-17T16:12:47.943Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/56/9c/b4b0c54d84da4a94b37bd44151e46d5e583c9534c7e02250b961b1b6d8a8/librt-0.8.1.tar.gz", hash = "sha256:be46a14693955b3bd96014ccbdb8339ee8c9346fbe11c1b78901b55125f14c73", upload-time = "2026-02-17T16:13:06.101Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/21/d39b0a87ac52fc98f621fb6f8060efb017a767ebbbac2f99fbcbc9ddc0d7/librt-0.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a28f2612ab566b17f3698b0da021ff9960610301607c9a5e8eaca62f5e1c350a", upload-time = "2026-02-17T16:11:41.604Z" }, + { url = "https://files.pythonhosted.org/packages/69/f1/46375e71441c43e8ae335905e069f1c54febee63a146278bcee8782c84fd/librt-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:60a78b694c9aee2a0f1aaeaa7d101cf713e92e8423a941d2897f4fa37908dab9", upload-time = "2026-02-17T16:11:43.268Z" }, + { url = "https://files.pythonhosted.org/packages/0a/33/c510de7f93bf1fa19e13423a606d8189a02624a800710f6e6a0a0f0784b3/librt-0.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:758509ea3f1eba2a57558e7e98f4659d0ea7670bff49673b0dde18a3c7e6c0eb", upload-time = "2026-02-17T16:11:44.28Z" }, + { url = "https://files.pythonhosted.org/packages/dd/36/e725903416409a533d92398e88ce665476f275081d0d7d42f9c4951999e5/librt-0.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:039b9f2c506bd0ab0f8725aa5ba339c6f0cd19d3b514b50d134789809c24285d", upload-time = "2026-02-17T16:11:45.462Z" }, + { url = "https://files.pythonhosted.org/packages/30/7a/8d908a152e1875c9f8eac96c97a480df425e657cdb47854b9efaa4998889/librt-0.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bb54f1205a3a6ab41a6fd71dfcdcbd278670d3a90ca502a30d9da583105b6f7", upload-time = "2026-02-17T16:11:46.542Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b8/a22c34f2c485b8903a06f3fe3315341fe6876ef3599792344669db98fcff/librt-0.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:05bd41cdee35b0c59c259f870f6da532a2c5ca57db95b5f23689fcb5c9e42440", upload-time = "2026-02-17T16:11:47.746Z" }, + { url = "https://files.pythonhosted.org/packages/79/6f/5c6fea00357e4f82ba44f81dbfb027921f1ab10e320d4a64e1c408d035d9/librt-0.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adfab487facf03f0d0857b8710cf82d0704a309d8ffc33b03d9302b4c64e91a9", upload-time = "2026-02-17T16:11:49.298Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a0/95ced4e7b1267fe1e2720a111685bcddf0e781f7e9e0ce59d751c44dcfe5/librt-0.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:153188fe98a72f206042be10a2c6026139852805215ed9539186312d50a8e972", upload-time = "2026-02-17T16:11:50.49Z" }, + { url = "https://files.pythonhosted.org/packages/93/c2/0517281cb4d4101c27ab59472924e67f55e375bc46bedae94ac6dc6e1902/librt-0.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:dd3c41254ee98604b08bd5b3af5bf0a89740d4ee0711de95b65166bf44091921", upload-time = "2026-02-17T16:11:51.783Z" }, + { url = "https://files.pythonhosted.org/packages/43/e8/37b3ac108e8976888e559a7b227d0ceac03c384cfd3e7a1c2ee248dbae79/librt-0.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e0d138c7ae532908cbb342162b2611dbd4d90c941cd25ab82084aaf71d2c0bd0", upload-time = "2026-02-17T16:11:53.561Z" }, + { url = "https://files.pythonhosted.org/packages/4b/5b/35812d041c53967fedf551a39399271bbe4257e681236a2cf1a69c8e7fa1/librt-0.8.1-cp312-cp312-win32.whl", hash = "sha256:43353b943613c5d9c49a25aaffdba46f888ec354e71e3529a00cca3f04d66a7a", upload-time = "2026-02-17T16:11:54.758Z" }, + { url = "https://files.pythonhosted.org/packages/de/d1/fa5d5331b862b9775aaf2a100f5ef86854e5d4407f71bddf102f4421e034/librt-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:ff8baf1f8d3f4b6b7257fcb75a501f2a5499d0dda57645baa09d4d0d34b19444", upload-time = "2026-02-17T16:11:55.748Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7c/c614252f9acda59b01a66e2ddfd243ed1c7e1deab0293332dfbccf862808/librt-0.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:0f2ae3725904f7377e11cc37722d5d401e8b3d5851fb9273d7f4fe04f6b3d37d", upload-time = "2026-02-17T16:11:56.801Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3c/f614c8e4eaac7cbf2bbdf9528790b21d89e277ee20d57dc6e559c626105f/librt-0.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e6bad1cd94f6764e1e21950542f818a09316645337fd5ab9a7acc45d99a8f35", upload-time = "2026-02-17T16:11:57.809Z" }, + { url = "https://files.pythonhosted.org/packages/ab/96/5836544a45100ae411eda07d29e3d99448e5258b6e9c8059deb92945f5c2/librt-0.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cf450f498c30af55551ba4f66b9123b7185362ec8b625a773b3d39aa1a717583", upload-time = "2026-02-17T16:11:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/06/53/f0b992b57af6d5531bf4677d75c44f095f2366a1741fb695ee462ae04b05/librt-0.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:eca45e982fa074090057132e30585a7e8674e9e885d402eae85633e9f449ce6c", upload-time = "2026-02-17T16:11:59.862Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ad/4848cc16e268d14280d8168aee4f31cea92bbd2b79ce33d3e166f2b4e4fc/librt-0.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c3811485fccfda840861905b8c70bba5ec094e02825598bb9d4ca3936857a04", upload-time = "2026-02-17T16:12:00.954Z" }, + { url = "https://files.pythonhosted.org/packages/52/05/27fdc2e95de26273d83b96742d8d3b7345f2ea2bdbd2405cc504644f2096/librt-0.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e4af413908f77294605e28cfd98063f54b2c790561383971d2f52d113d9c363", upload-time = "2026-02-17T16:12:02.108Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d0/78200a45ba3240cb042bc597d6f2accba9193a2c57d0356268cbbe2d0925/librt-0.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5212a5bd7fae98dae95710032902edcd2ec4dc994e883294f75c857b83f9aba0", upload-time = "2026-02-17T16:12:03.631Z" }, + { url = "https://files.pythonhosted.org/packages/af/72/a210839fa74c90474897124c064ffca07f8d4b347b6574d309686aae7ca6/librt-0.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e692aa2d1d604e6ca12d35e51fdc36f4cda6345e28e36374579f7ef3611b3012", upload-time = "2026-02-17T16:12:04.725Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c1/a03cc63722339ddbf087485f253493e2b013039f5b707e8e6016141130fa/librt-0.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4be2a5c926b9770c9e08e717f05737a269b9d0ebc5d2f0060f0fe3fe9ce47acb", upload-time = "2026-02-17T16:12:05.828Z" }, + { url = "https://files.pythonhosted.org/packages/58/f5/fff6108af0acf941c6f274a946aea0e484bd10cd2dc37610287ce49388c5/librt-0.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fd1a720332ea335ceb544cf0a03f81df92abd4bb887679fd1e460976b0e6214b", upload-time = "2026-02-17T16:12:07.09Z" }, + { url = "https://files.pythonhosted.org/packages/71/67/5a387bfef30ec1e4b4f30562c8586566faf87e47d696768c19feb49e3646/librt-0.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2af9e01e0ef80d95ae3c720be101227edae5f2fe7e3dc63d8857fadfc5a1d", upload-time = "2026-02-17T16:12:08.43Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/24f8502db11d405232ac1162eb98069ca49c3306c1d75c6ccc61d9af8789/librt-0.8.1-cp313-cp313-win32.whl", hash = "sha256:086a32dbb71336627e78cc1d6ee305a68d038ef7d4c39aaff41ae8c9aa46e91a", upload-time = "2026-02-17T16:12:09.633Z" }, + { url = "https://files.pythonhosted.org/packages/5c/73/c9fdf6cb2a529c1a092ce769a12d88c8cca991194dfe641b6af12fa964d2/librt-0.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:e11769a1dbda4da7b00a76cfffa67aa47cfa66921d2724539eee4b9ede780b79", upload-time = "2026-02-17T16:12:10.632Z" }, + { url = "https://files.pythonhosted.org/packages/d3/97/68f80ca3ac4924f250cdfa6e20142a803e5e50fca96ef5148c52ee8c10ea/librt-0.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:924817ab3141aca17893386ee13261f1d100d1ef410d70afe4389f2359fea4f0", upload-time = "2026-02-17T16:12:11.633Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6a/907ef6800f7bca71b525a05f1839b21f708c09043b1c6aa77b6b827b3996/librt-0.8.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6cfa7fe54fd4d1f47130017351a959fe5804bda7a0bc7e07a2cdbc3fdd28d34f", upload-time = "2026-02-17T16:12:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/1b/18/25e991cd5640c9fb0f8d91b18797b29066b792f17bf8493da183bf5caabe/librt-0.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:228c2409c079f8c11fb2e5d7b277077f694cb93443eb760e00b3b83cb8b3176c", upload-time = "2026-02-17T16:12:13.756Z" }, + { url = "https://files.pythonhosted.org/packages/a4/36/46820d03f058cfb5a9de5940640ba03165ed8aded69e0733c417bb04df34/librt-0.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7aae78ab5e3206181780e56912d1b9bb9f90a7249ce12f0e8bf531d0462dd0fc", upload-time = "2026-02-17T16:12:14.818Z" }, + { url = "https://files.pythonhosted.org/packages/59/18/5dd0d3b87b8ff9c061849fbdb347758d1f724b9a82241aa908e0ec54ccd0/librt-0.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:172d57ec04346b047ca6af181e1ea4858086c80bdf455f61994c4aa6fc3f866c", upload-time = "2026-02-17T16:12:16.513Z" }, + { url = "https://files.pythonhosted.org/packages/d1/96/ef04902aad1424fd7299b62d1890e803e6ab4018c3044dca5922319c4b97/librt-0.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b1977c4ea97ce5eb7755a78fae68d87e4102e4aaf54985e8b56806849cc06a3", upload-time = "2026-02-17T16:12:17.906Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ff/7e01f2dda84a8f5d280637a2e5827210a8acca9a567a54507ef1c75b342d/librt-0.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10c42e1f6fd06733ef65ae7bebce2872bcafd8d6e6b0a08fe0a05a23b044fb14", upload-time = "2026-02-17T16:12:19.108Z" }, + { url = "https://files.pythonhosted.org/packages/1e/8c/5b093d08a13946034fed57619742f790faf77058558b14ca36a6e331161e/librt-0.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4c8dfa264b9193c4ee19113c985c95f876fae5e51f731494fc4e0cf594990ba7", upload-time = "2026-02-17T16:12:20.331Z" }, + { url = "https://files.pythonhosted.org/packages/d3/cc/86b0b3b151d40920ad45a94ce0171dec1aebba8a9d72bb3fa00c73ab25dd/librt-0.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:01170b6729a438f0dedc4a26ed342e3dc4f02d1000b4b19f980e1877f0c297e6", upload-time = "2026-02-17T16:12:21.54Z" }, + { url = "https://files.pythonhosted.org/packages/fc/be/8588164a46edf1e69858d952654e216a9a91174688eeefb9efbb38a9c799/librt-0.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7b02679a0d783bdae30d443025b94465d8c3dc512f32f5b5031f93f57ac32071", upload-time = "2026-02-17T16:12:23.073Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f2/0b9279bea735c734d69344ecfe056c1ba211694a72df10f568745c899c76/librt-0.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:190b109bb69592a3401fe1ffdea41a2e73370ace2ffdc4a0e8e2b39cdea81b78", upload-time = "2026-02-17T16:12:24.275Z" }, + { url = "https://files.pythonhosted.org/packages/e9/cc/5f2a34fbc8aeb35314a3641f9956fa9051a947424652fad9882be7a97949/librt-0.8.1-cp314-cp314-win32.whl", hash = "sha256:e70a57ecf89a0f64c24e37f38d3fe217a58169d2fe6ed6d70554964042474023", upload-time = "2026-02-17T16:12:25.766Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/cd4d010ab2147339ca2b93e959c3686e964edc6de66ddacc935c325883d7/librt-0.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:7e2f3edca35664499fbb36e4770650c4bd4a08abc1f4458eab9df4ec56389730", upload-time = "2026-02-17T16:12:27.465Z" }, + { url = "https://files.pythonhosted.org/packages/84/0f/2143cb3c3ca48bd3379dcd11817163ca50781927c4537345d608b5045998/librt-0.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:0d2f82168e55ddefd27c01c654ce52379c0750ddc31ee86b4b266bcf4d65f2a3", upload-time = "2026-02-17T16:12:28.556Z" }, + { url = "https://files.pythonhosted.org/packages/d2/0e/9b23a87e37baf00311c3efe6b48d6b6c168c29902dfc3f04c338372fd7db/librt-0.8.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2c74a2da57a094bd48d03fa5d196da83d2815678385d2978657499063709abe1", upload-time = "2026-02-17T16:12:29.659Z" }, + { url = "https://files.pythonhosted.org/packages/db/9a/859c41e5a4f1c84200a7d2b92f586aa27133c8243b6cac9926f6e54d01b9/librt-0.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a355d99c4c0d8e5b770313b8b247411ed40949ca44e33e46a4789b9293a907ee", upload-time = "2026-02-17T16:12:31.516Z" }, + { url = "https://files.pythonhosted.org/packages/4c/28/10605366ee599ed34223ac2bf66404c6fb59399f47108215d16d5ad751a8/librt-0.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2eb345e8b33fb748227409c9f1233d4df354d6e54091f0e8fc53acdb2ffedeb7", upload-time = "2026-02-17T16:12:33.294Z" }, + { url = "https://files.pythonhosted.org/packages/af/8d/16ed8fd452dafae9c48d17a6bc1ee3e818fd40ef718d149a8eff2c9f4ea2/librt-0.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9be2f15e53ce4e83cc08adc29b26fb5978db62ef2a366fbdf716c8a6c8901040", upload-time = "2026-02-17T16:12:35.443Z" }, + { url = "https://files.pythonhosted.org/packages/89/1b/7bdf3e49349c134b25db816e4a3db6b94a47ac69d7d46b1e682c2c4949be/librt-0.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:785ae29c1f5c6e7c2cde2c7c0e148147f4503da3abc5d44d482068da5322fd9e", upload-time = "2026-02-17T16:12:36.656Z" }, + { url = "https://files.pythonhosted.org/packages/4e/8a/91fab8e4fd2a24930a17188c7af5380eb27b203d72101c9cc000dbdfd95a/librt-0.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d3a7da44baf692f0c6aeb5b2a09c5e6fc7a703bca9ffa337ddd2e2da53f7732", upload-time = "2026-02-17T16:12:37.849Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e0/c45a098843fc7c07e18a7f8a24ca8496aecbf7bdcd54980c6ca1aaa79a8e/librt-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5fc48998000cbc39ec0d5311312dda93ecf92b39aaf184c5e817d5d440b29624", upload-time = "2026-02-17T16:12:39.445Z" }, + { url = "https://files.pythonhosted.org/packages/82/30/07627de23036640c952cce0c1fe78972e77d7d2f8fd54fa5ef4554ff4a56/librt-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e96baa6820280077a78244b2e06e416480ed859bbd8e5d641cf5742919d8beb4", upload-time = "2026-02-17T16:12:40.889Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c1/55bfe1ee3542eba055616f9098eaf6eddb966efb0ca0f44eaa4aba327307/librt-0.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31362dbfe297b23590530007062c32c6f6176f6099646bb2c95ab1b00a57c382", upload-time = "2026-02-17T16:12:42.446Z" }, + { url = "https://files.pythonhosted.org/packages/2b/39/191d3d28abc26c9099b19852e6c99f7f6d400b82fa5a4e80291bd3803e19/librt-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc3656283d11540ab0ea01978378e73e10002145117055e03722417aeab30994", upload-time = "2026-02-17T16:12:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/b9/eb/7697f60fbe7042ab4e88f4ee6af496b7f222fffb0a4e3593ef1f29f81652/librt-0.8.1-cp314-cp314t-win32.whl", hash = "sha256:738f08021b3142c2918c03692608baed43bc51144c29e35807682f8070ee2a3a", upload-time = "2026-02-17T16:12:45.148Z" }, + { url = "https://files.pythonhosted.org/packages/7c/72/34bf2eb7a15414a23e5e70ecb9440c1d3179f393d9349338a91e2781c0fb/librt-0.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:89815a22daf9c51884fb5dbe4f1ef65ee6a146e0b6a8df05f753e2e4a9359bf4", upload-time = "2026-02-17T16:12:46.85Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c8/d148e041732d631fc76036f8b30fae4e77b027a1e95b7a84bb522481a940/librt-0.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:bf512a71a23504ed08103a13c941f763db13fb11177beb3d9244c98c29fb4a61", upload-time = "2026-02-17T16:12:47.943Z" }, +] + +[[package]] +name = "mcp" +version = "1.27.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "jsonschema" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/3c/347cf965d313f5d41764e7d46bea6ffe7d9ef13b983cc429b0340962a082/mcp-1.27.2.tar.gz", hash = "sha256:8e02db104096d1c25b28e64bde29a5c32b31bc241710213e12fd4d84985bdfef", upload-time = "2026-05-29T17:16:04.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/11/252c6f971dc4f16af1d98a1c469d8ba523aab00d1bb76b4d3bc1ff32eacc/mcp-1.27.2-py3-none-any.whl", hash = "sha256:d6ff5160c6ca65d93013626efb3fc249de683c30b2d8570755ceddd490344de5", upload-time = "2026-05-29T17:16:02.442Z" }, ] [[package]] @@ -411,136 +602,97 @@ dependencies = [ { name = "pathspec" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f8/5c/b0089fe7fef0a994ae5ee07029ced0526082c6cfaaa4c10d40a10e33b097/mypy-1.20.0.tar.gz", hash = "sha256:eb96c84efcc33f0b5e0e04beacf00129dd963b67226b01c00b9dfc8affb464c3", size = 3815028, upload-time = "2026-03-31T16:55:14.959Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/dd/3afa29b58c2e57c79116ed55d700721c3c3b15955e2b6251dd165d377c0e/mypy-1.20.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:002b613ae19f4ac7d18b7e168ffe1cb9013b37c57f7411984abbd3b817b0a214", size = 14509525, upload-time = "2026-03-31T16:55:01.824Z" }, - { url = "https://files.pythonhosted.org/packages/54/eb/227b516ab8cad9f2a13c5e7a98d28cd6aa75e9c83e82776ae6c1c4c046c7/mypy-1.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9336b5e6712f4adaf5afc3203a99a40b379049104349d747eb3e5a3aa23ac2e", size = 13326469, upload-time = "2026-03-31T16:51:41.23Z" }, - { url = "https://files.pythonhosted.org/packages/57/d4/1ddb799860c1b5ac6117ec307b965f65deeb47044395ff01ab793248a591/mypy-1.20.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f13b3e41bce9d257eded794c0f12878af3129d80aacd8a3ee0dee51f3a978651", size = 13705953, upload-time = "2026-03-31T16:48:55.69Z" }, - { url = "https://files.pythonhosted.org/packages/c5/b7/54a720f565a87b893182a2a393370289ae7149e4715859e10e1c05e49154/mypy-1.20.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9804c3ad27f78e54e58b32e7cb532d128b43dbfb9f3f9f06262b821a0f6bd3f5", size = 14710363, upload-time = "2026-03-31T16:53:26.948Z" }, - { url = "https://files.pythonhosted.org/packages/b2/2a/74810274848d061f8a8ea4ac23aaad43bd3d8c1882457999c2e568341c57/mypy-1.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:697f102c5c1d526bdd761a69f17c6070f9892eebcb94b1a5963d679288c09e78", size = 14947005, upload-time = "2026-03-31T16:50:17.591Z" }, - { url = "https://files.pythonhosted.org/packages/77/91/21b8ba75f958bcda75690951ce6fa6b7138b03471618959529d74b8544e2/mypy-1.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ecd63f75fdd30327e4ad8b5704bd6d91fc6c1b2e029f8ee14705e1207212489", size = 10880616, upload-time = "2026-03-31T16:52:19.986Z" }, - { url = "https://files.pythonhosted.org/packages/8a/15/3d8198ef97c1ca03aea010cce4f1d4f3bc5d9849e8c0140111ca2ead9fdd/mypy-1.20.0-cp312-cp312-win_arm64.whl", hash = "sha256:f194db59657c58593a3c47c6dfd7bad4ef4ac12dbc94d01b3a95521f78177e33", size = 9813091, upload-time = "2026-03-31T16:53:44.385Z" }, - { url = "https://files.pythonhosted.org/packages/d6/a7/f64ea7bd592fa431cb597418b6dec4a47f7d0c36325fec7ac67bc8402b94/mypy-1.20.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b20c8b0fd5877abdf402e79a3af987053de07e6fb208c18df6659f708b535134", size = 14485344, upload-time = "2026-03-31T16:49:16.78Z" }, - { url = "https://files.pythonhosted.org/packages/bb/72/8927d84cfc90c6abea6e96663576e2e417589347eb538749a464c4c218a0/mypy-1.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:367e5c993ba34d5054d11937d0485ad6dfc60ba760fa326c01090fc256adf15c", size = 13327400, upload-time = "2026-03-31T16:53:08.02Z" }, - { url = "https://files.pythonhosted.org/packages/ab/4a/11ab99f9afa41aa350178d24a7d2da17043228ea10f6456523f64b5a6cf6/mypy-1.20.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f799d9db89fc00446f03281f84a221e50018fc40113a3ba9864b132895619ebe", size = 13706384, upload-time = "2026-03-31T16:52:28.577Z" }, - { url = "https://files.pythonhosted.org/packages/42/79/694ca73979cfb3535ebfe78733844cd5aff2e63304f59bf90585110d975a/mypy-1.20.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555658c611099455b2da507582ea20d2043dfdfe7f5ad0add472b1c6238b433f", size = 14700378, upload-time = "2026-03-31T16:48:45.527Z" }, - { url = "https://files.pythonhosted.org/packages/84/24/a022ccab3a46e3d2cdf2e0e260648633640eb396c7e75d5a42818a8d3971/mypy-1.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:efe8d70949c3023698c3fca1e94527e7e790a361ab8116f90d11221421cd8726", size = 14932170, upload-time = "2026-03-31T16:49:36.038Z" }, - { url = "https://files.pythonhosted.org/packages/d8/9b/549228d88f574d04117e736f55958bd4908f980f9f5700a07aeb85df005b/mypy-1.20.0-cp313-cp313-win_amd64.whl", hash = "sha256:f49590891d2c2f8a9de15614e32e459a794bcba84693c2394291a2038bbaaa69", size = 10888526, upload-time = "2026-03-31T16:50:59.827Z" }, - { url = "https://files.pythonhosted.org/packages/91/17/15095c0e54a8bc04d22d4ff06b2139d5f142c2e87520b4e39010c4862771/mypy-1.20.0-cp313-cp313-win_arm64.whl", hash = "sha256:76a70bf840495729be47510856b978f1b0ec7d08f257ca38c9d932720bf6b43e", size = 9816456, upload-time = "2026-03-31T16:49:59.537Z" }, - { url = "https://files.pythonhosted.org/packages/4e/0e/6ca4a84cbed9e62384bc0b2974c90395ece5ed672393e553996501625fc5/mypy-1.20.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0f42dfaab7ec1baff3b383ad7af562ab0de573c5f6edb44b2dab016082b89948", size = 14483331, upload-time = "2026-03-31T16:52:57.999Z" }, - { url = "https://files.pythonhosted.org/packages/7d/c5/5fe9d8a729dd9605064691816243ae6c49fde0bd28f6e5e17f6a24203c43/mypy-1.20.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:31b5dbb55293c1bd27c0fc813a0d2bb5ceef9d65ac5afa2e58f829dab7921fd5", size = 13342047, upload-time = "2026-03-31T16:54:21.555Z" }, - { url = "https://files.pythonhosted.org/packages/4c/33/e18bcfa338ca4e6b2771c85d4c5203e627d0c69d9de5c1a2cf2ba13320ba/mypy-1.20.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49d11c6f573a5a08f77fad13faff2139f6d0730ebed2cfa9b3d2702671dd7188", size = 13719585, upload-time = "2026-03-31T16:51:53.89Z" }, - { url = "https://files.pythonhosted.org/packages/6b/8d/93491ff7b79419edc7eabf95cb3b3f7490e2e574b2855c7c7e7394ff933f/mypy-1.20.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d3243c406773185144527f83be0e0aefc7bf4601b0b2b956665608bf7c98a83", size = 14685075, upload-time = "2026-03-31T16:54:04.464Z" }, - { url = "https://files.pythonhosted.org/packages/b5/9d/d924b38a4923f8d164bf2b4ec98bf13beaf6e10a5348b4b137eadae40a6e/mypy-1.20.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a79c1eba7ac4209f2d850f0edd0a2f8bba88cbfdfefe6fb76a19e9d4fe5e71a2", size = 14919141, upload-time = "2026-03-31T16:54:51.785Z" }, - { url = "https://files.pythonhosted.org/packages/59/98/1da9977016678c0b99d43afe52ed00bb3c1a0c4c995d3e6acca1a6ebb9b4/mypy-1.20.0-cp314-cp314-win_amd64.whl", hash = "sha256:00e047c74d3ec6e71a2eb88e9ea551a2edb90c21f993aefa9e0d2a898e0bb732", size = 11050925, upload-time = "2026-03-31T16:51:30.758Z" }, - { url = "https://files.pythonhosted.org/packages/5e/e3/ba0b7a3143e49a9c4f5967dde6ea4bf8e0b10ecbbcca69af84027160ee89/mypy-1.20.0-cp314-cp314-win_arm64.whl", hash = "sha256:931a7630bba591593dcf6e97224a21ff80fb357e7982628d25e3c618e7f598ef", size = 10001089, upload-time = "2026-03-31T16:49:43.632Z" }, - { url = "https://files.pythonhosted.org/packages/12/28/e617e67b3be9d213cda7277913269c874eb26472489f95d09d89765ce2d8/mypy-1.20.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:26c8b52627b6552f47ff11adb4e1509605f094e29815323e487fc0053ebe93d1", size = 15534710, upload-time = "2026-03-31T16:52:12.506Z" }, - { url = "https://files.pythonhosted.org/packages/6e/0c/3b5f2d3e45dc7169b811adce8451679d9430399d03b168f9b0489f43adaa/mypy-1.20.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:39362cdb4ba5f916e7976fccecaab1ba3a83e35f60fa68b64e9a70e221bb2436", size = 14393013, upload-time = "2026-03-31T16:54:41.186Z" }, - { url = "https://files.pythonhosted.org/packages/a3/49/edc8b0aa145cc09c1c74f7ce2858eead9329931dcbbb26e2ad40906daa4e/mypy-1.20.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34506397dbf40c15dc567635d18a21d33827e9ab29014fb83d292a8f4f8953b6", size = 15047240, upload-time = "2026-03-31T16:54:31.955Z" }, - { url = "https://files.pythonhosted.org/packages/42/37/a946bb416e37a57fa752b3100fd5ede0e28df94f92366d1716555d47c454/mypy-1.20.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555493c44a4f5a1b58d611a43333e71a9981c6dbe26270377b6f8174126a0526", size = 15858565, upload-time = "2026-03-31T16:53:36.997Z" }, - { url = "https://files.pythonhosted.org/packages/2f/99/7690b5b5b552db1bd4ff362e4c0eb3107b98d680835e65823fbe888c8b78/mypy-1.20.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2721f0ce49cb74a38f00c50da67cb7d36317b5eda38877a49614dc018e91c787", size = 16087874, upload-time = "2026-03-31T16:52:48.313Z" }, - { url = "https://files.pythonhosted.org/packages/aa/76/53e893a498138066acd28192b77495c9357e5a58cc4be753182846b43315/mypy-1.20.0-cp314-cp314t-win_amd64.whl", hash = "sha256:47781555a7aa5fedcc2d16bcd72e0dc83eb272c10dd657f9fb3f9cc08e2e6abb", size = 12572380, upload-time = "2026-03-31T16:49:52.454Z" }, - { url = "https://files.pythonhosted.org/packages/76/9c/6dbdae21f01b7aacddc2c0bbf3c5557aa547827fdf271770fe1e521e7093/mypy-1.20.0-cp314-cp314t-win_arm64.whl", hash = "sha256:c70380fe5d64010f79fb863b9081c7004dd65225d2277333c219d93a10dad4dd", size = 10381174, upload-time = "2026-03-31T16:51:20.179Z" }, - { url = "https://files.pythonhosted.org/packages/21/66/4d734961ce167f0fd8380769b3b7c06dbdd6ff54c2190f3f2ecd22528158/mypy-1.20.0-py3-none-any.whl", hash = "sha256:a6e0641147cbfa7e4e94efdb95c2dab1aff8cfc159ded13e07f308ddccc8c48e", size = 2636365, upload-time = "2026-03-31T16:51:44.911Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/f8/5c/b0089fe7fef0a994ae5ee07029ced0526082c6cfaaa4c10d40a10e33b097/mypy-1.20.0.tar.gz", hash = "sha256:eb96c84efcc33f0b5e0e04beacf00129dd963b67226b01c00b9dfc8affb464c3", upload-time = "2026-03-31T16:55:14.959Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/dd/3afa29b58c2e57c79116ed55d700721c3c3b15955e2b6251dd165d377c0e/mypy-1.20.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:002b613ae19f4ac7d18b7e168ffe1cb9013b37c57f7411984abbd3b817b0a214", upload-time = "2026-03-31T16:55:01.824Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/227b516ab8cad9f2a13c5e7a98d28cd6aa75e9c83e82776ae6c1c4c046c7/mypy-1.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9336b5e6712f4adaf5afc3203a99a40b379049104349d747eb3e5a3aa23ac2e", upload-time = "2026-03-31T16:51:41.23Z" }, + { url = "https://files.pythonhosted.org/packages/57/d4/1ddb799860c1b5ac6117ec307b965f65deeb47044395ff01ab793248a591/mypy-1.20.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f13b3e41bce9d257eded794c0f12878af3129d80aacd8a3ee0dee51f3a978651", upload-time = "2026-03-31T16:48:55.69Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b7/54a720f565a87b893182a2a393370289ae7149e4715859e10e1c05e49154/mypy-1.20.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9804c3ad27f78e54e58b32e7cb532d128b43dbfb9f3f9f06262b821a0f6bd3f5", upload-time = "2026-03-31T16:53:26.948Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2a/74810274848d061f8a8ea4ac23aaad43bd3d8c1882457999c2e568341c57/mypy-1.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:697f102c5c1d526bdd761a69f17c6070f9892eebcb94b1a5963d679288c09e78", upload-time = "2026-03-31T16:50:17.591Z" }, + { url = "https://files.pythonhosted.org/packages/77/91/21b8ba75f958bcda75690951ce6fa6b7138b03471618959529d74b8544e2/mypy-1.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ecd63f75fdd30327e4ad8b5704bd6d91fc6c1b2e029f8ee14705e1207212489", upload-time = "2026-03-31T16:52:19.986Z" }, + { url = "https://files.pythonhosted.org/packages/8a/15/3d8198ef97c1ca03aea010cce4f1d4f3bc5d9849e8c0140111ca2ead9fdd/mypy-1.20.0-cp312-cp312-win_arm64.whl", hash = "sha256:f194db59657c58593a3c47c6dfd7bad4ef4ac12dbc94d01b3a95521f78177e33", upload-time = "2026-03-31T16:53:44.385Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a7/f64ea7bd592fa431cb597418b6dec4a47f7d0c36325fec7ac67bc8402b94/mypy-1.20.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b20c8b0fd5877abdf402e79a3af987053de07e6fb208c18df6659f708b535134", upload-time = "2026-03-31T16:49:16.78Z" }, + { url = "https://files.pythonhosted.org/packages/bb/72/8927d84cfc90c6abea6e96663576e2e417589347eb538749a464c4c218a0/mypy-1.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:367e5c993ba34d5054d11937d0485ad6dfc60ba760fa326c01090fc256adf15c", upload-time = "2026-03-31T16:53:08.02Z" }, + { url = "https://files.pythonhosted.org/packages/ab/4a/11ab99f9afa41aa350178d24a7d2da17043228ea10f6456523f64b5a6cf6/mypy-1.20.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f799d9db89fc00446f03281f84a221e50018fc40113a3ba9864b132895619ebe", upload-time = "2026-03-31T16:52:28.577Z" }, + { url = "https://files.pythonhosted.org/packages/42/79/694ca73979cfb3535ebfe78733844cd5aff2e63304f59bf90585110d975a/mypy-1.20.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555658c611099455b2da507582ea20d2043dfdfe7f5ad0add472b1c6238b433f", upload-time = "2026-03-31T16:48:45.527Z" }, + { url = "https://files.pythonhosted.org/packages/84/24/a022ccab3a46e3d2cdf2e0e260648633640eb396c7e75d5a42818a8d3971/mypy-1.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:efe8d70949c3023698c3fca1e94527e7e790a361ab8116f90d11221421cd8726", upload-time = "2026-03-31T16:49:36.038Z" }, + { url = "https://files.pythonhosted.org/packages/d8/9b/549228d88f574d04117e736f55958bd4908f980f9f5700a07aeb85df005b/mypy-1.20.0-cp313-cp313-win_amd64.whl", hash = "sha256:f49590891d2c2f8a9de15614e32e459a794bcba84693c2394291a2038bbaaa69", upload-time = "2026-03-31T16:50:59.827Z" }, + { url = "https://files.pythonhosted.org/packages/91/17/15095c0e54a8bc04d22d4ff06b2139d5f142c2e87520b4e39010c4862771/mypy-1.20.0-cp313-cp313-win_arm64.whl", hash = "sha256:76a70bf840495729be47510856b978f1b0ec7d08f257ca38c9d932720bf6b43e", upload-time = "2026-03-31T16:49:59.537Z" }, + { url = "https://files.pythonhosted.org/packages/4e/0e/6ca4a84cbed9e62384bc0b2974c90395ece5ed672393e553996501625fc5/mypy-1.20.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0f42dfaab7ec1baff3b383ad7af562ab0de573c5f6edb44b2dab016082b89948", upload-time = "2026-03-31T16:52:57.999Z" }, + { url = "https://files.pythonhosted.org/packages/7d/c5/5fe9d8a729dd9605064691816243ae6c49fde0bd28f6e5e17f6a24203c43/mypy-1.20.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:31b5dbb55293c1bd27c0fc813a0d2bb5ceef9d65ac5afa2e58f829dab7921fd5", upload-time = "2026-03-31T16:54:21.555Z" }, + { url = "https://files.pythonhosted.org/packages/4c/33/e18bcfa338ca4e6b2771c85d4c5203e627d0c69d9de5c1a2cf2ba13320ba/mypy-1.20.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49d11c6f573a5a08f77fad13faff2139f6d0730ebed2cfa9b3d2702671dd7188", upload-time = "2026-03-31T16:51:53.89Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8d/93491ff7b79419edc7eabf95cb3b3f7490e2e574b2855c7c7e7394ff933f/mypy-1.20.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d3243c406773185144527f83be0e0aefc7bf4601b0b2b956665608bf7c98a83", upload-time = "2026-03-31T16:54:04.464Z" }, + { url = "https://files.pythonhosted.org/packages/b5/9d/d924b38a4923f8d164bf2b4ec98bf13beaf6e10a5348b4b137eadae40a6e/mypy-1.20.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a79c1eba7ac4209f2d850f0edd0a2f8bba88cbfdfefe6fb76a19e9d4fe5e71a2", upload-time = "2026-03-31T16:54:51.785Z" }, + { url = "https://files.pythonhosted.org/packages/59/98/1da9977016678c0b99d43afe52ed00bb3c1a0c4c995d3e6acca1a6ebb9b4/mypy-1.20.0-cp314-cp314-win_amd64.whl", hash = "sha256:00e047c74d3ec6e71a2eb88e9ea551a2edb90c21f993aefa9e0d2a898e0bb732", upload-time = "2026-03-31T16:51:30.758Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e3/ba0b7a3143e49a9c4f5967dde6ea4bf8e0b10ecbbcca69af84027160ee89/mypy-1.20.0-cp314-cp314-win_arm64.whl", hash = "sha256:931a7630bba591593dcf6e97224a21ff80fb357e7982628d25e3c618e7f598ef", upload-time = "2026-03-31T16:49:43.632Z" }, + { url = "https://files.pythonhosted.org/packages/12/28/e617e67b3be9d213cda7277913269c874eb26472489f95d09d89765ce2d8/mypy-1.20.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:26c8b52627b6552f47ff11adb4e1509605f094e29815323e487fc0053ebe93d1", upload-time = "2026-03-31T16:52:12.506Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0c/3b5f2d3e45dc7169b811adce8451679d9430399d03b168f9b0489f43adaa/mypy-1.20.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:39362cdb4ba5f916e7976fccecaab1ba3a83e35f60fa68b64e9a70e221bb2436", upload-time = "2026-03-31T16:54:41.186Z" }, + { url = "https://files.pythonhosted.org/packages/a3/49/edc8b0aa145cc09c1c74f7ce2858eead9329931dcbbb26e2ad40906daa4e/mypy-1.20.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34506397dbf40c15dc567635d18a21d33827e9ab29014fb83d292a8f4f8953b6", upload-time = "2026-03-31T16:54:31.955Z" }, + { url = "https://files.pythonhosted.org/packages/42/37/a946bb416e37a57fa752b3100fd5ede0e28df94f92366d1716555d47c454/mypy-1.20.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555493c44a4f5a1b58d611a43333e71a9981c6dbe26270377b6f8174126a0526", upload-time = "2026-03-31T16:53:36.997Z" }, + { url = "https://files.pythonhosted.org/packages/2f/99/7690b5b5b552db1bd4ff362e4c0eb3107b98d680835e65823fbe888c8b78/mypy-1.20.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2721f0ce49cb74a38f00c50da67cb7d36317b5eda38877a49614dc018e91c787", upload-time = "2026-03-31T16:52:48.313Z" }, + { url = "https://files.pythonhosted.org/packages/aa/76/53e893a498138066acd28192b77495c9357e5a58cc4be753182846b43315/mypy-1.20.0-cp314-cp314t-win_amd64.whl", hash = "sha256:47781555a7aa5fedcc2d16bcd72e0dc83eb272c10dd657f9fb3f9cc08e2e6abb", upload-time = "2026-03-31T16:49:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/76/9c/6dbdae21f01b7aacddc2c0bbf3c5557aa547827fdf271770fe1e521e7093/mypy-1.20.0-cp314-cp314t-win_arm64.whl", hash = "sha256:c70380fe5d64010f79fb863b9081c7004dd65225d2277333c219d93a10dad4dd", upload-time = "2026-03-31T16:51:20.179Z" }, + { url = "https://files.pythonhosted.org/packages/21/66/4d734961ce167f0fd8380769b3b7c06dbdd6ff54c2190f3f2ecd22528158/mypy-1.20.0-py3-none-any.whl", hash = "sha256:a6e0641147cbfa7e4e94efdb95c2dab1aff8cfc159ded13e07f308ddccc8c48e", upload-time = "2026-03-31T16:51:44.911Z" }, ] [[package]] name = "mypy-extensions" version = "1.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", upload-time = "2025-04-22T14:54:24.164Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, -] - -[[package]] -name = "flowx" -version = "0.2.0" -source = { editable = "." } -dependencies = [ - { name = "databricks-sdk" }, - { name = "pyyaml" }, - { name = "sqlglot" }, -] - -[package.dev-dependencies] -dev = [ - { name = "coverage" }, - { name = "mypy" }, - { name = "pytest" }, - { name = "ruff" }, - { name = "types-pyyaml" }, -] -yq = [ - { name = "yq" }, -] - -[package.metadata] -requires-dist = [ - { name = "databricks-sdk", specifier = ">=0.40" }, - { name = "pyyaml", specifier = ">=6.0" }, - { name = "sqlglot", specifier = ">=25.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "coverage", specifier = ">=7.6.1,<8" }, - { name = "mypy", specifier = ">=1.18.2,<2" }, - { name = "pytest", specifier = ">=8.3.3,<9" }, - { name = "ruff", specifier = ">=0.14.0,<1" }, - { name = "types-pyyaml", specifier = ">=6.0.12.20250915,<7" }, + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", upload-time = "2025-04-22T14:54:22.983Z" }, ] -yq = [{ name = "yq", specifier = "~=3.4.3" }] [[package]] name = "packaging" version = "26.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", upload-time = "2026-01-21T20:50:39.064Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", upload-time = "2026-01-21T20:50:37.788Z" }, ] [[package]] name = "pathspec" version = "1.0.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", upload-time = "2026-01-27T03:59:46.938Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", upload-time = "2026-01-27T03:59:45.137Z" }, ] [[package]] name = "pluggy" version = "1.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", upload-time = "2025-05-15T12:30:07.975Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", upload-time = "2025-05-15T12:30:06.134Z" }, ] [[package]] name = "protobuf" version = "6.33.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", upload-time = "2026-03-18T19:05:00.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, - { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, - { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, - { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, - { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, - { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", upload-time = "2026-03-18T19:04:59.826Z" }, ] [[package]] name = "pyasn1" version = "0.6.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", upload-time = "2026-03-17T01:06:53.382Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", upload-time = "2026-03-17T01:06:52.036Z" }, ] [[package]] @@ -550,27 +702,145 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyasn1" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", upload-time = "2025-03-28T02:41:22.17Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", upload-time = "2025-03-28T02:41:19.028Z" }, ] [[package]] name = "pycparser" version = "3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", upload-time = "2026-01-21T14:26:50.693Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", upload-time = "2026-05-06T13:39:29.883Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", upload-time = "2026-05-08T13:40:06.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", upload-time = "2026-05-08T13:40:04.958Z" }, ] [[package]] name = "pygments" version = "2.20.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", upload-time = "2026-03-29T13:29:33.898Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pyjwt" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", upload-time = "2026-05-21T19:54:36.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", upload-time = "2026-05-21T19:54:35.362Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, ] [[package]] @@ -584,55 +854,106 @@ dependencies = [ { name = "pluggy" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "python-multipart" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", upload-time = "2026-06-04T16:18:58.647Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", upload-time = "2026-06-04T16:18:57.319Z" }, +] + +[[package]] +name = "pywin32" +version = "312" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", upload-time = "2026-06-04T07:49:28.836Z" }, + { url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", upload-time = "2026-06-04T07:49:31.85Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", upload-time = "2026-06-04T07:49:34.244Z" }, + { url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", upload-time = "2026-06-04T07:49:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", upload-time = "2026-06-04T07:49:38.876Z" }, + { url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", upload-time = "2026-06-04T07:49:41.083Z" }, + { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", upload-time = "2026-06-04T07:49:43.188Z" }, + { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", upload-time = "2026-06-04T07:49:45.34Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", upload-time = "2026-06-04T07:49:47.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", upload-time = "2026-06-04T07:49:50.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", upload-time = "2026-06-04T07:49:54.857Z" }, + { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", upload-time = "2026-06-04T07:49:57.531Z" }, ] [[package]] name = "pyyaml" version = "6.0.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, - { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, - { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, - { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, - { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, - { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, - { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, - { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, - { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, - { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, - { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, - { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, - { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, - { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, - { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, - { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, - { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, - { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, - { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, - { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, - { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, - { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, - { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, - { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, - { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, - { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, - { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, - { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, - { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, - { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, - { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", upload-time = "2025-10-13T15:30:47.625Z" }, ] [[package]] @@ -645,88 +966,249 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/43/25a8dcd3feedd735039a8f0b5b7e3b118232b5eae288c4fd9ab200d41094/rpds_py-2026.5.1.tar.gz", hash = "sha256:07b24fea40541e28570e5b795a4a38fbdcd12550c06bd0748005ecc8116ca256", upload-time = "2026-05-28T12:02:13.232Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e7/a78582dc57caa592dcc7d4fb69b61390561e908eb3d2f5df5928a8e354c0/rpds_py-2026.5.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3abe24a66e57adcfa645d718063a5fa5103ecc71ddbf26d78af8f9368018ff1d", upload-time = "2026-05-28T11:59:12.531Z" }, + { url = "https://files.pythonhosted.org/packages/a3/43/35e3f136343aef451e545ce8c38d36c2f93c0ed88703db8b64ba2b205c68/rpds_py-2026.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58b1d94308ddf0b1982f61f2eb54bf92997c9ece8a8093ef014250f4a517906c", upload-time = "2026-05-28T11:59:13.827Z" }, + { url = "https://files.pythonhosted.org/packages/20/e1/0f2160c5982d3157734d5cb3ed63d8b2d583a73c9864f77b666449f32cf8/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fa92420128dadce7f54bd73ba1825a273e9268fe9e35dbf7e6362890efa4e08", upload-time = "2026-05-28T11:59:15.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/11/ee0ba42aff83bf4effdbc576673c6be64c5e173978c3f6d537e94482f77d/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca653c6546386227cd9800d1bef6a348099acf8db4250341da6d90f663d6dfcb", upload-time = "2026-05-28T11:59:16.665Z" }, + { url = "https://files.pythonhosted.org/packages/11/df/d94aa6a499d4ac40afe2d7620f2c597fd3c0f182e854ad7cf3f596a81cb6/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66c93681c4729e4e3ecba31b8179fae083ff3118841672835140338b4b9867c1", upload-time = "2026-05-28T11:59:17.991Z" }, + { url = "https://files.pythonhosted.org/packages/1f/75/33d30f43bb2f458de11979486a591b1bf6e5651765ed1704c6197c2dc773/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ff257542e04796880e011e15cd4dc21c2599975df2aaa8f2c8495ca574e1a5", upload-time = "2026-05-28T11:59:19.434Z" }, + { url = "https://files.pythonhosted.org/packages/f4/1e/2c9096fc19d5fd084b0184ca2b651e659aa0a37e6fdbecf6ece47f147fe1/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6825cc329b290e93c5f6a9be2393118a763f6ccf6abd83704e0c102ca583644", upload-time = "2026-05-28T11:59:21Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e5/61ec9f8be8211ea7f48448195549e4aaf02004083475493b0e137702ecb2/rpds_py-2026.5.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:de42116e69cb53b911cc34aee5ab98f36c597b822545045d49e938818b99e5e4", upload-time = "2026-05-28T11:59:22.454Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ca/bcec1005c4f4a234f92a29078631fee49206c7265ccae966f18fd332e80e/rpds_py-2026.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0f920015df2a504bebaba6d4c31ccf3fcf942f92655c086da30b671aad19aa6", upload-time = "2026-05-28T11:59:23.845Z" }, + { url = "https://files.pythonhosted.org/packages/72/e6/4d5718c5cf26c522dc7c9999e238da1e77380b81d0c5d1df11e271ddfeb1/rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0408a24e44feb919423dc6d9da677cb5cddb894d2ca9e763967d156d9c60fab4", upload-time = "2026-05-28T11:59:25.184Z" }, + { url = "https://files.pythonhosted.org/packages/d4/25/2ee807bdb3e1f0b7eddf7782acd5665a8b5205a331a7d7244a52c4812fd9/rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cea68bcd53467561ae2f96a6bdad1544299ba97b5b0ddcd5ac3d376e5c781c24", upload-time = "2026-05-28T11:59:26.749Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c1/7d4c26f167f8c41501cc073d30ee22082b16ce358cf5b00ec97cbc7804ea/rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4be8b1d2a705cc37d08256004e1d07de143fa0075c8e85a3df020b776f62b732", upload-time = "2026-05-28T11:59:28.11Z" }, + { url = "https://files.pythonhosted.org/packages/04/1d/9d12b0a337bab46f4769f8857f4007e3b2d639e14f9a44a0efe157696e64/rpds_py-2026.5.1-cp312-cp312-win32.whl", hash = "sha256:6736718bd4fc49cbcb538ba30516fdbef161522acefb739657d48b97bd864fed", upload-time = "2026-05-28T11:59:29.689Z" }, + { url = "https://files.pythonhosted.org/packages/c5/93/e4116f2de7f56bc7406a76033dc501811ddeb22b7f056b92d632871ebb0c/rpds_py-2026.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:0a7d1eec967df0e9b22614a5e177622e0c89611d03727fa0cb48e45028907870", upload-time = "2026-05-28T11:59:31.033Z" }, + { url = "https://files.pythonhosted.org/packages/cb/53/6c3419d85eb2ec5938a37627c585b42d76a63bb731d6e42ed4b079ebf486/rpds_py-2026.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1841d067089e117142d79b98aa0df2f08b52f2ecc1819dd2700636c0db74a473", upload-time = "2026-05-28T11:59:32.318Z" }, + { url = "https://files.pythonhosted.org/packages/6c/32/14c961ad295f490eb0849ada8b79683e93a59b9de3afdd983eaf55fa6867/rpds_py-2026.5.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:efef4ac29c6ff495531eb17ee705b62841ecaa291b7c7077e848ea03e237164d", upload-time = "2026-05-28T11:59:33.655Z" }, + { url = "https://files.pythonhosted.org/packages/ca/bb/d1b85117967c11191441a7274ae616c65d93901d082c588f89a50a8da5ae/rpds_py-2026.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c39f5b67a8a2e67179ada2a954227d670fe65fa9098457f698f56ddf248709b3", upload-time = "2026-05-28T11:59:35Z" }, + { url = "https://files.pythonhosted.org/packages/7c/46/d84105f062e626a1b233f863907288a4708c2d833b8b4c6fb2764bc080c0/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5c30f3f04eef4fbd362226a6f31d7c8895ca4fbb6e0b790f6890a98d8da8559", upload-time = "2026-05-28T11:59:36.43Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/469d7959ce5b1201e1de135dc735b86db3b35dd0d1734f6a44246d5f061c/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:277f6c82f0580848796c7ecc8a7173aa3bfb928e4ff831261c2f60a81dc270db", upload-time = "2026-05-28T11:59:37.995Z" }, + { url = "https://files.pythonhosted.org/packages/dc/a2/57853d31a1116a561aa072794602ad3f6341e18d70a8523f1bd5b9fc1e5a/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63c2c4c213f1a4e3f3de28ecab029dbdee976324e729c0d7a55211be72576b02", upload-time = "2026-05-28T11:59:39.453Z" }, + { url = "https://files.pythonhosted.org/packages/99/63/3a8eabcad9314b7daf5c65f451d2c33d989235cd8a5762186cf2c3f5a4f8/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3350ec808fb538fe71a1f94dfaa0e29c598dfad805ce49f0caec5ae3183c652b", upload-time = "2026-05-28T11:59:40.896Z" }, + { url = "https://files.pythonhosted.org/packages/4b/25/05678d97fc25e2622df14dc530fb82023174ecfff6733991ed0d78f167bd/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1b964e3ab599e718dc46c018d104b1ebc007cbc6567d827c94a687fca56d77e", upload-time = "2026-05-28T11:59:42.626Z" }, + { url = "https://files.pythonhosted.org/packages/88/d1/8c90b6431e80a3b91b284a5c7c8c0c4f9c006444d90477a740d6e0f9c694/rpds_py-2026.5.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:19cb09fab7b7fc96b2a6e28f2e34b72a3705ff27b37edb77455316e5d3f3dc9b", upload-time = "2026-05-28T11:59:44.124Z" }, + { url = "https://files.pythonhosted.org/packages/ff/99/4638f672ab356682d633ee0da9255f5b67ce6efd0b85eb94ad3e255e65a5/rpds_py-2026.5.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abe76bcdba31e576cb83eeb8797aa0d882b738fef6dc65d0601fc753806a5b46", upload-time = "2026-05-28T11:59:47.177Z" }, + { url = "https://files.pythonhosted.org/packages/66/3f/3546524b6eb4cc2e1f363a3d638fa52f6c24faae3500c25fb488b02f1740/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8bff7073db3899158fff55ebf57b113a67030af26f80a18978f9f0aa60250ddf", upload-time = "2026-05-28T11:59:48.603Z" }, + { url = "https://files.pythonhosted.org/packages/c6/c3/7b3388c796fcf471bd17194242d4dc1a7608567c0fa422bcc1c5e79f9c1e/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8ba264fa49be666cd9cc56bf34ec7002fb3d27a4aee5bcb4d43d0d18feb1bb6f", upload-time = "2026-05-28T11:59:50.314Z" }, + { url = "https://files.pythonhosted.org/packages/61/1e/a3cb07f2795075d1d88efddae2f541359fde5f08c81ee114c29c2949c90a/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4860b603ddda0475a8885499b3729e90229d480105b42651962a5397d995fa89", upload-time = "2026-05-28T11:59:51.673Z" }, + { url = "https://files.pythonhosted.org/packages/a1/74/e758c03a5ef46f04c37f2651a2893db846d569ba8a7bca469d4b58939bcd/rpds_py-2026.5.1-cp313-cp313-win32.whl", hash = "sha256:7944270ae71383f6e2657dd7d5ce4eeb4ac2d0059a6738f0510583d462ab4842", upload-time = "2026-05-28T11:59:53.148Z" }, + { url = "https://files.pythonhosted.org/packages/70/ec/a2aca432db9c7359b40fa393eeeaa0d166c2f70175be956e75fa24197c44/rpds_py-2026.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:88647f43a73c4e01be19b04ceef0c8d3a1958153604d13c773becd8016f2a0cf", upload-time = "2026-05-28T11:59:54.505Z" }, + { url = "https://files.pythonhosted.org/packages/29/60/a73bfdd45b096574556acf303bbd9fa9eed36ca8a818b514e2a5d5fe2b9d/rpds_py-2026.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:453895624ecf7db7063b1004e44037522bbaef9ff6a945e59bc71662d7a03abd", upload-time = "2026-05-28T11:59:56.081Z" }, + { url = "https://files.pythonhosted.org/packages/18/e2/408105fd611823f00882aea810f3989a30d26b1bab8b6beb20f98c724e0e/rpds_py-2026.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:b4e4bc98639ec915f512fde3aa7a95e0041d95d9c3cc86eea841fa63cb1e8600", upload-time = "2026-05-28T11:59:57.448Z" }, + { url = "https://files.pythonhosted.org/packages/8d/58/5c4a43436843c90d0f6d19f82c200c80e3843ca9fa07b237623327f6d384/rpds_py-2026.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cacedb7a6e167680acba45ad5716e89067d225dc80da0d7040cae8c81d4572fa", upload-time = "2026-05-28T11:59:58.881Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c2/1a71acdacaf4e259b10278fb87b039ded3cf80041bcd89dd8a3ea702ded6/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68700371c5d7ae1412862ddfa719090925c93ecf351c566d66f09d04b136ea00", upload-time = "2026-05-28T12:00:00.516Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c8/535f3d9b65addd8e28aa87b83c6e526799c3717a88273db8ea795beeef7a/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:296c799becfa849c779c8725494fe9ed94959ed886787df4364b058465bad7f0", upload-time = "2026-05-28T12:00:02.394Z" }, + { url = "https://files.pythonhosted.org/packages/1c/91/dc033f313345c354ade914dbe73cdb90b615a4409ea02430d5356794f3d8/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3858b908218ee108d0bbfb2095ccc237648053c9bf98affad7cb079acaf1d97", upload-time = "2026-05-28T12:00:04.189Z" }, + { url = "https://files.pythonhosted.org/packages/27/fc/90fcbea459dbb8ddc18a2e0fd1de9412b48bc84ffff2db771cf714bacfd6/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4fb8d2e7cb2f850b169806d61d1b991738acec96500a75c30f49caf064ce7cef", upload-time = "2026-05-28T12:00:05.797Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1d/46cd11a228c9750684a798d98f878be6f614aa762438da7378f035e79e35/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27b74c10ed6a8f190f4287f53bcfea348b92a84a9c9f70d30183d1e6172d580d", upload-time = "2026-05-28T12:00:07.433Z" }, + { url = "https://files.pythonhosted.org/packages/24/4a/d9b0c6af3a1de03eb93741bbe8be2bdce84d8fda8224f3005451d86df389/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:b9a6528956191c48c52294a592dbd4a8386d7048bdb25c0efcb6b966466c6d83", upload-time = "2026-05-28T12:00:09.227Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b4/db7aaabdda6d020afc87d981bcc2f57a434c7dec60ecfc2ab3dd50b20351/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:af03e34e860047bc7a352b842856fcf78798fbb81132cc98bd2f907ab4eb9cd2", upload-time = "2026-05-28T12:00:10.779Z" }, + { url = "https://files.pythonhosted.org/packages/08/d6/070f6a41cbb343e2ac4171859bf3f3623e0ab002f72619d6d505313ec2de/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fea6e836d10abbe191d557d33bd58bd5987725fe63aa1eefe557d230209855bd", upload-time = "2026-05-28T12:00:12.443Z" }, + { url = "https://files.pythonhosted.org/packages/75/ab/1a71ea3589c4345dac0a0518f0e6a031cb42689277851b683c46d27463a5/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:fc0c0f878ea770a0a8a462456c5ad36fc9fe6358e6b76fdadc7f17575e0b8bf1", upload-time = "2026-05-28T12:00:14.09Z" }, + { url = "https://files.pythonhosted.org/packages/8a/22/9bf80a56069c0c443fcfefac639a86a744550a2898817a6dfd3e26654924/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e0b360f316d966b048b085857630b3cc51f3db2f07b06f440eac8f695374d1e3", upload-time = "2026-05-28T12:00:15.66Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/3b2c0a75c9e04125696f84ebdbbf304acf5a40b58ba4481cdb98a922c3ba/rpds_py-2026.5.1-cp313-cp313t-win32.whl", hash = "sha256:a2999883eedf72fdfb7520b92c7d4ec2572a71ff40239377aa604cc529eecafc", upload-time = "2026-05-28T12:00:17.291Z" }, + { url = "https://files.pythonhosted.org/packages/e7/8b/609157d5a25d37d4f29f92840ba531f416907c34ae5c5739dd21fc2bef98/rpds_py-2026.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e07be2a9d7122bd6e82dea89814ef8dc893feb1aae97fec1630f3263bbb30e55", upload-time = "2026-05-28T12:00:18.73Z" }, + { url = "https://files.pythonhosted.org/packages/d4/6f/19c1918a4b590d8de87e712e4abe4b3875771eff60216fb6153cf6665c68/rpds_py-2026.5.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1f2c391c3059798093b65df23aca2cac150460ae9c630d99dec83d703d9485b9", upload-time = "2026-05-28T12:00:20.217Z" }, + { url = "https://files.pythonhosted.org/packages/e5/60/a06fe7da34eca79dacbf958a2ba0c6eea85bc2b29de20080bf40f72f66fa/rpds_py-2026.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:413b424f7c4ee65ab5e5be91f5731be0f8b41a1ee2b12dfe810d716312e95a78", upload-time = "2026-05-28T12:00:21.711Z" }, + { url = "https://files.pythonhosted.org/packages/bf/ec/b2333b97b90e2a6ef6ca8ad386ee284968e74bcfe113b3f1a8d9036429a9/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c595a1d9255dce0599e13130d1440ab2506654f2b50294226ee06402f8fef63", upload-time = "2026-05-28T12:00:23.326Z" }, + { url = "https://files.pythonhosted.org/packages/14/7f/e00aae54067f2b488c4637961d5f58204d470795fc791085fa3f15060d2e/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1c27c5f6102eac8c03e7595a00827a53b271ba40a53b59ff8709170e0855ea4a", upload-time = "2026-05-28T12:00:24.89Z" }, + { url = "https://files.pythonhosted.org/packages/be/cc/423999bbb8ae8dc93c77fc1d5e984ade5eb89d237d3bb884ccfa72ae2890/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c7fcf61d44cacecaf3aea542b0e053db77972a4573e7ceda16fb2b399161195", upload-time = "2026-05-28T12:00:26.676Z" }, + { url = "https://files.pythonhosted.org/packages/0f/aa/c671bf660f12e68d3c52ff86c7066ed1372df5a0f4f2ff584e419b8207e7/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c817a189d4ee14290420e5ff051e4dd6baa13f3edf84685071dee07a6d538ee", upload-time = "2026-05-28T12:00:28.577Z" }, + { url = "https://files.pythonhosted.org/packages/19/c8/d63bb75b68afe77b229e3021c6031bcaf01da5db5b0e69d0d10f9ba679a7/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21846aac0ed2e0589f38c12dc44e77bb64e494b771eadbcf169cba00566ba7ba", upload-time = "2026-05-28T12:00:30.304Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/c51122014d8274ff37dc606d60049c3db7d83da02b5b282511e5a906a9a6/rpds_py-2026.5.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b317c87a13f769a4e787819bd508aaa5d69aa09b0880de9af6d3a8a54571cdec", upload-time = "2026-05-28T12:00:31.764Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f9/2790cb99c136a5363acdeacf5c27c56f3de0d4118a1f48fca83404c99c89/rpds_py-2026.5.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce87129d9f2c14fa6c4a8601fb80eb4488c80d38a20cd13758ef11123e14995d", upload-time = "2026-05-28T12:00:33.247Z" }, + { url = "https://files.pythonhosted.org/packages/e5/1b/e4fb584f8c75d35c38150ff6a332cda949e6f97acba1f4fd123b14ab56fe/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9cdddb6c1207d284d94fd1530adf57fbd797fe7c4b8704ba85f49414f2557e7d", upload-time = "2026-05-28T12:00:34.819Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f7/a6731b4216cb3793ea1af5391da240f5683dacc0d13e034fe5fc3503f240/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:4e237e139f94d3c036fd28eb9f564c99055476ff4ff05cd42be55ce349b5aa02", upload-time = "2026-05-28T12:00:36.268Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/2e051a81d95d8e63f4b35a1c463a87e8766bc3d083c067c5dfb6bf220747/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ed0954b524873214369184a9c82b0eaa45a3fbb9a798cd95b17e0d98499e7ea0", upload-time = "2026-05-28T12:00:37.82Z" }, + { url = "https://files.pythonhosted.org/packages/65/56/b5f6fdb2083e32bca8a8993d89e70db114b4756c9e2c38421328126689d2/rpds_py-2026.5.1-cp314-cp314-win32.whl", hash = "sha256:2d88621d6a7d4dfa633d21abe90f280bb205274e16b1d1e61c6ad4640b2453b7", upload-time = "2026-05-28T12:00:39.492Z" }, + { url = "https://files.pythonhosted.org/packages/fb/80/65a5aa96c155e611d1ed844e4e1f57f3e36b021f396d9f8585d756e6b90d/rpds_py-2026.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:cef8ac28d26f4dda3533060c20fbf80a325458fa9fd23ea72a73cdfa8e978838", upload-time = "2026-05-28T12:00:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/27/7c/ad185212e87b05f196daef92bc5f3caf07298eb47c295b5585c3dd3093ac/rpds_py-2026.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:eaaea962c68cdc68d4a533ba985ab8e9484277910bbfaa2ab3ef7732667bfed8", upload-time = "2026-05-28T12:00:43.15Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/e14ae18759020334646b031e708ab4158d653a938822bfb7b95ef2e93aa3/rpds_py-2026.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:21942f52dbbd5f8758bf021213d28bd45c39e873e65e2407faf5f1846f5761ad", upload-time = "2026-05-28T12:00:44.638Z" }, + { url = "https://files.pythonhosted.org/packages/31/9b/5f4a1e2f960bca3ac5d052b139dd31eed97b259f9d909173821760d542e8/rpds_py-2026.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f414556f6e3958300ff941e40c9f97e3dc9774ddd1b3434c475d73dd354bbed3", upload-time = "2026-05-28T12:00:46.14Z" }, + { url = "https://files.pythonhosted.org/packages/1a/71/1d9574d6a2fa20ab60eaa55c7467f5aa20cbc770f341a05f09c0876f59e2/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef1013a8625c74043210190b246f5b1551e09757c1f356c6e4160ef96c5bc081", upload-time = "2026-05-28T12:00:47.531Z" }, + { url = "https://files.pythonhosted.org/packages/0c/9a/37e99f4915a80aa71670263c1267f7ae0af95f53a3f61e6c3bdc016d4515/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cc68e231a77a5f0d774ae278a1f8e55c0456501820847c1e4efb3829f3441df6", upload-time = "2026-05-28T12:00:49.216Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ff/6e73f74b89d2e0715e0fc86b7dde893f9a61ae2f9b256ff3bdfe41ac4e94/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9baffb505aff33acc69b422a19f77806680f3c8632227d79f48de8a810d1c2c5", upload-time = "2026-05-28T12:00:51.111Z" }, + { url = "https://files.pythonhosted.org/packages/ea/e0/425faba25f59d74d4638b267f7c7a80e8649d2ef4db10a19b0c4a71e6e6f/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8d2f912928d426e8cfa396f7f3f8d29a59e6689c86dcca3c420730c1096322b", upload-time = "2026-05-28T12:00:52.77Z" }, + { url = "https://files.pythonhosted.org/packages/c6/76/7a41960e3fddae47fab43a28684d5da981401dffd88253de0944148654cb/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90f628283be835db980c941767d41c9a27b5239e54ba0a9c1335247e82406964", upload-time = "2026-05-28T12:00:54.215Z" }, + { url = "https://files.pythonhosted.org/packages/27/60/5f38dc70824fc6951b51d35377e577a3a3a4c81a6769cc5a2de25ebe0ad1/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:1ebb2f0ab7e16132995a72de805170e0203df0c3dd22e1ef1cd1fdd90bd7a131", upload-time = "2026-05-28T12:00:55.673Z" }, + { url = "https://files.pythonhosted.org/packages/60/1a/d60a38caa1505f4b9483c3fbbde12c94e1079154f4f401a6da96f7e77621/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f3df3d16ded76f1f8c9cdebd0e1ea55fdf4c23b812de189814da7cf229c22a81", upload-time = "2026-05-28T12:00:57.518Z" }, + { url = "https://files.pythonhosted.org/packages/87/ff/602fd3f174d6425f0bce05ad0dfbec0e96b38d0f7d08a79af5aa20083885/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9af8905b8f854990e40d5206aa5ac58d9b0fe0b7f351ff2bb086c20f6c8c6a47", upload-time = "2026-05-28T12:00:58.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c1/1be13327acdbead3eca1fde03b6a34dbb011f1e864e217f0d32cc1779a7f/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:036a36a87fb1cd3b214d11c4b3c4f7d2ddad933625dca1c900b56a057c07740a", upload-time = "2026-05-28T12:01:00.656Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d7/afb49b49d7f2be8b7ba1a9f0977fa5168003437b93086726f066544e8351/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:62ae3853454fe9ef283a03c96c2d835d39e84b14643a9d62c82ef0fb87d702ca", upload-time = "2026-05-28T12:01:02.22Z" }, + { url = "https://files.pythonhosted.org/packages/25/d1/dbef8c1f8a10f07beb62b5f054e20099fd9924b3ec001b8f0b6ac7813a85/rpds_py-2026.5.1-cp314-cp314t-win32.whl", hash = "sha256:6c3d771a46ec18b12af06ce36243a9a80b07a5d0515236332d90863ca8bb326a", upload-time = "2026-05-28T12:01:03.821Z" }, + { url = "https://files.pythonhosted.org/packages/2a/72/bfa4e61ab8e7dc1c8adf397e05e6cbdd4239357bd72b248d3de662f23915/rpds_py-2026.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c93c629be4636cf54337bd5f06c104d55e42ced54d681f6fe21ae510a65116f6", upload-time = "2026-05-28T12:01:05.194Z" }, + { url = "https://files.pythonhosted.org/packages/27/3a/7b5da92b640f67b6717ccafc83cdd06bfa7ff2395c3685c68922bb54d703/rpds_py-2026.5.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:3574b55c604b8f75dacb007136508bbc0db406e626301778096a133327e7f2fb", upload-time = "2026-05-28T12:01:06.722Z" }, + { url = "https://files.pythonhosted.org/packages/d7/8a/2aafd7ad355a1bd48ca76e2262b74b15e6432b5a1efe150efd4d779cd55d/rpds_py-2026.5.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:94068eb3ae6d43f5a786b7db96a406a34e6d5c24489feef32fd6e8946ea7b291", upload-time = "2026-05-28T12:01:08.441Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7d/6c9523c1abbe840a1b7fba3c516d48e1d3487cc80fea4366c4071cf56784/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a5b10e8ce894825f380a8f1b6444cf73c294dfea62afbb2d13e3a9e630cec1", upload-time = "2026-05-28T12:01:09.934Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5d/0b7b03fb1dc509321f01de3149784ab773e34c8573022029af8076afcb9c/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fc09f82e63d4bcd58149572f857a431bae851dc747e313c3b5bdf7abb907fda8", upload-time = "2026-05-28T12:01:11.48Z" }, + { url = "https://files.pythonhosted.org/packages/d7/e2/8ef6012999ebf1cb1c22f876d9ce5e63d960fd4631d2af3202d3f480aa25/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e10464d17df3b582745c25cec695cb9558bca2cb6ddb631aee1787fc72c767b2", upload-time = "2026-05-28T12:01:13.051Z" }, + { url = "https://files.pythonhosted.org/packages/80/af/1eeb029bec67582c226b7809172207cd005073af4ebd906e65ff494f4983/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ba05adbf15d994c38ec0b7ab32e858e5110c21e9009a00a86545fd220f84e038", upload-time = "2026-05-28T12:01:14.631Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/ffbe10711c4d766c1cab0557d6906c074f795814863c67b351355d29354a/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77c004fdc7b891967106f78ddfd7b076bfe6813c6139c6fff6aed3bcaa960b26", upload-time = "2026-05-28T12:01:16.153Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3a/30ba4a6ad457e5b070c18d742a33fb77d8d922b565cc881f8a5313d63bfe/rpds_py-2026.5.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:83bcf894486c9d78dd290d3c0124ff6dd8875d3025e2090a8ec49fcc37c55fdd", upload-time = "2026-05-28T12:01:17.809Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/62e242b53ce39c0814bd24e1a6e6eba6c92be716277745f317f9540a2e7b/rpds_py-2026.5.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3df104083952a0e0c6f10de33e440eabe98fb6317d23e1a58c68f6df08d01b9", upload-time = "2026-05-28T12:01:19.419Z" }, + { url = "https://files.pythonhosted.org/packages/38/c1/a770b9c186928a1ed0f7e6d7ae50e7f3950ed23e3f9e366dbc8e38cb55de/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:980450826cf22e133c57e0835070bdd0dd3f73b9b708c3ce223def2cb9469e14", upload-time = "2026-05-28T12:01:21.013Z" }, + { url = "https://files.pythonhosted.org/packages/21/7c/68e8579b95375b70d2a963103c42e705856cdb98569258bd807f4423891c/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:205dde846f24332ab0c1188699a043b8d165b79bb84529ce272c45048ff6be01", upload-time = "2026-05-28T12:01:22.548Z" }, + { url = "https://files.pythonhosted.org/packages/70/a1/a6135aed5730ff03ab957182259987ac11e55fb392a28dc6f0592048a280/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3966b82dd563176396df030f3dd52a6e54cb69b718e95e78bd555ed3d1e0185d", upload-time = "2026-05-28T12:01:24.118Z" }, + { url = "https://files.pythonhosted.org/packages/09/6e/f24201a76a84e6c49d0bdfdfcb735210e21701e9b21c5bfc0ba497dd62f6/rpds_py-2026.5.1-cp315-cp315-win32.whl", hash = "sha256:7818f8d0a415be74d2be3590b0a1c1f463a642f4d0217e7d10602dceef5b79aa", upload-time = "2026-05-28T12:01:25.522Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e4/966bc240bb0485fc265278f6de44d05834bf0b3618886e0b22e33d54c49a/rpds_py-2026.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:b3cc20c0d800af78fd0fac68086e28c1856cec51ea528bb81ea851aa40d39325", upload-time = "2026-05-28T12:01:27.062Z" }, + { url = "https://files.pythonhosted.org/packages/5c/5c/a15a59269cd5e74472734516c73795c15eccfc841b3d4b0228c3f53f19d0/rpds_py-2026.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:3609e9939a8a76cd904cf98a3f1f13b5dc7e150adeaee89e0ea09652ea213e16", upload-time = "2026-05-28T12:01:28.51Z" }, + { url = "https://files.pythonhosted.org/packages/e0/22/135ce03804e179a71ceb13be095deda4a279bc88f7a6b8fa161c5ad44e12/rpds_py-2026.5.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:5d333a7127d4b307601ac37792bee01bb95c867cbfacf21b6375b804d6bbd723", upload-time = "2026-05-28T12:01:30.214Z" }, + { url = "https://files.pythonhosted.org/packages/3b/5f/f1f6d2652eb9d848f6eb369d8db83a2da6249bb49ad2c2a48f45d54538d3/rpds_py-2026.5.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:b5f077b44a4f7808520f66dae234988d867deb9aed9be5da057ce9ba831b2a41", upload-time = "2026-05-28T12:01:31.656Z" }, + { url = "https://files.pythonhosted.org/packages/88/66/b74182775691ea2290c99e52ac8d5db844e56fbec90ce421f107658c8314/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d8f9b7b78c9538fc9e04e82ec0e888ff0c3cffcfad152c77e57cd09351a98a", upload-time = "2026-05-28T12:01:33.136Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8f/15e5a61d9f0a43902d36561d4f07cae6ae9f4716be825159fd72717f33af/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e3a8ae58895ac107ed934a6bf51e5846f95c53b9b940c2c6d310838fd5846358", upload-time = "2026-05-28T12:01:34.574Z" }, + { url = "https://files.pythonhosted.org/packages/02/c3/f859b12763a80540cdf2af0f15b19904cf756a71d7bdd3f82ff3e5b1bbf9/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0957cf3c2b8632ec7aaebffebea8005b353cc2a237b6e2ae3c2cac0820704cfb", upload-time = "2026-05-28T12:01:36.127Z" }, + { url = "https://files.pythonhosted.org/packages/1c/c7/ff27c2ac8411d30b03b1829fd88cae8dad1a4d0da48dd25e57c4038042e6/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c396c1304de421050b3681ea70f371874b54d41b0151e96109758144c231e30b", upload-time = "2026-05-28T12:01:37.635Z" }, + { url = "https://files.pythonhosted.org/packages/6e/67/fe92ee32a6cc05c77228a2f8b1762e7124f386ec20ff83d0757b762d58d0/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad1bff7f666b9598e573815affd666aac6a13a585dde336f843e33350c7fadc", upload-time = "2026-05-28T12:01:39.307Z" }, + { url = "https://files.pythonhosted.org/packages/f8/91/b4d6685c27aba55bd82f25b278be8237038117d05f9659a6213ad3408130/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:656a042550878f12d45752452d47094b7cfe5ad1e9d7b87b5a22ad3ae5ff8015", upload-time = "2026-05-28T12:01:41.043Z" }, + { url = "https://files.pythonhosted.org/packages/bd/79/2c1d832a53c8e0f8e98fc970ec257b950fecd4f62be2ab7182b500a0cbc8/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73c4bd4f70294737b5206a3e8e30ccadbf8a60301831c8ea23eec5dbeea1ecfa", upload-time = "2026-05-28T12:01:43.032Z" }, + { url = "https://files.pythonhosted.org/packages/78/c4/c98117b03c6a8581ab2c2dfccfe9a5ad82bd8128a3c28b46a6ad2d97c393/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:43bca78665423cabae77146f2fe7ce55272b6c8d55d82cca83effd42c7e13972", upload-time = "2026-05-28T12:01:44.648Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c1/bc479ca069200af730881b1bd525e3114b2b391a351509fcb1b772f28086/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:42d0f20e85e549c870749d0e247f0c10d318a45b7e9676d575d2dcb04a1b2e66", upload-time = "2026-05-28T12:01:46.337Z" }, + { url = "https://files.pythonhosted.org/packages/77/65/38ab2f90df44c2febfb63cc10ced40763d9b4bc94d173e734528663fe7f5/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b1be5c35683684d5331b93600c210e8367c254683d8a6df6bd21bd2da3a334fb", upload-time = "2026-05-28T12:01:48.109Z" }, + { url = "https://files.pythonhosted.org/packages/15/2d/ce1f605fe036aadd460e5822e578c6c7ec3a860936cca37d6e0f299daa77/rpds_py-2026.5.1-cp315-cp315t-win32.whl", hash = "sha256:75808f6c38ce7749bb68cc2770161aae5045e6c6f6781a9782e74b93304399df", upload-time = "2026-05-28T12:01:49.648Z" }, + { url = "https://files.pythonhosted.org/packages/79/cb/966040123eb102371559746908ef2c9471f4d43e17ec9a645a2258dab64b/rpds_py-2026.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:90bd6630002a1c7f09e7843dd79f0d24f3d2897cc25a753480917865d14f15b3", upload-time = "2026-05-28T12:01:51.408Z" }, ] [[package]] name = "ruff" version = "0.15.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/14/b0/73cf7550861e2b4824950b8b52eebdcc5adc792a00c514406556c5b80817/ruff-0.15.8.tar.gz", hash = "sha256:995f11f63597ee362130d1d5a327a87cb6f3f5eae3094c620bcc632329a4d26e", size = 4610921, upload-time = "2026-03-26T18:39:38.675Z" } +sdist = { url = "https://files.pythonhosted.org/packages/14/b0/73cf7550861e2b4824950b8b52eebdcc5adc792a00c514406556c5b80817/ruff-0.15.8.tar.gz", hash = "sha256:995f11f63597ee362130d1d5a327a87cb6f3f5eae3094c620bcc632329a4d26e", upload-time = "2026-03-26T18:39:38.675Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/92/c445b0cd6da6e7ae51e954939cb69f97e008dbe750cfca89b8cedc081be7/ruff-0.15.8-py3-none-linux_armv6l.whl", hash = "sha256:cbe05adeba76d58162762d6b239c9056f1a15a55bd4b346cfd21e26cd6ad7bc7", size = 10527394, upload-time = "2026-03-26T18:39:41.566Z" }, - { url = "https://files.pythonhosted.org/packages/eb/92/f1c662784d149ad1414cae450b082cf736430c12ca78367f20f5ed569d65/ruff-0.15.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d3e3d0b6ba8dca1b7ef9ab80a28e840a20070c4b62e56d675c24f366ef330570", size = 10905693, upload-time = "2026-03-26T18:39:30.364Z" }, - { url = "https://files.pythonhosted.org/packages/ca/f2/7a631a8af6d88bcef997eb1bf87cc3da158294c57044aafd3e17030613de/ruff-0.15.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6ee3ae5c65a42f273f126686353f2e08ff29927b7b7e203b711514370d500de3", size = 10323044, upload-time = "2026-03-26T18:39:33.37Z" }, - { url = "https://files.pythonhosted.org/packages/67/18/1bf38e20914a05e72ef3b9569b1d5c70a7ef26cd188d69e9ca8ef588d5bf/ruff-0.15.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdce027ada77baa448077ccc6ebb2fa9c3c62fd110d8659d601cf2f475858d94", size = 10629135, upload-time = "2026-03-26T18:39:44.142Z" }, - { url = "https://files.pythonhosted.org/packages/d2/e9/138c150ff9af60556121623d41aba18b7b57d95ac032e177b6a53789d279/ruff-0.15.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12e617fc01a95e5821648a6df341d80456bd627bfab8a829f7cfc26a14a4b4a3", size = 10348041, upload-time = "2026-03-26T18:39:52.178Z" }, - { url = "https://files.pythonhosted.org/packages/02/f1/5bfb9298d9c323f842c5ddeb85f1f10ef51516ac7a34ba446c9347d898df/ruff-0.15.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:432701303b26416d22ba696c39f2c6f12499b89093b61360abc34bcc9bf07762", size = 11121987, upload-time = "2026-03-26T18:39:55.195Z" }, - { url = "https://files.pythonhosted.org/packages/10/11/6da2e538704e753c04e8d86b1fc55712fdbdcc266af1a1ece7a51fff0d10/ruff-0.15.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d910ae974b7a06a33a057cb87d2a10792a3b2b3b35e33d2699fdf63ec8f6b17a", size = 11951057, upload-time = "2026-03-26T18:39:19.18Z" }, - { url = "https://files.pythonhosted.org/packages/83/f0/c9208c5fd5101bf87002fed774ff25a96eea313d305f1e5d5744698dc314/ruff-0.15.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2033f963c43949d51e6fdccd3946633c6b37c484f5f98c3035f49c27395a8ab8", size = 11464613, upload-time = "2026-03-26T18:40:06.301Z" }, - { url = "https://files.pythonhosted.org/packages/f8/22/d7f2fabdba4fae9f3b570e5605d5eb4500dcb7b770d3217dca4428484b17/ruff-0.15.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f29b989a55572fb885b77464cf24af05500806ab4edf9a0fd8977f9759d85b1", size = 11257557, upload-time = "2026-03-26T18:39:57.972Z" }, - { url = "https://files.pythonhosted.org/packages/71/8c/382a9620038cf6906446b23ce8632ab8c0811b8f9d3e764f58bedd0c9a6f/ruff-0.15.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ac51d486bf457cdc985a412fb1801b2dfd1bd8838372fc55de64b1510eff4bec", size = 11169440, upload-time = "2026-03-26T18:39:22.205Z" }, - { url = "https://files.pythonhosted.org/packages/4d/0d/0994c802a7eaaf99380085e4e40c845f8e32a562e20a38ec06174b52ef24/ruff-0.15.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c9861eb959edab053c10ad62c278835ee69ca527b6dcd72b47d5c1e5648964f6", size = 10605963, upload-time = "2026-03-26T18:39:46.682Z" }, - { url = "https://files.pythonhosted.org/packages/19/aa/d624b86f5b0aad7cef6bbf9cd47a6a02dfdc4f72c92a337d724e39c9d14b/ruff-0.15.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8d9a5b8ea13f26ae90838afc33f91b547e61b794865374f114f349e9036835fb", size = 10357484, upload-time = "2026-03-26T18:39:49.176Z" }, - { url = "https://files.pythonhosted.org/packages/35/c3/e0b7835d23001f7d999f3895c6b569927c4d39912286897f625736e1fd04/ruff-0.15.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c2a33a529fb3cbc23a7124b5c6ff121e4d6228029cba374777bd7649cc8598b8", size = 10830426, upload-time = "2026-03-26T18:40:03.702Z" }, - { url = "https://files.pythonhosted.org/packages/f0/51/ab20b322f637b369383adc341d761eaaa0f0203d6b9a7421cd6e783d81b9/ruff-0.15.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:75e5cd06b1cf3f47a3996cfc999226b19aa92e7cce682dcd62f80d7035f98f49", size = 11345125, upload-time = "2026-03-26T18:39:27.799Z" }, - { url = "https://files.pythonhosted.org/packages/37/e6/90b2b33419f59d0f2c4c8a48a4b74b460709a557e8e0064cf33ad894f983/ruff-0.15.8-py3-none-win32.whl", hash = "sha256:bc1f0a51254ba21767bfa9a8b5013ca8149dcf38092e6a9eb704d876de94dc34", size = 10571959, upload-time = "2026-03-26T18:39:36.117Z" }, - { url = "https://files.pythonhosted.org/packages/1f/a2/ef467cb77099062317154c63f234b8a7baf7cb690b99af760c5b68b9ee7f/ruff-0.15.8-py3-none-win_amd64.whl", hash = "sha256:04f79eff02a72db209d47d665ba7ebcad609d8918a134f86cb13dd132159fc89", size = 11743893, upload-time = "2026-03-26T18:39:25.01Z" }, - { url = "https://files.pythonhosted.org/packages/15/e2/77be4fff062fa78d9b2a4dea85d14785dac5f1d0c1fb58ed52331f0ebe28/ruff-0.15.8-py3-none-win_arm64.whl", hash = "sha256:cf891fa8e3bb430c0e7fac93851a5978fc99c8fa2c053b57b118972866f8e5f2", size = 11048175, upload-time = "2026-03-26T18:40:01.06Z" }, + { url = "https://files.pythonhosted.org/packages/4a/92/c445b0cd6da6e7ae51e954939cb69f97e008dbe750cfca89b8cedc081be7/ruff-0.15.8-py3-none-linux_armv6l.whl", hash = "sha256:cbe05adeba76d58162762d6b239c9056f1a15a55bd4b346cfd21e26cd6ad7bc7", upload-time = "2026-03-26T18:39:41.566Z" }, + { url = "https://files.pythonhosted.org/packages/eb/92/f1c662784d149ad1414cae450b082cf736430c12ca78367f20f5ed569d65/ruff-0.15.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d3e3d0b6ba8dca1b7ef9ab80a28e840a20070c4b62e56d675c24f366ef330570", upload-time = "2026-03-26T18:39:30.364Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f2/7a631a8af6d88bcef997eb1bf87cc3da158294c57044aafd3e17030613de/ruff-0.15.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6ee3ae5c65a42f273f126686353f2e08ff29927b7b7e203b711514370d500de3", upload-time = "2026-03-26T18:39:33.37Z" }, + { url = "https://files.pythonhosted.org/packages/67/18/1bf38e20914a05e72ef3b9569b1d5c70a7ef26cd188d69e9ca8ef588d5bf/ruff-0.15.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdce027ada77baa448077ccc6ebb2fa9c3c62fd110d8659d601cf2f475858d94", upload-time = "2026-03-26T18:39:44.142Z" }, + { url = "https://files.pythonhosted.org/packages/d2/e9/138c150ff9af60556121623d41aba18b7b57d95ac032e177b6a53789d279/ruff-0.15.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12e617fc01a95e5821648a6df341d80456bd627bfab8a829f7cfc26a14a4b4a3", upload-time = "2026-03-26T18:39:52.178Z" }, + { url = "https://files.pythonhosted.org/packages/02/f1/5bfb9298d9c323f842c5ddeb85f1f10ef51516ac7a34ba446c9347d898df/ruff-0.15.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:432701303b26416d22ba696c39f2c6f12499b89093b61360abc34bcc9bf07762", upload-time = "2026-03-26T18:39:55.195Z" }, + { url = "https://files.pythonhosted.org/packages/10/11/6da2e538704e753c04e8d86b1fc55712fdbdcc266af1a1ece7a51fff0d10/ruff-0.15.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d910ae974b7a06a33a057cb87d2a10792a3b2b3b35e33d2699fdf63ec8f6b17a", upload-time = "2026-03-26T18:39:19.18Z" }, + { url = "https://files.pythonhosted.org/packages/83/f0/c9208c5fd5101bf87002fed774ff25a96eea313d305f1e5d5744698dc314/ruff-0.15.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2033f963c43949d51e6fdccd3946633c6b37c484f5f98c3035f49c27395a8ab8", upload-time = "2026-03-26T18:40:06.301Z" }, + { url = "https://files.pythonhosted.org/packages/f8/22/d7f2fabdba4fae9f3b570e5605d5eb4500dcb7b770d3217dca4428484b17/ruff-0.15.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f29b989a55572fb885b77464cf24af05500806ab4edf9a0fd8977f9759d85b1", upload-time = "2026-03-26T18:39:57.972Z" }, + { url = "https://files.pythonhosted.org/packages/71/8c/382a9620038cf6906446b23ce8632ab8c0811b8f9d3e764f58bedd0c9a6f/ruff-0.15.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ac51d486bf457cdc985a412fb1801b2dfd1bd8838372fc55de64b1510eff4bec", upload-time = "2026-03-26T18:39:22.205Z" }, + { url = "https://files.pythonhosted.org/packages/4d/0d/0994c802a7eaaf99380085e4e40c845f8e32a562e20a38ec06174b52ef24/ruff-0.15.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c9861eb959edab053c10ad62c278835ee69ca527b6dcd72b47d5c1e5648964f6", upload-time = "2026-03-26T18:39:46.682Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/d624b86f5b0aad7cef6bbf9cd47a6a02dfdc4f72c92a337d724e39c9d14b/ruff-0.15.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8d9a5b8ea13f26ae90838afc33f91b547e61b794865374f114f349e9036835fb", upload-time = "2026-03-26T18:39:49.176Z" }, + { url = "https://files.pythonhosted.org/packages/35/c3/e0b7835d23001f7d999f3895c6b569927c4d39912286897f625736e1fd04/ruff-0.15.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c2a33a529fb3cbc23a7124b5c6ff121e4d6228029cba374777bd7649cc8598b8", upload-time = "2026-03-26T18:40:03.702Z" }, + { url = "https://files.pythonhosted.org/packages/f0/51/ab20b322f637b369383adc341d761eaaa0f0203d6b9a7421cd6e783d81b9/ruff-0.15.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:75e5cd06b1cf3f47a3996cfc999226b19aa92e7cce682dcd62f80d7035f98f49", upload-time = "2026-03-26T18:39:27.799Z" }, + { url = "https://files.pythonhosted.org/packages/37/e6/90b2b33419f59d0f2c4c8a48a4b74b460709a557e8e0064cf33ad894f983/ruff-0.15.8-py3-none-win32.whl", hash = "sha256:bc1f0a51254ba21767bfa9a8b5013ca8149dcf38092e6a9eb704d876de94dc34", upload-time = "2026-03-26T18:39:36.117Z" }, + { url = "https://files.pythonhosted.org/packages/1f/a2/ef467cb77099062317154c63f234b8a7baf7cb690b99af760c5b68b9ee7f/ruff-0.15.8-py3-none-win_amd64.whl", hash = "sha256:04f79eff02a72db209d47d665ba7ebcad609d8918a134f86cb13dd132159fc89", upload-time = "2026-03-26T18:39:25.01Z" }, + { url = "https://files.pythonhosted.org/packages/15/e2/77be4fff062fa78d9b2a4dea85d14785dac5f1d0c1fb58ed52331f0ebe28/ruff-0.15.8-py3-none-win_arm64.whl", hash = "sha256:cf891fa8e3bb430c0e7fac93851a5978fc99c8fa2c053b57b118972866f8e5f2", upload-time = "2026-03-26T18:40:01.06Z" }, ] [[package]] name = "sqlglot" version = "30.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/64/89299aefc6ebdf4fc899f5dc14c7fcb7eb9da9290a2b4d615ae7ab884b17/sqlglot-30.8.0.tar.gz", hash = "sha256:1c5f93fb742dd9aaa75eee6bb33a637794a858b9a86375fac23a2dc0f7bc127e", size = 5869750, upload-time = "2026-05-13T09:04:38.923Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/64/89299aefc6ebdf4fc899f5dc14c7fcb7eb9da9290a2b4d615ae7ab884b17/sqlglot-30.8.0.tar.gz", hash = "sha256:1c5f93fb742dd9aaa75eee6bb33a637794a858b9a86375fac23a2dc0f7bc127e", upload-time = "2026-05-13T09:04:38.923Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/88/4e/80705091aaf9c95e125d243f0aa871bc9f3670b4c9d963e6bad3b3dce8ff/sqlglot-30.8.0-py3-none-any.whl", hash = "sha256:af903378c331d5b72277a1b41118f07bc3e50cf4478e2d47eed12c96ee6a22a4", size = 687831, upload-time = "2026-05-13T09:04:36.336Z" }, + { url = "https://files.pythonhosted.org/packages/88/4e/80705091aaf9c95e125d243f0aa871bc9f3670b4c9d963e6bad3b3dce8ff/sqlglot-30.8.0-py3-none-any.whl", hash = "sha256:af903378c331d5b72277a1b41118f07bc3e50cf4478e2d47eed12c96ee6a22a4", upload-time = "2026-05-13T09:04:36.336Z" }, +] + +[[package]] +name = "sse-starlette" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/2b/58abc2d1fd397e7dde08e947e05c884d8ef2f78d5e2588c17a12d42d6994/sse_starlette-3.4.4.tar.gz", hash = "sha256:07e0fa0460138baf25cdd5fb28683472c3995dc1642225191b3832d62526bcb0", upload-time = "2026-05-12T17:37:17.019Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/67/805710444ea8cc75fbf70b920ed431a560c4bf9c57f7d5a3117213189399/sse_starlette-3.4.4-py3-none-any.whl", hash = "sha256:3f4dd50d8aed2771a091f3a83000323fc3844541c16b4fe585ae2420cc6df973", upload-time = "2026-05-12T17:37:15.601Z" }, +] + +[[package]] +name = "starlette" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/44/ec35f1b6e83094b997da438a02c8c9b0ade2b1e84cfc48bd4656780760a6/starlette-1.2.1.tar.gz", hash = "sha256:9b9b5ebb992e67d6093741e63c2f59e4f6fff986f81163c087867bd7b924b3f6", upload-time = "2026-05-31T01:07:51.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/54/196d0c1db10af76baa4f64894448505d60d3cdf70ef92cbb35f46a4e4c71/starlette-1.2.1-py3-none-any.whl", hash = "sha256:4de0082d08c8f6764a85a54cf1120d6939507a19905c7768acad2a9f875d2b89", upload-time = "2026-05-31T01:07:50.09Z" }, ] [[package]] name = "tomlkit" version = "0.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", size = 161875, upload-time = "2026-05-10T07:38:22.245Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", upload-time = "2026-05-10T07:38:22.245Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", size = 41328, upload-time = "2026-05-10T07:38:23.517Z" }, + { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", upload-time = "2026-05-10T07:38:23.517Z" }, ] [[package]] name = "types-pyyaml" version = "6.0.12.20250915" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/69/3c51b36d04da19b92f9e815be12753125bd8bc247ba0470a982e6979e71c/types_pyyaml-6.0.12.20250915.tar.gz", hash = "sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3", size = 17522, upload-time = "2025-09-15T03:01:00.728Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/69/3c51b36d04da19b92f9e815be12753125bd8bc247ba0470a982e6979e71c/types_pyyaml-6.0.12.20250915.tar.gz", hash = "sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3", upload-time = "2025-09-15T03:01:00.728Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/e0/1eed384f02555dde685fff1a1ac805c1c7dcb6dd019c916fe659b1c1f9ec/types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6", size = 20338, upload-time = "2025-09-15T03:00:59.218Z" }, + { url = "https://files.pythonhosted.org/packages/bd/e0/1eed384f02555dde685fff1a1ac805c1c7dcb6dd019c916fe659b1c1f9ec/types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6", upload-time = "2025-09-15T03:00:59.218Z" }, ] [[package]] name = "typing-extensions" version = "4.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", upload-time = "2025-10-01T02:14:41.687Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", upload-time = "2025-10-01T02:14:40.154Z" }, ] [[package]] name = "urllib3" version = "2.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.49.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/1f/fa18009dea8469069cca78a4e877a008ab78f08b064bfc9ab891579077ff/uvicorn-0.49.0.tar.gz", hash = "sha256:ebf4271aa580d9de97f93192d4595176df6e91f9aae919ca73e4fc07df1e66a3", upload-time = "2026-06-03T22:01:30.448Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, + { url = "https://files.pythonhosted.org/packages/88/fa/e1388bbcf24ef3274f45c0c1c7b501fd14971037c1b6ee23610553307497/uvicorn-0.49.0-py3-none-any.whl", hash = "sha256:ba3d14c3ee7e41c6c654c46c9eb489d33213cdd30aa1696eab1374337c13f68f", upload-time = "2026-06-03T22:01:29.037Z" }, ] [[package]] name = "xmltodict" version = "1.0.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz", hash = "sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61", size = 26124, upload-time = "2026-02-22T02:21:22.074Z" } +sdist = { url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz", hash = "sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61", upload-time = "2026-02-22T02:21:22.074Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl", hash = "sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a", size = 13580, upload-time = "2026-02-22T02:21:21.039Z" }, + { url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl", hash = "sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a", upload-time = "2026-02-22T02:21:21.039Z" }, ] [[package]] @@ -739,7 +1221,7 @@ dependencies = [ { name = "tomlkit" }, { name = "xmltodict" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/6a/eb9721ed0929d0f55d167c2222d288b529723afbef0a07ed7aa6cca72380/yq-3.4.3.tar.gz", hash = "sha256:ba586a1a6f30cf705b2f92206712df2281cd320280210e7b7b80adcb8f256e3b", size = 33214, upload-time = "2024-04-27T15:39:43.29Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/6a/eb9721ed0929d0f55d167c2222d288b529723afbef0a07ed7aa6cca72380/yq-3.4.3.tar.gz", hash = "sha256:ba586a1a6f30cf705b2f92206712df2281cd320280210e7b7b80adcb8f256e3b", upload-time = "2024-04-27T15:39:43.29Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/ba/d1b21f3e57469030bd6536b91bb28fedd2511d4e68b5a575f2bdb3a3dbb6/yq-3.4.3-py3-none-any.whl", hash = "sha256:547e34bc3caacce83665fd3429bf7c85f8e8b6b9aaee3f953db1ad716ff3434d", size = 18812, upload-time = "2024-04-27T15:39:41.652Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ba/d1b21f3e57469030bd6536b91bb28fedd2511d4e68b5a575f2bdb3a3dbb6/yq-3.4.3-py3-none-any.whl", hash = "sha256:547e34bc3caacce83665fd3429bf7c85f8e8b6b9aaee3f953db1ad716ff3434d", upload-time = "2024-04-27T15:39:41.652Z" }, ] From 0a66850f2a35805b6f494b918c03f3c325c19393 Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:54:32 -0400 Subject: [PATCH 18/77] Update readme (#4) * Refactor expression parsing (#1) * Improve control flow conversion (#2) Improves conversion of control flow, expression-based parameters, and schedule triggers. Co-authored-by: Isaac * docs: update README install and usage for marketplace + setup Updates README.md to align with new installation and usage patterns. Co-authored-by: Isaac * Remove legacy files --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0ba13d5..356fa27 100644 --- a/README.md +++ b/README.md @@ -36,23 +36,55 @@ flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline de ## Quick Start -1. Install the plugin in Claude Code: - ```bash - claude plugin install ghanse/flowx +1. Add the flowx marketplace and install the plugin in Claude Code: ``` + /plugin marketplace add databricks-solutions/flowx + /plugin install flowx@flowx + ``` + Then run `/reload-plugins` to activate it. -2. Run the end-to-end migration: +2. Set up the runtime (run once): + ``` + /flowx:flowx-setup + ``` + This auto-detects your environment and prepares the right execution path — + a local Python virtual environment for Claude Code, or a deployed MCP server + for Databricks Genie Code. See [Setup](#setup) for details. + +3. Run the end-to-end migration: ``` /flowx:flowx-migrate ``` Or run individual phases: ``` - /flowx:flowx-discover # Parse ADF JSON, produce inventory + complexity report - /flowx:flowx-convert # Deterministic + agentic translation + /flowx:flowx-discover # Parse ADF JSON, produce inventory + complexity report + /flowx:flowx-convert # Deterministic + agentic translation /flowx:flowx-package # Generate DABs project ``` +## Setup + +`/flowx:flowx-setup` keys off the `DATABRICKS_RUNTIME_VERSION` environment variable +(the same signal the rest of the plugin uses to detect Databricks) and prepares one +of two execution paths: + +- **Local / Claude Code (virtual environment).** The phases run from the plugin's + CLI. Setup runs `scripts/bootstrap.sh`, which creates a `.venv`, installs + `requirements.txt`, and writes the resolved interpreter path to a + `.migration-venv` marker file that the phase skills read. Optionally, a local + (stdio) MCP server can be registered to drive the phases through MCP tools + instead of the CLI. + +- **Databricks Genie Code (MCP server, no virtual environment).** The phases run as + a single `flowx` MCP tool hosted on a Databricks App. Setup runs `app/deploy.sh`, + which stages a self-contained bundle, syncs it to `/Workspace/Shared/mcp-flowx`, + and deploys the `mcp-flowx` app. You then grant app/data access and register the + app under Genie Code **Settings → MCP Servers**. No venv is created on this path. + +Run setup once before any other flowx skill, or again whenever the environment is +missing. + ## Supported ADF Activity Types ### Deterministic (16 types) @@ -145,6 +177,10 @@ make clean # Remove build artifacts - Python 3.12+ - [uv](https://docs.astral.sh/uv/) package manager +These prerequisites are for contributing to the flowx project. Plugin *users* do not need +`uv` — `/flowx:flowx-setup` provisions the runtime (a pip-based `.venv` locally, or +the MCP server on Databricks). + ## Contributing 1. Fork the repository From 805882c1cca3533a10c98cd7018736844e33c548 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Mon, 6 Jul 2026 11:24:26 -0400 Subject: [PATCH 19/77] Add SDK-based installation --- FIX_0603_CHANGES.md | 672 ------------------ README.md | 130 +++- app/README.md | 34 +- app/deploy_app.py | 115 +++ app/deploy_helpers.py | 120 ++++ docs/README.md | 2 +- docs/app/(home)/page.tsx | 2 +- docs/app/layout.config.tsx | 4 +- docs/content/docs/architecture.mdx | 2 +- docs/content/docs/installation.mdx | 224 +++--- scripts/bootstrap.sh | 6 +- skills/flowx-convert/SKILL.md | 23 +- .../references/activity-mapping.md | 26 +- skills/flowx-discover/SKILL.md | 13 +- skills/flowx-migrate/references/workflow.md | 10 +- skills/flowx-setup/SKILL.md | 29 +- skills/setup/SKILL.md | 94 --- src/flowx/models/adf_ast.py | 2 - src/flowx/models/ir.py | 4 - src/flowx/parser/adf_loader.py | 44 +- src/flowx/preparer/workflow_preparer.py | 5 +- src/flowx/translator/engine.py | 14 +- tests/integration/test_end_to_end.py | 6 +- tests/unit/test_adf_loader.py | 29 +- tests/unit/test_until_agentic_handler.py | 2 - 25 files changed, 552 insertions(+), 1060 deletions(-) delete mode 100644 FIX_0603_CHANGES.md create mode 100644 app/deploy_app.py create mode 100644 app/deploy_helpers.py delete mode 100644 skills/setup/SKILL.md diff --git a/FIX_0603_CHANGES.md b/FIX_0603_CHANGES.md deleted file mode 100644 index 5f1e5f0..0000000 --- a/FIX_0603_CHANGES.md +++ /dev/null @@ -1,672 +0,0 @@ -# FIX_0603 Changes - -Track of changes applied to branch `fix-0603` from the validation-implementation workflow. - -## Iteration 1 — 2026-06-03 - -Implemented 9 of 13 plan changes (4 deferred to a future iteration: see "Deferred" below). - -### Change: `expr-resolver-globalparams-and-wrappers` (P0) - -- **Title**: Teach expression resolver about factory globalParameters, @json/@string/@array no-op wrappers, trailing-whitespace expressions, item() safe-nav, and @linkedService().X refs. -- **Rationale**: Drives NB-1, NB-4 (partial), NB-5, CF-003, LSC-001, LSC-005, VAR-003, VAR-005 — five separate gaps trace to the same resolver weaknesses. Loading globalParameters once at parse time and stripping leading-`@` wrappers unblocks all of them without refactor. -- **Files changed**: - - `src/flowx/models/adf_ast.py` — added `AdfDefinitions.global_parameters`. - - `src/flowx/models/ir.py` — added `TranslationContext.global_parameters` and `linked_service_parameters`, plus `get_global_parameter` / `get_linked_service_parameter` / `with_linked_service_parameters` helpers. - - `src/flowx/parser/adf_loader.py` — added `factory_dir` loader and `_parse_factory_global_parameters` helper; same for the ARM-template branch. - - `src/flowx/parser/expression_parser.py` — added `_resolve_pipeline_global_param`, `_resolve_linked_service_param`, `_resolve_item_safe_nav`, `_unwrap_noop_call`; widened `_FUNCTION_CALL_RE` to accept trailing whitespace. - - `src/flowx/translator/engine.py` — thread `definitions.global_parameters` into the seeded `TranslationContext`. -- **Tests added**: 5 new test classes in `tests/unit/test_expression_parser.py` covering global parameters (literal substitution + dict-shape + missing fallback + concat reduction), no-op wrappers (json/string/array), trailing-whitespace function calls, item safe-nav, and linked-service parameter resolution. -- **Commit**: `2cd8929` - -### Change: `linked-service-parameter-resolution` (P0) - -- **Title**: Resolve `@linkedService().X` against activity-supplied parameters with LS defaultValue fallback. -- **Rationale**: NB-4 and LSC-001 — `AdfLinkedServiceReference` dropped the activity's `parameters` dict at parse time. 166/327 emitted databricks.yml files had `spark_version: '@linkedService().clusterVersion'` and failed to deploy. Also coerces `num_workers='1'` strings to int. -- **Files changed**: - - `src/flowx/models/adf_ast.py` — added `AdfLinkedServiceReference.parameters` field. - - `src/flowx/parser/adf_loader.py` — populate `parameters` from `linkedServiceName.parameters`. - - `src/flowx/translator/engine.py` — added `_resolve_ls_parameters`, `_substitute_ls_params`, `_coerce_int`; `_extract_cluster_config` now accepts `ls_param_overrides` and walks string values through `_substitute_ls_params` before reading fields. -- **Tests added**: `TestCommonAttributes.test_linked_service_parameter_overrides_cluster_version` in `tests/unit/test_translators.py`. -- **Commit**: `e33ca26` - -### Change: `linked-service-cluster-field-coverage` (P1) - -- **Title**: Extend `_extract_cluster_config` to lift `spark_env_vars`, `custom_tags`, `driver_node_type_id`, `init_scripts`, `data_security_mode`, `cluster_log_conf`; propagate to emitted clusters. -- **Rationale**: NB-3, LSC-003 — five LS keys lost; default-cluster builder never consulted per-task cluster. -- **Files changed**: - - `src/flowx/translator/engine.py` — `_extract_cluster_config` now lifts the extended fields. - - `src/flowx/bundler/dab_writer.py` — added `_infer_bundle_cluster_extras`; `_build_default_cluster` now accepts extras and merges them into `new_cluster`. -- **Tests added**: `TestCommonAttributes.test_extended_cluster_fields_propagated` in `tests/unit/test_translators.py`; `TestClusterExtrasPropagation.test_extras_merged_into_default_cluster` in `tests/unit/test_bundler.py`. -- **Commit**: `b033123` - -### Change: `library-resolution-and-stub-binding` (P0) - -- **Title**: Resolve library jar/whl/maven/pypi expressions; bind stub notebooks with jar libraries to a real cluster. -- **Rationale**: NB-1 + LSC-005 + NB-2 — library jar paths shipped as literal `@concat(...)`; stub tasks with libraries were skipped during cluster binding, so the Jobs API rejected them. -- **Files changed**: - - `src/flowx/translator/activity_translators/notebook.py` — added `_resolve_libraries` that walks each library entry through `resolve_expression`. - - `src/flowx/bundler/dab_writer.py` — `_bind_cluster_to_notebook_tasks` now binds the default cluster for any stub or serverless-mode task that ships jar/whl/maven/pypi libraries. -- **Tests added**: 2 tests in `tests/unit/test_translators.py` (library resolution + global parameter substitution); 3 tests in `tests/unit/test_bundler.py` under `TestStubLibraryBinding`. -- **Commit**: `9b1af19` - -### Change: `base-parameters-cleanup-and-stub-widgets` (P1) - -- **Title**: Strip unresolved ADF expressions from stub notebook base_parameters. -- **Rationale**: NB-5 — stub notebook tasks shipped raw `@string(coalesce(...))` strings as widget defaults; `dbutils.widgets.get` returns them verbatim, which fails at runtime. -- **Files changed**: - - `src/flowx/bundler/dab_writer.py` — dropped the `notebook_path.startswith('/')` gate on `_extract_manual_parameters_from_existing_notebook_tasks` so both absolute-path and bundle-relative stubs are walked. -- **Tests added**: `TestStubBaseParameterCleanup.test_stub_notebook_strips_unresolvable_adf_expression` in `tests/unit/test_bundler.py`. -- **Commit**: `e1ba336` - -### Change: `lookup-file-dataset-support` (P0) - -- **Title**: Resolve Lookup `typeProperties.dataset` and emit `spark.read` for file-source lookups instead of `spark.sql('')`. -- **Rationale**: Three gaps in the lookup dimension share root cause — translator never read `typeProperties.dataset`, code_generator emitted `spark.sql('')`, multiline JSON handling was missing. -- **Files changed**: - - `src/flowx/translator/activity_translators/lookup.py` — resolve `typeProperties.dataset` / `activity.inputs` to the bound `AdfDataset`; surface `dataset_type`, `container`, `folder_path`, `file_name`, `multiLineJson` onto `source_properties` for file-source dataset types. - - `src/flowx/preparer/code_generator.py` — added `_is_file_lookup`, `_file_lookup_body`; the existing JDBC / non-DB branches now fall through to the file-source branch when the dataset is file-shaped. -- **Tests added**: `TestLookupTranslator.test_translate_lookup_resolves_json_file_dataset` in `tests/unit/test_translators.py`; `TestGenerateLookupNotebook.test_file_source_lookup_emits_spark_read` in `tests/unit/test_code_generator.py`. -- **Commit**: `3b669f6` - -### Change: `foreach-inner-extra-tasks` (P0) - -- **Title**: Carry inner-activity `extra_tasks` through ForEach inner-job assembly so IfCondition/Switch branches survive. -- **Rationale**: CF-001 — for_each preparer's multi-child path appended only `child_prepared.task`, dropping `extra_tasks` where IfCondition/Switch branch bodies live. 12 pipelines lost branches. -- **Files changed**: - - `src/flowx/preparer/activity_preparers/for_each.py` — multi-child path now `.extend(child_prepared.extra_tasks)`; single-child path escalates to the sub-job pattern when the sole child contributes extra_tasks. -- **Tests added**: 2 tests in `tests/unit/test_preparers.py` under `TestForEachPreparer` (multi-child IfCondition; single-child IfCondition escalation). -- **Commit**: `8eceaea` - -### Change: `dependency-multi-condition-mapping` (P1) - -- **Title**: Map ADF `dependencyConditions` lists to the correct DAB `run_if` value instead of picking the first. -- **Rationale**: CF-004 — `[Succeeded, Failed]` (= "run regardless") was reduced to `Succeeded`, silently reversing semantics for log/error-handler tasks. -- **Files changed**: - - `src/flowx/translator/engine.py` — added `_map_dependency_conditions` helper that reduces lists per documented mapping rules; `_build_base_kwargs` uses it. -- **Tests added**: `TestCommonAttributes.test_dependency_multi_condition_succeeded_and_failed_maps_to_completed` in `tests/unit/test_translators.py`. -- **Commit**: `fe37f9f` - -### Change: `expression-resolver-bool-and-numeric-coercion` (P1) - -- **Title**: Coerce LS / parameter values during resolution — stringified booleans to YAML booleans, numeric strings to ints where the cluster spec demands int. -- **Rationale**: VAR-006 — bool default `false` ends up as string `'False'`; num_workers `'1'` as string. The numeric coercion was already part of change 2; this change adds the bool/int/float pipeline-parameter coercion. -- **Files changed**: - - `src/flowx/translator/engine.py` — added `_coerce_parameter_default`; pipeline-parameter entries now carry `type` + properly-typed default. - - `src/flowx/bundler/dab_writer.py` — `pipeline_dict_to_ir` honours non-string defaults verbatim (no string-coercion when value is already bool/int/float). -- **Tests added**: 3 tests in `tests/unit/test_translators.py` covering bool/int/string coercion. -- **Commit**: `5f75b01` - -### Change: `pipeline-parameters-and-variables-round-trip` (P0, partial) - -- **Title**: Round-trip `pipeline.parameters` through `translation_report`. -- **Rationale**: VAR-001 — `_load_report`'s aggregated branch dropped pipeline parameters, so 329/340 bundles referenced `{{job.parameters.X}}` without declaring them. -- **Scope note**: Implemented only the parameter round-trip via the aggregator. Variable-init bridge tasks (VAR-004) and full variable-cache plumbing not implemented this iteration. -- **Files changed**: - - `src/flowx/bundler/dab_writer.py` — aggregator now reads `translation.parameters` and `ir.parameters`, threads them through `pipeline_dict_to_workflow`. -- **Tests added**: `TestAggregatedReportPipelineParameters.test_load_report_carries_pipeline_parameters` in `tests/unit/test_bundler.py`. -- **Commit**: `1aa3322` - -## Deferred - -The following P0/P1 plan items were not addressed in this iteration. They each require broader cross-file refactors than was safe inside a single iteration with the time budget available. - -- **`condition-and-switch-expression-bridge` (P0)** — bridging IfCondition/Switch expressions through synthetic SetVariable tasks requires changes across the if_condition / switch translators **and** preparers **and** execute_pipeline preparer (5 files), plus the inner-task wiring for `run_job_task.job_parameters`. The transformation is non-trivial and risks regressions in existing condition_task / case_task tests. Plan for next iteration: implement behind a feature flag, then turn on after corpus validation. -- **`triggers-to-schedules` (P0)** — end-to-end schedule support touches engine + workflow_preparer + dab_writer for trigger ingestion, N:M binding, parameter overrides, and Tumbling/BlobEvents/Custom routing to SETUP.md. Plan for next iteration: land in a single focused PR. -- **`credential-and-secret-recognition` (P0)** — AzureKeyVaultSecret + AzureBlobFS CredentialReference + MSI auth across 4 files. Plan for next iteration: tackle one credential family per commit. -- **Variable defaults & init bridges (partial — half of `pipeline-parameters-and-variables-round-trip`)** — VAR-004 / VAR-006 (variable defaultValue dropped + parameter type dropped) need the variable-init bridge task pattern, which interlocks with the condition/switch bridge work above. - -## Summary - -- 10 commits on `fix-0603` (this section's 9 changes + one for partial round-trip) -- 528 unit tests pass after the final commit (baseline: 499 — net 29 new tests) -- No tests broken; no `--no-verify` or `--amend` used. - -## Iteration 2 — 2026-06-03 - -Implemented 11 of 13 plan changes (C-13 folded into C-07 since they share the same surface; one residual cleanup deferred — see notes below). - -### C-01 — Collapse `@concat()` to a literal when every part resolves to a literal (P0) - -- **Rationale**: NB-ITER2-1 / LSC2-004. `_resolve_concat` / `_handle_concat` always emitted `kind='notebook_code'`. Library jar paths whose concat parts all resolve to literals shipped as Python source instead of strings; `notebook._resolve_libraries` only inlines literals, so installs silently broke in 185 bundles. -- **Files**: `src/flowx/parser/expression_parser.py`, `tests/unit/test_expression_parser.py`, `tests/unit/test_translators.py`. -- **Tests**: `TestConcat.test_concat_literals`, `TestConcat.test_concat_collapses_when_all_parts_resolve_to_literals`, `TestNotebookTranslator.test_translate_notebook_resolves_library_with_globals` (updated to assert literal jar path). -- **Commit**: `cb5ddb7`. - -### C-02 — Unwrap `{value, type:'Expression'}` dicts in LS params and cluster fields (P0) - -- **Rationale**: NB-ITER2-2 / LSC2-003. `_substitute_ls_params` and `_extract_cluster_config` preserved the ADF expression dict-wrapper shape; `custom_tags` emitted `{DigitalCase: {value: APP0001, type: Expression}}` (166 bundles) which is invalid for Databricks `Map[String,String]`. -- **Files**: `src/flowx/translator/engine.py`, `tests/unit/test_translators.py`. -- **Tests**: `TestCommonAttributes.test_ls_param_expression_wrapper_unwrapped_in_custom_tags`, `TestCommonAttributes.test_ls_param_resolved_against_factory_global_parameters` (covers C-03 too). -- **Commit**: `267b08d` (also covers C-03 since both modifications share `_resolve_ls_parameters`). - -### C-03 — Run activity-supplied LS parameter values through `resolve_expression` with factory globals (P0) - -- **Rationale**: NB-ITER2-3 / LSC2-002. Activities passing `clusterVersion: {value:'@pipeline().globalParameters.clusterVersion', type:'Expression'}` left the raw expression in 50 IR JSONs and 4 bundles. `_resolve_ls_parameters` now threads the translation context so `@pipeline().globalParameters.X` collapses to its factory value. -- **Files**: `src/flowx/translator/engine.py`, `tests/unit/test_translators.py`. -- **Tests**: `TestCommonAttributes.test_ls_param_resolved_against_factory_global_parameters`. -- **Commit**: folded into `267b08d`. - -### C-04 — Walk nested activities when collecting workflow cluster hints (P0) - -- **Rationale**: NB-ITER2-4 / LSC2-001. `prepare_workflow` iterated only `pipeline.tasks`. Pipelines whose only `DatabricksNotebook` lived inside an `IfCondition` / `Switch` / `ForEach` branch shipped the default `Standard_DS3_v2 / 15.4.x` cluster fallback in 26 bundles. New `_iter_activity_with_descendants` BFS-walks each top-level activity. -- **Files**: `src/flowx/preparer/workflow_preparer.py`, `tests/unit/test_preparers.py`. -- **Tests**: `TestPrepareWorkflow.test_prepare_workflow_collects_cluster_hints_from_nested_activities`, `TestPrepareWorkflow.test_prepare_workflow_collects_cluster_hints_from_switch_default_branch`. -- **Commit**: `84b491a`. - -### C-05 — Synthesise init SetVariable tasks for variables with defaultValue (P0) - -- **Rationale**: VAREX-002. Pipeline variables carrying `defaultValue` (164 in corpus, 127 expression-typed) were never materialised as IR tasks, so `_resolve_variable`'s self-referential fallback produced 333 dangling `{{tasks.X.values.X}}` refs. `translate_pipeline` now prepends a `_init_` SetVariableActivity per default-valued variable and registers the synth task in `variable_cache`. `_resolve_variable` additionally returns `None` when no setter is known to surface unresolved variables as raw expressions instead of placeholders. -- **Files**: `src/flowx/translator/engine.py`, `src/flowx/parser/expression_parser.py`, `tests/unit/test_translators.py`, `tests/unit/test_preparers.py` (Switch unresolved-variable assertion updated), `tests/unit/test_expression_parser.py` (rename `test_variable_fallback_to_name` → `test_variable_returns_none_when_no_setter`). -- **Tests**: `TestVariableInitTasks.test_default_valued_variable_yields_init_task`, `TestVariableInitTasks.test_default_valued_variable_with_concat_expression`. -- **Commit**: `d80c665`. - -### C-06 — Route ForEach inner-job `@variables()` refs via task-value, not undeclared job parameter (P0) - -- **Rationale**: VAREX-004. `collect_inner_job_params` previously declared `variables()` refs as inner-job parameters and mapped them to `{{job.parameters.}}` on the parent. Parent never declared those names so the inner job received an empty string. When a parent setter task_key is known, variables now route through `{{tasks..values.}}` and are NOT declared on the inner-job parameter list. -- **Files**: `src/flowx/bundler/inner_job_params.py`, `src/flowx/preparer/activity_preparers/for_each.py`, `src/flowx/preparer/workflow_preparer.py`, `tests/unit/test_for_each_inner_job_params.py` (new). -- **Tests**: 4 new tests in `tests/unit/test_for_each_inner_job_params.py`. -- **Commit**: `c119ab3`. - -### C-07 — Bridge IfCondition / Switch condition_task operands through a hidden SetVariable task (P0) - -- **Rationale**: CF-iter2-001 / CF-iter2-003 / VAREX-003. Per Databricks docs, `condition_task.left` / `.right` must be literal / job-parameter / task-value / task-parameter refs. Operands carrying ADF function calls (`@toUpper`, `@coalesce`, `@and`, `@or`, `@not`, `@empty`, `@concat`, ...) shipped as raw ADF expressions in 84 bundles. New `lower_to_bridge` / `merge_bridge_requests` in `activity_translators/resolve.py` turn `notebook_code` resolver results into a `BridgeRequest` that the IfCondition / Switch translators stash on the IR; the corresponding preparers synthesise a bridge notebook task and rewrite the operand to the task-value reference. Dropped the legacy `NOT_EQUAL '0'` fallback when a bridge succeeds. -- **Files**: `src/flowx/models/ir.py` (new bridge_notebook_* fields on IfCondition/Switch), `src/flowx/translator/activity_translators/resolve.py`, `src/flowx/translator/activity_translators/if_condition.py`, `src/flowx/translator/activity_translators/switch.py`, `src/flowx/preparer/activity_preparers/if_condition.py`, `src/flowx/preparer/activity_preparers/switch.py`, `src/flowx/translator/engine.py` (serialise bridge fields). -- **Tests**: `TestIfConditionTranslator.test_translate_if_condition_empty_bridges_via_notebook`, `TestIfConditionPreparer.test_prepare_if_condition_emits_bridge_task`, `TestSwitchTranslator.test_translate_switch_function_call_routes_through_bridge`, plus `TestSwitchPreparer.test_resolve_switch_on_expression_is_idempotent_for_dab_refs` (covers C-13). -- **Commit**: `7494fde`. - -### C-08 — Bridge ForEach `for_each_task.inputs` through a seed task when items expression is notebook_code (P0) - -- **Rationale**: CF-iter2-002. Per docs, `inputs` accepts a JSON array literal or `{{tasks.X.values.Y}}` or `{{job.parameters.X}}`; `@split(...)` was rejected. 12 bundles emitted `inputs: '@split(...)'` verbatim. New `_resolve_for_each_inputs_with_bridge` emits a seed notebook task that computes the array and rewrites `inputs` to its task value. -- **Files**: `src/flowx/preparer/activity_preparers/for_each.py`, `tests/unit/test_preparers.py`. -- **Tests**: `TestForEachPreparer.test_prepare_for_each_bridges_split_items_via_seed_task`. -- **Commit**: `92036a8`. - -### C-09 — ExecutePipeline parameter resolution refuses notebook_code result kinds (P0) - -- **Rationale**: VAREX-001. 62 bundles had ExecutePipeline parameter values like `'json: ' + dbutils.widgets.get('configFile')` because `resolve.py` accepted any result.kind. The sub-job's widget received the Python source text. The translator now resolves each parameter through `resolve_expression`, accepts only literal / dab_ref results, and surfaces `notebook_code` results as `parameter_approximations` for SETUP.md. -- **Files**: `src/flowx/translator/activity_translators/execute_pipeline.py`, `tests/unit/test_translators.py`. -- **Tests**: `TestExecutePipelineTranslator.test_translate_execute_pipeline_drops_notebook_code_parameters`. -- **Commit**: `34e0bd9`. - -### C-10 — Compile AdfTrigger objects into Pipeline.schedule and emit DAB schedule / trigger blocks (P0) - -- **Rationale**: SCHED-001. `definitions.triggers` was parsed but never read; `Pipeline.schedule` was always `None`; 327 bundles shipped without any schedule metadata. `translate_pipeline` now matches triggers by their `pipelineReference` and compiles the first matching one into a structured schedule dict (`ScheduleTrigger.recurrence` → quartz cron + timezone; `BlobEventsTrigger` → `trigger.file_arrival`; Tumbling/Custom → manual-setup hints). Windows timezone names (`Romance Standard Time`) normalise to IANA (`Europe/Madrid`). `runtimeState='Stopped'` flips `pause_status` to `PAUSED`. `PreparedWorkflow.schedule` and `pipeline_dict_to_ir` thread the spec to the DAB writer. -- **Files**: `src/flowx/preparer/workflow_preparer.py`, `src/flowx/translator/engine.py`, `src/flowx/bundler/dab_writer.py`, `tests/unit/test_translators.py`, `tests/unit/test_bundler.py`. -- **Tests**: 7 new tests in `TestScheduleCompilation` covering Daily / Weekly / Tumbling / Blob / Custom triggers, timezone normalisation, and pause-status; 2 new bundler tests (`TestScheduleEmission`) verifying the schedule / trigger blocks land in the rendered YAML. -- **Commit**: `bc10f5b`. - -### C-11 — Emit per-secret SecretInstructions from Web activity auth payloads (P1) - -- **Rationale**: LSC2-005. `web_activity.prepare` always emitted a single `auth-credential` SecretInstruction regardless of the underlying ADF auth shape. AzureKeyVaultSecret payloads carry `store.referenceName` and `secretName` but both were dropped; CredentialReference (managed identity) was wrongly treated as a static secret. The preparer now walks nested fields (`password` / `secret` / `clientSecret` / `pfx` / `key`), materialises per-Key Vault SecretInstructions, and routes CredentialReference / MSI payloads to a `manual_credential` SetupTask so SETUP.md flags them. -- **Files**: `src/flowx/preparer/activity_preparers/web_activity.py`, `tests/unit/test_preparers.py`. -- **Tests**: `TestWebActivityPreparer.test_prepare_web_activity_key_vault_secret_uses_vault_scope_and_secret_name`, `TestWebActivityPreparer.test_prepare_web_activity_credential_reference_emits_setup_note`. -- **Commit**: `bce740d`. - -### C-12 — Extend `_strip_dangling_task_value_refs` to cover run_job_task and condition_task fields (P1) - -- **Rationale**: VAREX-005. `_strip_dangling_task_value_refs` only walked `notebook_task.base_parameters`. 333 dangling `{{tasks.X.values.Y}}` refs survived into resource YAMLs because cross-job (`run_job_task.job_parameters`) and control-flow (`condition_task.left/.right`) surfaces weren't covered. The walker now also visits these fields and blanks dangling refs so SETUP.md §4 can flag them. -- **Files**: `src/flowx/bundler/dab_writer.py`, `tests/unit/test_bundler.py`. -- **Tests**: `TestStripDanglingTaskValueRefs.test_strips_dangling_run_job_task_job_parameters`, `TestStripDanglingTaskValueRefs.test_strips_dangling_condition_task_operands`, `TestStripDanglingTaskValueRefs.test_recurses_into_for_each_task_body`. -- **Commit**: `836a45d`. - -### C-13 — Idempotent Switch on-expression resolver (P1) - -- **Rationale**: CF-iter2-004. `preparer/activity_preparers/switch.py::resolve_switch_on_expression` constructed an empty `TranslationContext()` and re-resolved on-expression on the JSON-reload path, destructively stripping refs the translator had already lowered. Updated to pass through anything already containing `{{...}}` or a translator-side `__BRIDGE__::` placeholder; only bare `@`-prefixed expressions are re-resolved. -- **Files**: covered in C-07 commit (`src/flowx/preparer/activity_preparers/switch.py`). -- **Tests**: `TestSwitchPreparer.test_resolve_switch_on_expression_is_idempotent_for_dab_refs`. -- **Commit**: folded into `7494fde`. - -### Skipped / partial - -- The plan's C-13 acceptance check additionally calls for `grep -nE 'TranslationContext\(\s*\)' src/flowx/preparer/` returning zero results. The current code still constructs bare `TranslationContext()` in ~12 sites across `preparer/code_generator.py`, `preparer/activity_preparers/{execute_pipeline,for_each,switch,databricks_job,filter,notebook,helpers}.py`. The switch fix is the highest-impact one (it was the only example called out in the rationale); the remaining sites operate at the preparer layer where global parameters and `variable_cache` aren't readily available, so converting them to thread a typed context is a larger refactor. Deferred to a follow-on iteration. - -## Summary (iteration 2) - -- 11 new commits on `fix-0603` (C-01 .. C-12, with C-13 folded into C-07) -- 559 unit tests pass after the final iteration-2 commit (iteration-1 baseline: 528 — net 31 new tests) -- No tests broken; no `--no-verify` or `--amend` used. - -## Iteration 3 — 2026-06-03 / 2026-06-04 - -Implemented all 18 plan items across 15 commits (C-13 .. C-27). Three plan items overlapped on single edits and were bundled: C-13 folds three resolver-widening fixes; C-19 folds two ForEach inner-workflow fixes; C-21 fixes the same bool-stringification bug at three call sites. - -### C-13 — Propagate globals into ForEach child context; widen LS/library resolvers to accept dab_ref (P0) - -- **Rationale**: NB-ITER3-001 / CF3-002 / LSC3-004 (ForEach child context drops global_parameters / linked_service_parameters) + NB-ITER3-002 / LSC3-003 / VAREX3-006 (`_resolve_ls_parameters` refuses `dab_ref`) + NB-ITER3-004 (notebook `_resolve_libraries` refuses `dab_ref`). -- **Files**: `src/flowx/translator/activity_translators/for_each.py`, `src/flowx/translator/engine.py`, `src/flowx/translator/activity_translators/notebook.py`. -- **Commit**: `504000c`. - -### C-14 — Preserve bridge_notebook_code/imports/required_parameters on IR roundtrip (P0) - -- **Rationale**: CF3-001 / VAREX3-001. `_reconstruct_ir` dropped bridge fields, leaving 109 bundles shipping `left: __BRIDGE__::result` with no actual `_bridge` task. -- **Files**: `src/flowx/bundler/dab_writer.py`, `tests/unit/test_bundler.py`. -- **Commit**: `a1955e4`. - -### C-15 — IfCondition emits right='False' (not '' or '0') against bridge task values (P0) - -- **Rationale**: CF3-003 / VAREX3-004. 76 bundles emitted `right: ''` and 19 emitted `right: '0'` from `@not(...)` / legacy truthy fallback paths; neither compares correctly against Python `'True'/'False'`. -- **Files**: `src/flowx/translator/activity_translators/if_condition.py`, `tests/unit/test_translators.py`. -- **Commit**: `a643a8c`. - -### C-16 — Lower single-segment item()?.X safe-nav to notebook_code (P0) - -- **Rationale**: CF3-005 / VAREX3-005. `expression_parser._resolve_item_safe_nav` returned None when `len(parts) < 2`, blocking 4 Switch on-expressions and SetVariable expressions from bridge lowering. -- **Files**: `src/flowx/parser/expression_parser.py`, `tests/unit/test_expression_parser.py`. -- **Commit**: `8f7d6d6`. - -### C-17 — Emit single_user_name alongside data_security_mode: SINGLE_USER (P0) - -- **Rationale**: NB-ITER3-003. 189 bundles failed `databricks bundle validate` with "single_user_name must be set when data_security_mode is SINGLE_USER". Three cluster builders set SINGLE_USER mode without the name. Use `${workspace.current_user.userName}` as the closest deployable analog of ADF's MSI auth. -- **Files**: `src/flowx/bundler/dab_writer.py`. -- **Tests**: `TestSingleUserNameOnSingleUserClusters` (3 cases) in `tests/unit/test_bundler.py`. -- **Commit**: `b41cc24`. - -### C-18 — Carry schedule through aggregated translations report into pipeline_dict (P0) - -- **Rationale**: SCHED3-001. `_load_report`'s aggregated branch built pipeline_dict from `{name, tasks, parameters}` and never copied `schedule`. 0 of 327 bundles contained `quartz_cron_expression` despite 8 trigger-referenced pipelines having populated schedule blocks. -- **Files**: `src/flowx/bundler/dab_writer.py`. -- **Tests**: `TestAggregatedReportSchedule` (2 cases) in `tests/unit/test_bundler.py`. -- **Commit**: `5666708`. - -### C-19 — Propagate cluster_hints + variable_task_keys into ForEach inner workflows (P0 + P1) - -- **Rationale**: LSC3-001 — inner-job PreparedWorkflow drops cluster hints from nested activities, so inner-job YAMLs ship the bundle default `Standard_DS3_v2 / 15.4.x` even when the parent default cluster carries LS-derived `spark_env_vars / custom_tags / driver_node_type_id`. CF3-006 — multi-child `collect_inner_job_params` call dropped the `variable_task_keys` kwarg that the single-child path threaded. -- **Files**: `src/flowx/preparer/activity_preparers/for_each.py`. -- **Tests**: `TestForEachPreparer.test_for_each_inner_workflow_carries_cluster_hints_from_inner_activity`, `TestForEachPreparer.test_for_each_single_child_inner_workflow_carries_cluster_hints` in `tests/unit/test_preparers.py`; `TestVariableTaskKeysRouting.test_multi_child_for_each_threads_variable_task_keys` in `tests/unit/test_for_each_inner_job_params.py`. -- **Commit**: `ece7f01`. - -### C-20 — Stop emitting fake auth-credential secret for MSI WebActivity auth (P0) - -- **Rationale**: LSC3-002. MSI / ManagedServiceIdentity has no static secret, so reading `auth-credential` from a secret scope shipped a broken bearer-token call against a non-existent secret in 14 generated notebooks across 5 source pipelines. The notebook now raises `NotImplementedError` pointing at SETUP.md. ServicePrincipal auth keeps the secret-based flow. -- **Files**: `src/flowx/preparer/code_generator.py`. -- **Tests**: `TestGenerateWebActivityNotebook.test_auth_block_msi_raises_not_implemented` in `tests/unit/test_code_generator.py`. -- **Commit**: `5f2c0eb`. - -### C-21 — Render Boolean variable values as lowercase true/false (P0) - -- **Rationale**: VAREX3-002. Python title-case `'True'/'False'` silently inverted ADF Boolean comparisons like `@equals(variables('continue'), true)` across 28 occurrences. Fix at three call sites: `resolve_expression()` bool literal path (root cause), `_build_variable_init_activities` fallback, and `set_variable.py` translator fallback. -- **Files**: `src/flowx/translator/engine.py`, `src/flowx/translator/activity_translators/set_variable.py`, `src/flowx/parser/expression_parser.py`. -- **Tests**: `TestVariableInitTasks.test_default_valued_boolean_variable_renders_lowercase`, `TestVariableInitTasks.test_set_variable_with_raw_bool_value_renders_lowercase` in `tests/unit/test_translators.py`; updated `TestLiterals.test_boolean` in `tests/unit/test_expression_parser.py`; updated `test_boolean_value` in `tests/unit/test_resolve_field.py`. -- **Commit**: `e7c35d6`. - -### C-22 — Case-insensitive dataset / linked service lookup + LS URL threading (P1) - -- **Rationale**: LSC3-005. ADF identifiers are documented as case-insensitive, but the loader keys dicts by original case; 2 generated lookup notebooks shipped `spark.sql('')` because a lowercase reference didn't match a mixed-case dataset filename. Also threads `linked_service.typeProperties.url` so the file lookup body assembles a fully-qualified `abfss://...` widget default. -- **Files**: `src/flowx/models/adf_ast.py` (added `get_dataset` / `get_linked_service`), `src/flowx/translator/activity_translators/lookup.py`, `src/flowx/preparer/code_generator.py`. -- **Tests**: `TestLookupCaseInsensitiveAndLinkedService` (2 cases) in `tests/unit/test_translators.py`. -- **Commit**: `b9ec3d6`. - -### C-23 — Resolve attribute chains on function-call results (P1) - -- **Rationale**: CF3-004. `resolve_expression('@toUpper(json(pipeline().parameters.items).type)')` returned None because the bare function dispatcher only matches when the function call is the outermost token. New `_resolve_function_call_with_attribute` helper detects `funcName(args).attr.attr...`, resolves the function call, then chains `.get('')` for each segment. -- **Files**: `src/flowx/parser/expression_parser.py`. -- **Tests**: `TestFunctionCallWithAttribute` (2 cases) in `tests/unit/test_expression_parser.py`. -- **Commit**: `696941d`. - -### C-24 — Emit trigger.periodic for Day/Week/Month recurrence with interval > 1 (P1) - -- **Rationale**: SCHED3-002. `_recurrence_to_quartz_cron` silently dropped `interval` for Day / Week / Month, producing cron that fired every day/week/month rather than every Nth. quartz cron cannot represent "every N days/weeks/months" without enumeration; DAB `trigger.periodic` takes `{interval, unit}` directly. -- **Files**: `src/flowx/translator/engine.py`, `src/flowx/bundler/dab_writer.py`. -- **Tests**: `TestScheduleCompilation.test_schedule_trigger_interval_3_days_emits_periodic`, `test_schedule_trigger_interval_2_weeks_emits_periodic`, `test_schedule_trigger_interval_1_day_still_cron` in `tests/unit/test_translators.py`; `TestScheduleEmission.test_periodic_trigger_emitted` in `tests/unit/test_bundler.py`. -- **Commit**: `563f3e8`. - -### C-25 — Carry pipelineReference.parameters from triggers into job parameter defaults (P1) - -- **Rationale**: SCHED3-003. `_compile_pipeline_schedule` read only `pipelineReference` from each trigger entry and silently dropped per-pipeline `parameters` overrides like `{applicationName: 'app0001', negocio: 'GLP'}`. Scheduled runs received bare pipeline defaults. New `_extract_trigger_parameter_overrides` attaches the override map as `parameter_overrides` on the schedule spec; `_build_job_resource` mutates matching `job.parameters[*].default` after applying the schedule. -- **Files**: `src/flowx/translator/engine.py`, `src/flowx/bundler/dab_writer.py`. -- **Tests**: `TestScheduleCompilation.test_trigger_carries_per_pipeline_parameter_overrides` in `tests/unit/test_translators.py`; `TestScheduleEmission.test_trigger_parameter_overrides_mutate_job_parameter_defaults` in `tests/unit/test_bundler.py`. -- **Commit**: `4b2eba0`. - -### C-26 — Emit manual_variable_rollup SetupTask for cross-ForEach variable reads (P1) - -- **Rationale**: VAREX3-003. When a SetVariable for `X` lives only inside a ForEach inner-job and a sibling task reads `@variables('X')`, the read gets the stale init value because task values cannot cross `run_job_task` boundaries. Minimum-viable fix: detect the pattern in `prepare_workflow`, emit a SetupTask of type `manual_variable_rollup`, surface it in SETUP.md so the user adds a roll-up notebook before the sibling runs. -- **Files**: `src/flowx/preparer/workflow_preparer.py`, `src/flowx/bundler/prereqs_writer.py`, `src/flowx/bundler/dab_writer.py`. -- **Tests**: `TestCrossForEachVariableReadDetection` (2 cases) in `tests/unit/test_preparers.py`; `TestManualVariableRollupSetupMd` in `tests/unit/test_bundler.py`. -- **Commit**: `87bf35e`. - -### C-27 — Union scan_notebooks_for_secrets with workflow.secrets in SETUP.md (P1) - -- **Rationale**: LSC3-006. `scan_notebooks_for_secrets` walked notebooks (yielding the MSI fake `auth-credential` before C-20) while `create_secrets.py` was written from `workflow.secrets` (real AKV scopes). SETUP.md Option A vs Option B disagreed. `build_prereqs` now accepts `secret_instructions`, unions them with the notebook scan (dedupe by (scope, key)) so both options stay in sync. -- **Files**: `src/flowx/bundler/prereqs_writer.py`, `src/flowx/bundler/dab_writer.py`. -- **Tests**: `TestSecretsUnion` (2 cases) in `tests/unit/test_prereqs_writer.py` (new file). -- **Commit**: `b64cc5a`. - -## Summary (iteration 3) - -- 15 new commits on `fix-0603` (C-13 .. C-27) -- 595 unit tests pass after the final iteration-3 commit (iteration-2 baseline: 559 — net 36 new tests) -- No tests broken; no `--no-verify` or `--amend` used. -- All 18 P0/P1 plan items implemented end-to-end. C-13, C-19, and C-21 each fold 2-3 closely-related plan items into a single commit since the underlying fix is the same edit (no scope creep). - -## Iteration 4 — 2026-06-04 - -Implemented all 12 plan items across 11 commits. C-28 + C-30 are bundled -into one commit since they share NotebookActivity IR fields and the -preparer infrastructure. - -### C-28 + C-30 — Dynamic notebookPath dispatch stub + unresolved library SetupTask (P0 + P1) - -- **Rationale**: NB-ITER4-001 — the notebook translator passed `notebookPath` - through `resolve_field` so a `notebook_code` result (e.g. - `@trim(json(...).notebook_path)`) shipped as the workspace path. - NB-ITER4-003 — `_resolve_libraries` silently passed through unresolved jar - paths so the cluster tried to install a file literally named - `@concat(...)`. -- **Files**: `src/flowx/translator/activity_translators/notebook.py`, - `src/flowx/models/ir.py`, - `src/flowx/preparer/activity_preparers/notebook.py`, - `src/flowx/bundler/prereqs_writer.py`, - `src/flowx/bundler/dab_writer.py`, - `src/flowx/translator/engine.py`. -- **Tests**: `test_translate_notebook_dynamic_path_marks_unresolved`, - `test_translate_notebook_unresolved_library_captured` in - `tests/unit/test_translators.py`; - `test_prepare_notebook_dispatch_stub_for_unresolved_path`, - `test_prepare_notebook_emits_unresolved_library_setup_task` in - `tests/unit/test_preparers.py`. -- **Commit**: `27ed860`. - -### C-29 — Filter unparseable spark_version / node_type_id from cluster_hints (P0) - -- **Rationale**: NB-ITER4-002 — `_infer_bundle_cluster_defaults` picked the - most-common spark_version regardless of whether it parsed as a real DBR - version, so an `@if(equals(item()?.photon,true),...)` expression landed - in `databricks.yml` as the spark_version default and `bundle deploy` - rejected it. -- **Files**: `src/flowx/bundler/dab_writer.py`. -- **Tests**: `TestUnparseableClusterHintsFiltered` (3 cases) in - `tests/unit/test_bundler.py`. -- **Commit**: `a6f9c05`. - -### C-31 — Move ForEach items-expression bridge to translator (P0) - -- **Rationale**: CF4-001 — the for_each preparer constructed a bare - `TranslationContext()` to re-resolve `@split(variables('fecha'),',')`, - but `variable_cache` is empty on the JSON-reload path so the bridge - never fired and DAB rejected the raw @split call as - `for_each_task.inputs`. -- **Files**: `src/flowx/translator/activity_translators/for_each.py`, - `src/flowx/models/ir.py`, - `src/flowx/preparer/activity_preparers/for_each.py`, - `src/flowx/bundler/dab_writer.py`, - `src/flowx/translator/engine.py`. -- **Tests**: `test_prepare_for_each_uses_ir_bridge_for_variable_based_split` - in `tests/unit/test_preparers.py`. -- **Commit**: `d666402`. - -### C-32 — IfCondition truthy fallback Boolean-aware right operand (P1) - -- **Rationale**: CF4-002 — the legacy `right: '0'` fallback against - Boolean variable refs is always-true (post-C-21 SetVariable writes - lowercase `'true'/'false'` strings), so the false branch became - unreachable. -- **Files**: `src/flowx/translator/activity_translators/if_condition.py`. -- **Tests**: - `test_translate_if_condition_boolean_variable_uses_lowercase_false` - in `tests/unit/test_translators.py`. -- **Commit**: `d546a86`. - -### C-33 — SetVariable: lower split[N]/2-arg substring and surface unresolved @-expressions (P1) - -- **Rationale**: Merged VAREX4-001 + CF4-003 — 212 SetVariableActivity - entries shipped raw `@concat(...)` text with `value_kind='literal'` - because `resolve_expression` returned None for nested constructs like - `split(...)[N]` and the 2-arg `substring(x, start)`. Bundles that - remained unresolvable now blank the variable and emit a - `manual_variable_init` SetupTask. -- **Files**: `src/flowx/parser/expression_parser.py`, - `src/flowx/translator/activity_translators/set_variable.py`, - `src/flowx/models/ir.py`, - `src/flowx/preparer/activity_preparers/set_variable.py`, - `src/flowx/translator/engine.py`, - `src/flowx/bundler/dab_writer.py`. -- **Tests**: `test_substring_two_arg_form`, `test_split_with_subscript` - in `tests/unit/test_expression_parser.py`; - `test_translate_set_variable_split_subscript_lowers_to_notebook_code`, - `test_translate_set_variable_unresolved_expression_blanks_value` - in `tests/unit/test_translators.py`. -- **Commit**: `3a68586`. - -### C-34 — Expression parser: preserve quoted-string and Boolean literal types in codegen (P1) - -- **Rationale**: Merged VAREX4-002 + VAREX4-003 — both live in - `_resolve_function_call` / `_arg_to_code`. Quoted args like `'12'` - collapsed to bare tokens (`... == 12`, wrong type) and `'09'` produced - a SyntaxError (leading-zero integer). Boolean tokens `true/false` - emitted Python `True/False` but the SetVariable side serialised - lowercase strings post-C-21. -- **Files**: `src/flowx/parser/expression_parser.py`, - `src/flowx/models/ir.py`. -- **Tests**: `test_equals_quoted_string_emits_repr`, - `test_less_quoted_leading_zero_is_valid_python`, - `test_equals_bool_literal_emits_lowercase_string` in - `tests/unit/test_expression_parser.py`. -- **Commit**: `2ff3c10`. - -### C-35 — Anchor _ITEM_FIELD_RE for chained item().a.b lowering (P1) - -- **Rationale**: CF4-004 — the unanchored `_ITEM_FIELD_RE` matched the - first segment of `item().condition.name` and returned - `{{input.condition}}`, silently dropping `.name`. -- **Files**: `src/flowx/parser/expression_parser.py`. -- **Tests**: `test_item_field_multi_segment_lowers_to_notebook_code` - in `tests/unit/test_expression_parser.py`. -- **Commit**: `2177a11`. - -### C-36 — Preserve hours/minutes/weekDays on periodic schedules (P1) - -- **Rationale**: SCHED4-001 — `_recurrence_to_periodic` dropped - `schedule.minutes/hours/weekDays/monthDays` for Day/Week/Month with - interval > 1, so a schedule declaring "every 3 days at 02:00 UTC" - silently fired at midnight. -- **Files**: `src/flowx/translator/engine.py`, - `src/flowx/preparer/workflow_preparer.py`. -- **Tests**: Extended - `test_schedule_trigger_interval_3_days_emits_periodic` in - `tests/unit/test_translators.py`; - `TestManualScheduleTimeOfDaySetupTask` in - `tests/unit/test_preparers.py`. -- **Commit**: `37845f6`. - -### C-37 — File-Lookup: unwrap expression-dict folder/file + abfss URL rewrite (P0) - -- **Rationale**: Merged LSC4-001 + LSC4-003. LSC4-001 (P0) — folder_path - / file_name shipped as raw expression dicts; `.strip('/')` crashed the - bundler with AttributeError, taking down 4 pipelines. LSC4-003 (P1) — - AzureBlobFS https URLs joined to folder/filename produce notebooks - that can't read the source on a Databricks cluster. -- **Files**: `src/flowx/translator/activity_translators/lookup.py`, - `src/flowx/preparer/code_generator.py`. -- **Tests**: `test_file_lookup_coerces_expression_dict_path_components`, - `test_file_lookup_rewrites_https_to_abfss` in - `tests/unit/test_code_generator.py`. -- **Commit**: `a9e2113`. - -### C-38 — Web activity notebook threads resolved Key Vault scope/key (P0) - -- **Rationale**: LSC4-002 — `generate_web_activity_notebook` hard-coded - `scope=task_key, key='auth-credential'` regardless of what the C-11 - preparer resolved. 11 generated notebooks across multiple pipelines - read the wrong secret at runtime even though the C-11 SecretInstruction - carried the real values. -- **Files**: `src/flowx/preparer/code_generator.py`, - `src/flowx/preparer/activity_preparers/web_activity.py`. -- **Tests**: Extended - `test_prepare_web_activity_key_vault_secret_uses_vault_scope_and_secret_name` - in `tests/unit/test_preparers.py` to assert the rendered notebook - contains the resolved scope and key. -- **Commit**: `72f6211`. - -### C-39 — Emit manual_credential SetupTask for MSI / CredentialReference cluster auth (P1) - -- **Rationale**: LSC4-004 — every default cluster ships - `single_user_name: ${workspace.current_user.userName}` regardless of - source ADF authentication. MSI / CredentialReference workloads now - silently run as the deploying human user with no warning. -- **Files**: `src/flowx/translator/engine.py`, - `src/flowx/preparer/workflow_preparer.py`, - `src/flowx/bundler/prereqs_writer.py` (rendering, landed in C-28 - commit alongside the dispatch-stub SETUP.md surface), - `src/flowx/bundler/dab_writer.py` (config aggregation, landed in - C-28 commit). -- **Tests**: `TestManualCredentialFromMsiLinkedService` in - `tests/unit/test_bundler.py`. -- **Commit**: `d8c99a6`. - -## Summary (iteration 4) - -- 11 new commits on `fix-0603` (C-28..C-39, with C-28+C-30 folded into a - single commit since they share NotebookActivity IR fields and the - preparer infrastructure). -- 616 unit tests pass after the final iteration-4 commit (iteration-3 - baseline: 595 — net 21 new tests). -- No tests broken; no `--no-verify` or `--amend` used. -- All 12 P0/P1 plan items implemented end-to-end. P2 items - (NB-ITER4-004, SCHED4-002) intentionally excluded per scoping rules. - -## Iteration 5 — 2026-06-04 - -Implemented all 8 P0/P1 gaps plus the trivially-related P2 (CF5-002, folded -into C-43) across 8 commits (C-40..C-47). C-40..C-42 were committed in an -earlier pass; C-43..C-47 complete the iteration. - -### C-40 — Mine num_workers into the default job_cluster instead of hardcoding 1 (P1) - -- **Rationale**: `_infer_bundle_cluster_extras` omitted `num_workers` and - `_build_default_cluster` hardcoded `num_workers: 1`, even though - `workflow_preparer` stores the full cluster dict (with num_workers) into - `cluster_hints` and the IR carries num_workers != 1 for 122 tasks across - 40 pipelines. ADF clusters with 2-4 workers deployed as 1-worker - clusters with no warning. -- **Files**: `src/flowx/bundler/dab_writer.py`. -- **Tests**: num_workers=2 cluster_hint -> emitted default_cluster - new_cluster.num_workers == 2, in `tests/unit/test_bundler.py`. -- **Commit**: `038888b`. - -### C-41 — IfCondition on a literal-seeded Boolean variable emits right:'false' not '0' (P1) - -- **Rationale**: `_operand_is_known_boolean` only checked - `get_variable_dab_ref`, which reads `variable_value_cache`; that cache is - populated only when value_kind == 'dab_ref', so default-valued Boolean - variables seeded via `_build_variable_init_activities` were never - recognized as Boolean. The fallback emitted NOT_EQUAL(left, '0'), always - true for a 'true'/'false' string, making the false branch dead code. -- **Files**: `src/flowx/models/ir.py`, - `src/flowx/translator/engine.py`, - `src/flowx/translator/activity_translators/if_condition.py`. -- **Tests**: `test_translate_if_condition_boolean_variable_by_declared_type` - in `tests/unit/test_translators.py`. -- **Commit**: `db2a7fb`. - -### C-42 — Resolve Set Pipeline Return Value list-of-pairs inner expression (P1) - -- **Rationale**: `set_variable.py` recognized only str and - `{type:Expression}` shapes; a `pipelineReturnValue` value that is a list - of `{key, value:{type:Expression,content:...}}` pairs failed - `_is_adf_expression` and fell to `str(value_raw)`, then the bundler - blanked it to ''. The inner `@variables('executionOutputs')` is - resolvable in the same IR, so the ref is droppable rather than lost. -- **Files**: `src/flowx/translator/activity_translators/set_variable.py`. -- **Tests**: list-of-pairs value with a resolvable inner `@variables()` ref - asserts value_kind == 'dab_ref', in `tests/unit/test_translators.py`. -- **Commit**: `ec8ca13`. - -### C-43 — Inner-ForEach IfCondition bridges locally; bundler warns when blanking a condition (P1, folds CF5-002) - -- **Rationale**: For an inner IfCondition whose operand resolves to a - parent-job task value (`{{tasks._init_continue.values.continue}}`), the - init task lives only in the parent job. When the ForEach body split into - an inner job, `_strip_dangling_task_value_refs` silently blanked - `condition_task.left/right` to '', making NOT_EQUAL('','0') always TRUE - and running the true branch unconditionally with no SETUP.md signal. - if_condition now recomputes a known-Boolean operand locally via a - BridgeRequest (mirroring the Switch path) when a seeded literal default is - available. `_strip_dangling_task_value_refs` now returns the - (task_key, field, original_ref) tuples it blanks; `write_bundle` threads - them into the Prereqs so SETUP.md gets a 'Conditions neutralized to - always-true' section (folds CF5-002: a predicate is never neutralized - silently). -- **Files**: - `src/flowx/translator/activity_translators/if_condition.py`, - `src/flowx/translator/engine.py`, `src/flowx/models/ir.py`, - `src/flowx/bundler/dab_writer.py`, - `src/flowx/bundler/prereqs_writer.py`. -- **Tests**: - `test_translate_if_condition_boolean_variable_bridges_when_default_literal_known` - in `tests/unit/test_translators.py`; - extended `test_strips_dangling_condition_task_operands` plus - `test_neutralized_condition_renders_setup_section` in - `tests/unit/test_bundler.py`. -- **Commit**: `36158e5`. - -### C-44 — Cron derives hour/minute from startTime when ScheduleTrigger has no schedule.hours/minutes (P1) - -- **Rationale**: `_recurrence_to_quartz_cron` read only schedule.minutes/ - hours and fell back to '0'/'0'; startTime was never read. A daily - trigger with startTime 21:00 UTC and no schedule block emitted - '0 0 0 * * ?' (midnight), a 21-hour offset, silently. Per ADF docs the - first-execution time (from startTime) is the default time-of-day. -- **Files**: `src/flowx/translator/engine.py`. -- **Tests**: `test_schedule_trigger_derives_time_of_day_from_start_time` - (Day recurrence, startTime '2023-03-15T21:00:00Z' -> '0 0 21 * * ?') in - `tests/unit/test_translators.py`. -- **Commit**: `6ec5c37`. - -### C-45 — Month-frequency periodic trigger no longer emits the invalid DAB unit MONTHS (P1) - -- **Rationale**: `_recurrence_to_periodic` mapped Month -> MONTHS, but the - Databricks Jobs API PeriodicTriggerConfigurationTimeUnit enum only - defines DAYS, HOURS, WEEKS — MONTHS is rejected by bundle validate/ - deploy. Drop Month from the unit_map (single-month routes to quartz cron - via monthDays); an interval > 1 Month emits a manual_setup schedule note. -- **Files**: `src/flowx/translator/engine.py`. -- **Tests**: - `test_schedule_trigger_interval_2_months_does_not_emit_months_unit` in - `tests/unit/test_translators.py`. -- **Commit**: `5b50b1c`. - -### C-46 — Generate create_secrets.py against the Databricks SDK, not dbutils.secrets writes (P1) - -- **Rationale**: `setup_generator` emitted `dbutils.secrets.createScope` and - `dbutils.secrets.put`. The `dbutils.secrets` submodule is read-only - (get / getBytes / list / listScopes only); both calls raise - AttributeError on the first cell. Generate against - `WorkspaceClient().secrets.create_scope` (RESOURCE_ALREADY_EXISTS - try/except) and `w.secrets.put_secret`. -- **Files**: `src/flowx/bundler/setup_generator.py`, - `tests/unit/test_bundler.py`. -- **Tests**: updated the two `createScope`/`put` asserts to - `create_scope`/`put_secret` (plus WorkspaceClient + negative asserts) in - `test_secrets_setup_notebook_content` and the write_bundle round-trip - test in `tests/unit/test_bundler.py`. -- **Commit**: `acdbb40`. - -### C-47 — File-source Lookup substitutes dataset() parameter refs before baking the abfss:// path (P1) - -- **Rationale**: `lookup.py` read the dataset reference (whose parameters - bind digitalCase/fileName to pipeline params) but never applied them. - `_unwrap_expression` only unwrapped the expression dict, leaving - folderPath '@toLower(dataset().digitalCase)' and fileName - '@dataset().fileName' verbatim; the code generator baked a literal broken - 'abfss://.../@toLower(dataset().digitalCase)/...' default that spark.read - cannot load. Build a dataset-parameter scope, substitute dataset().X, - resolve the result; `_assemble_file_lookup_source_path` drops any leaked - raw dataset() component as a safety net. -- **Files**: - `src/flowx/translator/activity_translators/lookup.py`, - `src/flowx/preparer/code_generator.py`. -- **Tests**: - `test_translate_lookup_substitutes_dataset_parameter_refs` in - `tests/unit/test_translators.py`. -- **Commit**: `f360a68`. - -## Summary (iteration 5) - -- 8 commits on `fix-0603` (C-40..C-47); C-40..C-42 landed in an earlier - pass, C-43..C-47 completed the iteration. CF5-002 (P2) folded into C-43. -- 624 unit tests pass after the final iteration-5 commit (iteration-4 - baseline: 616 — net 8 new tests). -- No tests broken; no `--no-verify` or `--amend` used. -- All 8 P0/P1 plan items implemented end-to-end. P2 NB-ITER5-002 - intentionally excluded per the plan's dedup decision. diff --git a/README.md b/README.md index 0ba13d5..e06e533 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # flowx -ADF to Databricks Lakeflow Jobs translator via Declarative Automation Bundles. +ADF to Databricks Lakeflow Jobs translator, delivered as agent skills. -flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). It deterministically translates known activity types and falls back to agentic LLM-assisted translation for complex or rare types. +flowx converts Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). It deterministically translates known activity types and falls back to agentic (LLM-assisted) translation for complex or rare types. flowx runs as a set of [agent skills](skills/) usable from Databricks Genie Code, Claude Code, or any tool that supports the Agent Skills standard. ## Architecture @@ -10,23 +10,23 @@ flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline de flowx Pipeline ================== - ADF JSON (UC Volumes) + ADF JSON (UC Volumes / Workspace) | v +------------------+ - | 1. PROFILE | Parse ADF ARM/JSON exports + | 1. DISCOVER | Parse ADF ARM/JSON exports | adf_loader.py | -> Typed AST -> metadata/inventory.json +------------------+ | v +------------------+ - | 2. TRANSLATE | Registry dispatch + topological sort + | 2. CONVERT | Registry dispatch + topological sort | engine.py | -> Pipeline IR (deterministic + agentic gaps) +------------------+ | v +------------------+ - | 3. PREPARE | IR -> DAB YAML + notebooks + setup scripts + | 3. PACKAGE | IR -> DAB YAML + notebooks + setup scripts | dab_writer.py | -> Deployable DABs project +------------------+ | @@ -34,24 +34,66 @@ flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline de databricks bundle validate / deploy ``` -## Quick Start +The phases are exposed two ways: as **skills** the agent runs directly (via a Python virtual +environment locally), and as a single **MCP tool** hosted on a Databricks App (for Genie Code). See +[Running flowx as an MCP server](#running-flowx-as-an-mcp-server). -1. Install the plugin in Claude Code: - ```bash - claude plugin install ghanse/flowx - ``` +## Installation -2. Run the end-to-end migration: - ``` - /flowx:flowx-migrate - ``` +flowx installs in one of two shapes depending on where your agent runs. Full, step-by-step +instructions for both are in the [installation docs](docs/content/docs/installation.mdx); the +summary: - Or run individual phases: - ``` - /flowx:flowx-discover # Parse ADF JSON, produce inventory + complexity report - /flowx:flowx-convert # Deterministic + agentic translation - /flowx:flowx-package # Generate DABs project - ``` +### Databricks Genie Code + +The phases run as an MCP server (a Databricks App). Clone the repo into **`/Workspace/Shared`** +(so the app's service principal can read the source), copy `skills/` into your skills folder, then +run the setup skill: + +``` +@flowx-setup +``` + +On Databricks, `flowx-setup` deploys the `mcp-flowx` app for you. You can also deploy it directly by +running the **`app/deploy_app.py`** notebook (SDK-based, works on serverless), or `app/deploy.sh` +from a workspace web terminal. Then add the app under Genie Code **Settings → MCP Servers → Add +Server → Custom MCP server**. + +### Claude Code (and other local agent harnesses) + +flowx is a Claude Code plugin distributed through its marketplace: + +``` +/plugin marketplace add databricks-solutions/flowx +/plugin install flowx@flowx +``` + +Run `/reload-plugins`, then set up the local runtime once: + +``` +/flowx:flowx-setup +``` + +This provisions a Python virtual environment (via `scripts/bootstrap.sh`) and writes a +`.migration-venv` marker the phase skills read. No `uv` is required for plugin users. + +## Usage + +Run the end-to-end migration: + +``` +/flowx:flowx-migrate +``` + +Or run individual phases: + +``` +/flowx:flowx-discover # Parse ADF JSON, produce inventory + complexity report +/flowx:flowx-convert # Deterministic + agentic translation +/flowx:flowx-package # Generate DABs project +``` + +(In Genie Code, invoke the same skills with the `@` prefix, e.g. `@flowx-migrate`.) ## Supported ADF Activity Types @@ -78,25 +120,28 @@ flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline de ### Agentic Fallback (12 types) +Activities with complex semantics, or without a direct Databricks equivalent, are translated by the +agent using LLM-assisted reasoning from the activity's ARM JSON. + | ADF Activity | Strategy | |---|---| -| ExecuteDataFlow | LLM-assisted via adf-to-databricks-plugin | -| SqlServerStoredProcedure | LLM-assisted via adf-to-databricks-plugin | -| AzureFunction | LLM-assisted via adf-to-databricks-plugin | -| WebHook | LLM-assisted via adf-to-databricks-plugin | -| Custom | LLM-assisted via adf-to-databricks-plugin | -| ExecuteSSISPackage | LLM-assisted via adf-to-databricks-plugin | -| AzureMLExecutePipeline | LLM-assisted via adf-to-databricks-plugin | -| GetMetadata | LLM-assisted via adf-to-databricks-plugin | -| Validation | LLM-assisted via adf-to-databricks-plugin | -| Fail | LLM-assisted via adf-to-databricks-plugin | -| Script | LLM-assisted via adf-to-databricks-plugin | -| Until | LLM-assisted via adf-to-databricks-plugin | +| ExecuteDataFlow | LLM-assisted (agentic) | +| SqlServerStoredProcedure | LLM-assisted (agentic) | +| AzureFunction | LLM-assisted (agentic) | +| WebHook | LLM-assisted (agentic) | +| Custom | LLM-assisted (agentic) | +| ExecuteSSISPackage | LLM-assisted (agentic) | +| AzureMLExecutePipeline | LLM-assisted (agentic) | +| GetMetadata | LLM-assisted (agentic) | +| Validation | LLM-assisted (agentic) | +| Fail | LLM-assisted (agentic) | +| Script | LLM-assisted (agentic) | +| Until | LLM-assisted (agentic) | ## How It Works ### Phase 1: Discover -Reads ADF JSON definitions from Unity Catalog volumes, normalizes ARM template format, parses into typed AST nodes, and classifies each activity as deterministic, agentic, or unsupported. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`. +Reads ADF JSON definitions from Unity Catalog volumes (or a `/Workspace` Git folder), normalizes ARM template format, parses into typed AST nodes, and classifies each activity as deterministic, agentic, or unsupported. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`. ### Phase 2: Convert Applies deterministic translators via registry dispatch, resolves dependencies through topological sort, and threads immutable `TranslationContext` through control-flow visitors. Agentic gaps are flagged for LLM-assisted translation. Produces Pipeline IR. @@ -125,16 +170,24 @@ flowx_output/ SETUP.md # Setup instructions (package) metadata/ inventory.json # discover: activity inventory - profile_report.csv # profile: per-pipeline complexity report + profile_report.csv # discover: per-pipeline complexity report .arm.json # discover: verbatim original ADF/ARM source configuration.json # modify: collected configuration answers - .work/ # transient intermediates (translation report, IR, gaps.json); pruned by prepare + .work/ # transient intermediates (translation report, IR, gaps.json); pruned by package ``` +## Running flowx as an MCP server + +The phases are also packaged as [Model Context Protocol](https://modelcontextprotocol.io) tools (in +[`src/flowx/mcp/`](src/flowx/mcp)) so an agent can invoke them directly instead of shelling out to +the CLI. The server exposes a single `flowx(command, parameters)` tool to stay under host tool +limits. For Databricks Genie Code it runs as a Databricks App; see the [app README](app/README.md) +for deployment (SDK notebook or CLI script) and Genie Code registration. + ## Development ```bash -make dev # Install dependencies +make dev # Install dependencies (uses uv) make test # Run unit tests make integration # Run integration tests make fmt # Format + lint (ruff + mypy) @@ -145,6 +198,9 @@ make clean # Remove build artifacts - Python 3.12+ - [uv](https://docs.astral.sh/uv/) package manager +These prerequisites are for contributing to the flowx project. Plugin *users* do not need `uv` — +`flowx-setup` provisions the runtime (a pip-based `.venv` locally, or the MCP server on Databricks). + ## Contributing 1. Fork the repository diff --git a/app/README.md b/app/README.md index 6f52aa3..c60cbf7 100644 --- a/app/README.md +++ b/app/README.md @@ -57,14 +57,25 @@ Register the stdio server with a local MCP client, e.g.: ## Deploy as a Databricks App (for Genie Code) +**Recommended — the `deploy_app.py` notebook (SDK, runs on serverless).** Open `app/deploy_app.py` +in the workspace and run it, setting the `repo_root` widget to the flowx checkout +(e.g. `/Workspace/Shared/flowx`). It uses the Databricks SDK to stage a self-contained bundle (the +app entrypoint plus a vendored copy of the pure-Python `flowx` package) to `/Workspace/Shared/mcp-flowx` +and create/deploy the app (default name **`mcp-flowx`**). Because it uploads through the SDK Workspace +API, it runs directly in a serverless Genie Code session — no CLI needed. The deployment logic lives in +`app/deploy_helpers.py` (imported by the notebook, so it is also unit-testable). + +**CLI alternative — `deploy.sh`:** + ```bash -# Authenticated Databricks CLI (v0.230+) required. +# Authenticated Databricks CLI (v0.230+) required — run from a web terminal or local machine. ./app/deploy.sh ``` -`deploy.sh` stages a self-contained bundle in a temporary directory outside the repo (the app entrypoint -plus a vendored copy of the pure-Python `flowx` package), syncs it to your -workspace, and creates/deploys the app (default name **`mcp-flowx`**). +`deploy.sh` stages a self-contained bundle in a temporary directory outside the repo, syncs it to your +workspace via `databricks sync`, and creates/deploys the app. `databricks apps deploy` / `sync` are +not available from serverless notebook Python, which is why the notebook above is preferred inside +Genie Code. > **Clone into `/Workspace/Shared`.** `deploy.sh` deploys the app source from > `/Workspace/Shared/` because the app's service principal **cannot read @@ -75,9 +86,9 @@ workspace, and creates/deploys the app (default name **`mcp-flowx`**). End-to-end, to use it from Genie Code: -1. **Deploy** with `./app/deploy.sh`. The app is named `mcp-flowx` and deploys the - source from `/Workspace/Shared/mcp-flowx` (override with `APP_SOURCE_PATH`). The - script prints the app URL; the MCP endpoint is `/mcp`. +1. **Deploy** by running the `app/deploy_app.py` notebook (or `./app/deploy.sh` from a CLI session). + The app is named `mcp-flowx` and deploys the source from `/Workspace/Shared/mcp-flowx`. The + deployer prints the app URL; the MCP endpoint is `/mcp`. 2. **Grant app access:** give **Can use** on `mcp-flowx` to the users / service principals that will call it (Apps UI → *Permissions*, or `databricks apps set-permissions mcp-flowx ...`). @@ -100,10 +111,11 @@ set the app env var `FLOWX_ALLOWED_ORIGINS` to your workspace URL and redeploy. MCP access is capped at **20 tools** across all servers; flowx exposes just **one** tool (`flowx`, with 12 commands), so it uses a single slot. -> Run `./app/deploy.sh` from a Databricks CLI session (workspace web terminal or a -> local machine) — `databricks apps` deploy is not available from serverless -> notebook Python. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp) -> and [host a custom MCP server](https://docs.databricks.com/aws/en/generative-ai/mcp/custom-mcp). +> In serverless Genie Code, deploy with the `app/deploy_app.py` notebook (SDK-based). `./app/deploy.sh` +> must run from a Databricks CLI session (workspace web terminal or a local machine), since +> `databricks apps` deploy is not available from serverless notebook Python. See [Connect Genie Code +> to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp) and [host a +> custom MCP server](https://docs.databricks.com/aws/en/generative-ai/mcp/custom-mcp). ## Troubleshooting diff --git a/app/deploy_app.py b/app/deploy_app.py new file mode 100644 index 0000000..335993f --- /dev/null +++ b/app/deploy_app.py @@ -0,0 +1,115 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC # Deploy the flowx MCP server (SDK, no CLI) +# MAGIC +# MAGIC Deploys the flowx MCP server as a Databricks App using the **Python SDK** only, so it runs +# MAGIC directly from a **serverless** Databricks / Genie Code session — where `app/deploy.sh` cannot, +# MAGIC because `databricks apps deploy` / `databricks sync` require a CLI session (web terminal or a +# MAGIC local machine). +# MAGIC +# MAGIC The deployment logic lives in `deploy_helpers.py` (imported below) so it stays importable and +# MAGIC unit-testable. Because the bundle is uploaded through the **Workspace API** (`ImportFormat.RAW`), +# MAGIC there is no `databricks sync` and none of its footguns: no `.gitignore` dropping staged files, +# MAGIC and no stray `databricks.yml` making the CLI abort with "please specify target". + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Configuration +# MAGIC +# MAGIC - **repo_root** *(required)* — the flowx checkout: the directory containing `app/` and +# MAGIC `src/flowx` (e.g. `/Workspace/Shared/flowx`). +# MAGIC - **app_name** — the app to create/deploy (default `mcp-flowx`; the `mcp-` prefix auto-lists it +# MAGIC in the AI Playground). +# MAGIC - **source_code_path** — workspace dir the app deploys from. Must be readable by the app's +# MAGIC service principal, so it defaults under `/Workspace/Shared` — **not** a private +# MAGIC `/Workspace/Users/` home, which the app SP cannot read. + +# COMMAND ---------- + +dbutils.widgets.text("repo_root", "/Workspace/Shared/flowx", "flowx repo root (contains app/ and src/flowx)") +dbutils.widgets.text("app_name", "mcp-flowx", "App name") +dbutils.widgets.text("source_code_path", "", "Workspace source path (blank = /Workspace/Shared/)") + +repo_root = dbutils.widgets.get("repo_root").strip() +app_name = dbutils.widgets.get("app_name").strip() +source_code_path = dbutils.widgets.get("source_code_path").strip() or f"/Workspace/Shared/{app_name}" +app_description = "flowx MCP server — ADF to Databricks Lakeflow migration tools" + +# COMMAND ---------- + +import os +import sys + +# Make the sibling deploy_helpers module importable from the user-specified checkout. +sys.path.insert(0, os.path.join(repo_root, "app")) + +from databricks.sdk import WorkspaceClient + +import deploy_helpers + +deploy_helpers.validate_repo_root(repo_root) +workspace_client = WorkspaceClient() + +print(f"repo_root : {repo_root}") +print(f"app_name : {app_name}") +print(f"source_code_path : {source_code_path}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Stage the self-contained bundle at the workspace source path +# MAGIC +# MAGIC Uploads the app files and a fresh copy of the `flowx` package to `source_code_path`. +# MAGIC `ImportFormat.RAW` lands each file verbatim (not as a notebook) and `overwrite=True` makes +# MAGIC redeploys idempotent. + +# COMMAND ---------- + +flowx_module_count = deploy_helpers.stage_app_bundle(workspace_client, repo_root, source_code_path) +print(f"Staged app files + {flowx_module_count} flowx module files to {source_code_path}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Create the app if needed, then deploy +# MAGIC +# MAGIC `create_and_wait` / `deploy_and_wait` block until the operation reaches a terminal state +# MAGIC (default 20-minute timeout each). + +# COMMAND ---------- + +was_created = deploy_helpers.ensure_app_exists(workspace_client, app_name, app_description) +print(f"App '{app_name}' {'created' if was_created else 'already existed — reusing it'}.") + +print(f"Deploying '{app_name}' from {source_code_path} ...") +deployment = deploy_helpers.deploy_app_source(workspace_client, app_name, source_code_path) + +deployed_app = workspace_client.apps.get(name=app_name) +app_url = deployed_app.url or "" +deployment_state = deployment.status.state if deployment.status else "unknown" +print(f"Deployment status: {deployment_state}") +print(f"App URL : {app_url or '(pending — re-run workspace_client.apps.get)'}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Next steps to use it in Genie Code + +# COMMAND ---------- + +print(f""" +Deployed app: {app_name} +MCP endpoint: {app_url or ''}/mcp +Health check: {app_url or ''}/ + +Next steps: + 1. App access — grant 'Can use' on '{app_name}' to the users / service principals that will + call it (Apps UI > Permissions). + 2. Data access — grant the app's service principal read/write on the catalogs, schemas, and + UC volumes the migration touches (and any SQL warehouse used by the reporting tools). + 3. Add it in Genie Code (Agent mode): Settings > MCP Servers > Add Server > Custom MCP server + > select '{app_name}' > Save. The single 'flowx' tool appears immediately. + 4. If a browser CORS error appears, set the app env var FLOWX_ALLOWED_ORIGINS to your + workspace URL and redeploy (re-run this notebook). +""") diff --git a/app/deploy_helpers.py b/app/deploy_helpers.py new file mode 100644 index 0000000..fedcdbf --- /dev/null +++ b/app/deploy_helpers.py @@ -0,0 +1,120 @@ +"""Helpers for deploying the flowx MCP server as a Databricks App via the Python SDK. + +Kept separate from ``deploy_app`` (the deployment notebook) so the deployment logic is +importable and unit-testable outside a notebook session. Every function takes an explicit +``WorkspaceClient`` and plain paths, so nothing here depends on ``dbutils`` or a notebook +runtime. + +Mirrors what ``deploy.sh`` does, minus the CLI: assemble a self-contained source bundle +(``app.py``, ``app.yaml``, ``requirements.txt``, and a vendored copy of the ``flowx`` package) +at a workspace path, then create/deploy the app from it. +""" + +import os + +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.apps import App, AppDeployment, AppDeploymentMode +from databricks.sdk.service.workspace import ImportFormat + +# Top-level files the app needs alongside the vendored flowx package. +APP_BUNDLE_FILES = ("app.py", "app.yaml", "requirements.txt") + + +def validate_repo_root(repo_root: str) -> None: + """Raise if ``repo_root`` is not a flowx checkout (missing app/app.py or src/flowx).""" + if not repo_root: + raise ValueError( + "repo_root is required: set it to the flowx checkout — the directory " + "containing app/ and src/flowx." + ) + has_app_entrypoint = os.path.isfile(os.path.join(repo_root, "app", "app.py")) + has_flowx_package = os.path.isdir(os.path.join(repo_root, "src", "flowx")) + if not (has_app_entrypoint and has_flowx_package): + raise ValueError( + f"{repo_root!r} does not look like a flowx checkout " + "(expected app/app.py and src/flowx)." + ) + + +def upload_file(workspace_client: WorkspaceClient, local_path: str, workspace_path: str) -> None: + """Upload one local file to a workspace path verbatim (RAW), creating parent dirs.""" + workspace_client.workspace.mkdirs(os.path.dirname(workspace_path)) + with open(local_path, "rb") as file_handle: + workspace_client.workspace.upload( + workspace_path, file_handle, format=ImportFormat.RAW, overwrite=True + ) + + +def upload_directory( + workspace_client: WorkspaceClient, local_directory: str, workspace_directory: str +) -> int: + """Recursively upload a directory tree, skipping ``__pycache__`` and ``.pyc`` files. + + Returns the number of files uploaded. + """ + uploaded_file_count = 0 + for current_directory, subdirectories, file_names in os.walk(local_directory): + subdirectories[:] = [name for name in subdirectories if name != "__pycache__"] + for file_name in file_names: + if file_name.endswith(".pyc"): + continue + local_path = os.path.join(current_directory, file_name) + relative_path = os.path.relpath(local_path, local_directory) + workspace_path = f"{workspace_directory}/{relative_path.replace(os.sep, '/')}" + upload_file(workspace_client, local_path, workspace_path) + uploaded_file_count += 1 + return uploaded_file_count + + +def stage_app_bundle( + workspace_client: WorkspaceClient, repo_root: str, source_code_path: str +) -> int: + """Assemble the self-contained app bundle at ``source_code_path`` in the workspace. + + Uploads the top-level app files and a fresh copy of the ``flowx`` package (the prior + copy is deleted first so removed modules do not linger). Returns the flowx module count. + """ + app_directory = os.path.join(repo_root, "app") + workspace_client.workspace.mkdirs(source_code_path) + + for file_name in APP_BUNDLE_FILES: + upload_file( + workspace_client, + os.path.join(app_directory, file_name), + f"{source_code_path}/{file_name}", + ) + + package_source_directory = os.path.join(repo_root, "src", "flowx") + package_workspace_directory = f"{source_code_path}/flowx" + try: + workspace_client.workspace.delete(package_workspace_directory, recursive=True) + except Exception: + pass # first deploy: nothing to delete + return upload_directory( + workspace_client, package_source_directory, package_workspace_directory + ) + + +def ensure_app_exists( + workspace_client: WorkspaceClient, app_name: str, app_description: str +) -> bool: + """Create the app if it does not already exist. Returns True if it was created.""" + try: + workspace_client.apps.get(name=app_name) + return False + except Exception: + workspace_client.apps.create_and_wait(App(name=app_name, description=app_description)) + return True + + +def deploy_app_source( + workspace_client: WorkspaceClient, app_name: str, source_code_path: str +) -> AppDeployment: + """Deploy the staged source to the app and block until deployment reaches terminal state.""" + return workspace_client.apps.deploy_and_wait( + app_name=app_name, + app_deployment=AppDeployment( + source_code_path=source_code_path, + mode=AppDeploymentMode.SNAPSHOT, + ), + ) diff --git a/docs/README.md b/docs/README.md index f151cad..1434dba 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # flowx docs -Documentation site for [flowx](https://github.com/ghanse/flowx), built with [fumadocs](https://fumadocs.dev) and deployed to GitHub Pages. +Documentation site for [flowx](https://github.com/databricks-solutions/flowx), built with [fumadocs](https://fumadocs.dev) and deployed to GitHub Pages. ## Local development diff --git a/docs/app/(home)/page.tsx b/docs/app/(home)/page.tsx index 021cb6b..9365553 100644 --- a/docs/app/(home)/page.tsx +++ b/docs/app/(home)/page.tsx @@ -16,7 +16,7 @@ export default function HomePage() { Read the docs View on GitHub diff --git a/docs/app/layout.config.tsx b/docs/app/layout.config.tsx index 9741c3e..39c5777 100644 --- a/docs/app/layout.config.tsx +++ b/docs/app/layout.config.tsx @@ -14,9 +14,9 @@ export const baseOptions: BaseLayoutProps = { }, { text: 'GitHub', - url: 'https://github.com/ghanse/flowx', + url: 'https://github.com/databricks-solutions/flowx', external: true, }, ], - githubUrl: 'https://github.com/ghanse/flowx', + githubUrl: 'https://github.com/databricks-solutions/flowx', }; diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx index bed4d9e..fbdb0f8 100644 --- a/docs/content/docs/architecture.mdx +++ b/docs/content/docs/architecture.mdx @@ -78,7 +78,7 @@ The MCP server runs in whichever transport fits the calling tool. This is chosen own service principal ``` -See [Installation](/docs/installation#running-flowx-as-an-mcp-server) for the exact commands and the [app README](https://github.com/ghanse/flowx/tree/main/app) for deployment details. +See [Installation](/docs/installation#running-flowx-as-an-mcp-server) for the exact commands and the [app README](https://github.com/databricks-solutions/flowx/tree/main/app) for deployment details. A Databricks App can't read the user's workspace / UC Volume files (`/Volumes/...` is **not** auto-mounted). Two ways to get data in/out of the `flowx` tool: diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index 619c98d..cb9f5d9 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -1,147 +1,151 @@ --- title: Installation -description: Install flowx in Databricks Genie Code, Claude Code, or other agentic tools. +description: Install flowx in Databricks Genie Code or a local agent harness. --- -import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; import { Callout } from 'fumadocs-ui/components/callout'; import { Steps, Step } from 'fumadocs-ui/components/steps'; -flowx is a set of [agent skills](https://github.com/ghanse/flowx/tree/main/skills) that can be installed and used with AI coding assistants. -To use these skills, install flowx as a plugin using your AI assistant's preferred installation method. +flowx is a set of [agent skills](https://github.com/databricks-solutions/flowx/tree/main/skills) that run from an AI coding assistant. How you install them depends on where your agent runs: - - -Clone the flowx repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos). Clone it under **`/Workspace/Shared`** (e.g. `/Workspace/Shared/flowx`) rather than your private `/Workspace/Users/` home — when you later deploy the MCP server, the app's service principal must be able to read the source, and it has no access to private user folders by default. Then copy the `skills/` directory into a user-level skills folder: +- **Databricks Genie Code** runs the phases as a hosted **MCP server** (a Databricks App). No local Python environment is involved — the app vendors flowx's code and dependencies. +- **A local agent harness (Claude Code, or any Agent Skills tool)** runs the phases from a local **Python virtual environment**, optionally exposing them over a local MCP server too. -```bash -databricks workspace import-dir skills /Users//.assistant/skills -``` +Pick the matching section below and follow it end to end. -To make flowx available for other workspace users, copy `skills/` into a workspace-level skills folder: +## Installing flowx for Databricks Genie Code -```bash -databricks workspace import-dir skills /Workspace/.assistant/skills -``` +In Genie Code the phases run as the single `flowx` tool on a Databricks App you deploy and then register as a custom MCP server. -Genie Code picks up skills from these directories automatically. Skills fire automatically when their description matches your request. -To invoke a specific skill, use the `@` prefix (e.g. `@flowx-migrate translate the ADF pipelines I exported to /Volumes/main/default/adf_export`). + + +### Clone flowx into a shared workspace location -See the [Databricks Genie Code Skills documentation](https://docs.databricks.com/aws/en/genie-code/skills) for more details. - +Clone the repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos), under **`/Workspace/Shared`** (e.g. `/Workspace/Shared/flowx`). - -flowx is packaged as a Claude Code plugin. The plugin manifest lives at [`.claude-plugin/plugin.json`](https://github.com/ghanse/flowx/blob/main/.claude-plugin/plugin.json). To install -flowx, run the following command from a Claude Code session: + +The MCP app's service principal cannot read private `/Workspace/Users/` folders by default. Cloning into `/Workspace/Shared` keeps the repo, the deployed app source, and team access all in a location every user and the app's service principal can reach. If your workspace restricts `/Workspace/Shared`, use any other all-users location and pass it to the deployer. + + -```bash -/plugin marketplace add ghanse/flowx -/plugin install flowx -``` + +### Copy the skills into your skills folder -You can also copy the skill folders into your local `/.claude/skills` folder: +Genie Code picks up skills from your `.assistant/skills` folder automatically. Copy `skills/` into a user-level folder: ```bash -cp -R skills/{flowx-setup,flowx-discover,flowx-convert,flowx-package,flowx-migrate} ~/.claude/skills/ +databricks workspace import-dir skills /Users//.assistant/skills ``` -Once installed, the skills can be invoked using `/flowx:flowx-migrate`, `/flowx:flowx-discover`, etc. Claude Code can also invoke the skills when prompted. See the [Claude Code Plugins documentation](https://docs.claude.com/en/docs/claude-code/plugins) for more details. - - - -Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install the flowx skills. The general pattern: - -1. Copy each skill folder (`skills/flowx-setup`, `skills/flowx-discover`, `skills/flowx-convert`, `skills/flowx-package`, `skills/flowx-migrate`) into the tool's configured skills directory. -2. Make sure the path contains `SKILL.md` directly, -3. Restart the tool if it caches skill metadata at startup. - - -If your tool expects a single Markdown file instead of a directory tree, use the following command to flatten flowx's skills files: +Or make flowx available to all workspace users with a workspace-level folder: ```bash -cat skills/*/SKILL.md > flowx-skills.md +databricks workspace import-dir skills /Workspace/.assistant/skills ``` - - - - -## Running flowx as an MCP server -flowx's phases are also packaged as [Model Context Protocol](https://modelcontextprotocol.io) tools (in [`src/flowx/mcp/`](https://github.com/ghanse/flowx/tree/main/src/flowx/mcp)) so an agent can invoke them directly instead of shelling out to the CLI. The `setup` skill wires this up automatically based on your environment; you can also do it manually. See [Architecture](/docs/architecture) for how the tool layer maps onto the phases. - -### Configuring the MCP server for Databricks Genie Code - -Genie Code connects to a **hosted** MCP endpoint, so the flowx tools run as a Databricks App that you add in Genie Code's **Custom MCP server** picker. End to end: - - - -#### Clone flowx - -Follow the **Databricks Genie Code** install steps above to clone the repo and copy `skills/` into your skills folder. - - -The MCP app's service principal cannot read private `/Workspace/Users/` folders by default, and `deploy.sh` deploys the source from `/Workspace/Shared/`. Cloning into `/Workspace/Shared` keeps the repo, the deployed source, and team access all in a location every user and the app's service principal can reach. If your workspace restricts `/Workspace/Shared`, use any other folder all users (and the app service principal) can read and pass it via `APP_SOURCE_PATH`. - +Skills fire automatically when their description matches your request. To invoke one explicitly, use the `@` prefix (e.g. `@flowx-migrate translate the ADF pipelines I exported to /Volumes/main/default/adf_export`). See the [Genie Code Skills docs](https://docs.databricks.com/aws/en/genie-code/skills) for more. -#### Run the setup skill +### Run the setup skill -Ask your agent to *"set up the flowx environment"* (or run `bash /scripts/bootstrap.sh`). On Databricks the `setup` skill detects the environment and, after creating the venv, runs the app deployment in the next step for you. Run it from a workspace web terminal if your Genie session can't shell out to the Databricks CLI. +Ask your agent to *"set up the flowx environment"* (or run `@flowx-setup`). On Databricks, setup detects the environment and prepares the MCP path — it does **not** create a virtual environment, because the phases run through the deployed app rather than a local interpreter. -#### Deploy the MCP server +### Deploy the MCP server -```bash -bash /app/deploy.sh -``` +Deploy the `mcp-flowx` Databricks App from the flowx checkout. The recommended way runs entirely in the workspace, including on **serverless** compute: -`deploy.sh` stages a self-contained bundle (the app entrypoint plus a vendored copy of the flowx source), syncs it to **`/Workspace/Shared/mcp-flowx`** (a location the app's service principal can read — override with `APP_SOURCE_PATH`), and creates/deploys the **`mcp-flowx`** app. The script prints the app URL; the MCP endpoint is **`/mcp`**. +Open and run the **`app/deploy_app.py`** notebook. Set its `repo_root` widget to your checkout (e.g. `/Workspace/Shared/flowx`); it uses the Databricks SDK to stage a self-contained source bundle (app entrypoint plus a vendored copy of the flowx package) to `/Workspace/Shared/mcp-flowx` and create/deploy the app. The notebook prints the app URL; the MCP endpoint is **`/mcp`**. - -`databricks apps` deploy commands require a Databricks CLI session and must be run from the workspace web terminal or a local machine. + +You can instead run `bash app/deploy.sh` from a **workspace web terminal or a local machine**. The `databricks apps deploy` / `databricks sync` commands it uses require a CLI session and are **not** available from serverless notebook Python — which is why `deploy_app.py` (SDK-based) is preferred inside Genie Code. -#### Grant access +### Grant access - **App access:** grant **Can use** on the `mcp-flowx` app to the users or service principals that will call it (Apps UI → *Permissions*, or `databricks apps set-permissions`). - **Data access:** grant the app's own service principal access to the catalogs, schemas, and Unity Catalog volumes the migration reads from and writes to, plus any SQL warehouse used by the reporting commands (`flowx(command="record_results")` / `flowx(command="install_dashboard")`). -#### Register the MCP server +### Register the MCP server in Genie Code -MCP servers are available in Genie Code [Agent mode](https://learn.microsoft.com/en-us/azure/databricks/genie-code/use-genie-code#modes). To add the flowx MCP server: +MCP servers are available in Genie Code [Agent mode](https://learn.microsoft.com/en-us/azure/databricks/genie-code/use-genie-code#modes): 1. In the Genie Code panel, click **⚙ Settings**. 2. Under **MCP Servers**, click **+ Add Server**. 3. Choose **Custom MCP server** and select the **`mcp-flowx`** Databricks App. 4. Click **Save**. -The single `flowx` tool will be available when you use Genie Code in Agent mode. +The single `flowx` tool is available when you use Genie Code in Agent mode. -Databricks requires a custom MCP app to be: -* Deployed in the same workspace -* Reachable at `https:///mcp` - -If Genie Code cannot connect to the flowx MCP server, set the app's `FLOWX_ALLOWED_ORIGINS` environment variable to your workspace URL and redeploy. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp). +A custom MCP app must be deployed in the **same workspace** and reachable at `https:///mcp`. If Genie Code cannot connect, set the app's `FLOWX_ALLOWED_ORIGINS` environment variable to your workspace URL and redeploy. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp). -#### Verify the MCP Server +### Verify -Open the health endpoint `/` (returns `{"status":"ok"}`), or ask Genie Code *"what flowx MCP tools are available?"*. You should see the single `flowx` tool. +Open the health endpoint `/` (returns `{"status":"ok"}`), or ask Genie Code *"what flowx MCP tools are available?"*. You should see the single `flowx` tool. You can now run `@flowx-migrate` (or the individual phase skills). -### Configuring the MCP server for other agent tools +## Installing flowx for a local agent harness (Claude Code) -Deploy the MCP server locally to use flowx with other agent tools. Install the MCP server stack into a local Python virtual environment and run it over stdio: +Locally, flowx installs as a Claude Code plugin and runs its phases from a Python virtual environment. + + + +### Install the plugin + +flowx is distributed through its Claude Code marketplace. From a Claude Code session: + +```bash +/plugin marketplace add databricks-solutions/flowx +/plugin install flowx@flowx +``` + +Then run `/reload-plugins` to activate it. + + +You can also copy the skill folders straight into your local skills directory: + +```bash +cp -R skills/{flowx-setup,flowx-discover,flowx-convert,flowx-package,flowx-migrate} ~/.claude/skills/ +``` + + + + +### Run the setup skill + +Run `/flowx:flowx-setup` (or ask *"set up the flowx environment"*) **once** before any phase. flowx's Python modules depend on third-party packages (`pyyaml`, `databricks-sdk`, `sqlglot`), so setup provisions an isolated virtual environment via `scripts/bootstrap.sh`. It will: + +1. Check that `python3`, `pip`, and the `venv` module are available. +2. Create the virtual environment at `/.venv`. +3. Install `requirements.txt` into it with `pip`. +4. Write the resolved interpreter path to the marker file `/.migration-venv`, which the phase skills read. + +The environment is created once and reused. No `uv` is required for plugin users. + + +If `python3`, `pip`, or the `venv` module are missing, the script prints a warning and exits **without** creating anything. Install Python, then re-run setup: + +* **macOS:** `brew install python` +* **Debian/Ubuntu:** `sudo apt-get install python3 python3-venv python3-pip` +* **Windows:** [python.org/downloads](https://www.python.org/downloads/) (enable "Add python.exe to PATH") + + + + +### (Optional) Run the phases over a local MCP server + +The phase skills call the venv CLI directly, so this step is optional. To instead drive the phases through MCP tools locally, install the MCP server stack into the venv and register the stdio server with your MCP client: ```bash PY="$(cat /.migration-venv)" @@ -149,8 +153,6 @@ PY="$(cat /.migration-venv)" PYTHONPATH="/src" "$PY" -m flowx.mcp ``` -Register the server with your MCP client (use the interpreter path from the marker file for `command`): - ```json { "mcpServers": { @@ -166,53 +168,35 @@ Register the server with your MCP client (use the interpreter path from the mark If you prefer an installed package over `PYTHONPATH`, run `pip install -e ".[mcp]"` from the plugin root; then `python -m flowx.mcp` works without setting `PYTHONPATH`. + -## Running flowx as a Python process - -flowx's skills invoke Python modules that may depend on third-party packages. The `setup` skill provisions an isolated virtual environment with the required dependencies. - -Run it **once** after installing the skills, before `discover`, `convert`, `package`, or `migrate`. Just ask your agent: + +### Verify -> Set up the flowx environment +Open Claude Code and ask *"What flowx skills do you have available?"*. You should see all five skills (`flowx-setup`, `flowx-discover`, `flowx-convert`, `flowx-package`, `flowx-migrate`). Invoke them with `/flowx:flowx-migrate`, `/flowx:flowx-discover`, etc. -The setup script can also be run directly from the plugin root: - -```bash -bash /scripts/bootstrap.sh -``` - -Running the setup process will: - -1. Check that `python3`, `pip`, and `venv` are available. -2. Create the virtual environment if it doesn't already exist. When running under Databricks (Genie Code or notebooks, detected via `DATABRICKS_RUNTIME_VERSION`), the venv is created at `/Workspace/Users//.migration-skills`; everywhere else it is created at `/.venv`. -3. Install the `requirements.txt` dependencies into your virtual environment using `pip`. -4. Write the resolved interpreter path to the marker file `/.migration-venv`. - -The environment is created once and reused. Re-running the script simply confirms the venv exists and its dependencies are satisfied. - - -If `python3`, `pip`, or the `venv` module are missing, the script will print a warning and exit **without** creating anything. -To install Python in your environment, run one of the following commands: - -* **macOS:** `brew install python` -* **Debian/Ubuntu:** `sudo apt-get install python3 python3-venv python3-pip` -* **Windows:** [python.org/downloads](https://www.python.org/downloads/) (enable "Add python.exe to PATH") - - -After the venv exists, every Python command the skills run uses the interpreter recorded in the marker file, with `src/` on `PYTHONPATH`. Read the interpreter path from `/.migration-venv` rather than hardcoding it: +If you hit a `ModuleNotFoundError` while running a phase, the venv is missing or incomplete — re-run `/flowx:flowx-setup`. Every Python command the skills run uses the interpreter recorded in `/.migration-venv`, with `src/` on `PYTHONPATH`: ```bash export PYTHONPATH="/src" PY="$(cat /.migration-venv)" "$PY" -m flowx.adapter inputs discover ``` + + -The marker file points at `/.venv` locally or `/Workspace/Users//.migration-skills` on Databricks. The agent normally runs these commands for you; they are handy for troubleshooting a `ModuleNotFoundError`. +### Other AI tools -## Verifying the installation +Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install the flowx skills: -Open your agent and ask: +1. Copy each skill folder (`skills/flowx-setup`, `skills/flowx-discover`, `skills/flowx-convert`, `skills/flowx-package`, `skills/flowx-migrate`) into the tool's configured skills directory, so the path contains `SKILL.md` directly. +2. Restart the tool if it caches skill metadata at startup. +3. Follow the **local agent harness** setup above to provision the Python environment (`scripts/bootstrap.sh`). -> What flowx skills do you have available? + +If your tool expects a single Markdown file instead of a directory tree, concatenate the skills: -You should see all five skills listed with their descriptions. If only some appear, double-check the install path your tool watches for skills. +```bash +cat skills/*/SKILL.md > flowx-skills.md +``` + diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index ead7187..50cb58e 100644 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -137,9 +137,9 @@ if [ ! -x "$VENV_DIR/bin/python" ]; then rm -rf "$VENV_DIR" "$PYTHON_BIN" -m venv --without-pip "$VENV_DIR" echo "Bootstrapping pip via get-pip.py ..." - curl -sSL https://bootstrap.pypa.io/get-pip.py -o /tmp/_orchestra_get_pip.py - "$VENV_DIR/bin/python" /tmp/_orchestra_get_pip.py --quiet - rm -f /tmp/_orchestra_get_pip.py + curl -sSL https://bootstrap.pypa.io/get-pip.py -o /tmp/_flowx_get_pip.py + "$VENV_DIR/bin/python" /tmp/_flowx_get_pip.py --quiet + rm -f /tmp/_flowx_get_pip.py fi else echo "Using existing virtual environment at $VENV_DIR ..." diff --git a/skills/flowx-convert/SKILL.md b/skills/flowx-convert/SKILL.md index 5a84d32..5cd2b43 100644 --- a/skills/flowx-convert/SKILL.md +++ b/skills/flowx-convert/SKILL.md @@ -2,8 +2,8 @@ name: flowx-convert description: > Translate parsed ADF pipeline AST into Databricks IR (intermediate representation). - Runs deterministic translators for known activity types, then invokes agentic skills - from adf-to-databricks-plugin for gaps. + Runs deterministic translators for known activity types, then performs agentic + (LLM-assisted) translation for the remaining gaps. triggers: - "translate ADF" - "convert ADF" @@ -22,7 +22,7 @@ This is phase 2 of the flowx migration workflow. It consumes the ADF source (pro The translation follows a **deterministic-first** strategy: 1. Activities with known, well-defined mappings are translated by built-in Python translators -2. Activities that require interpretation, expression conversion, or lack a Python translator are handled by agent skills from the `adf-to-databricks-plugin` +2. Activities that require interpretation, expression conversion, or lack a Python translator are handled by agentic (LLM-assisted) translation performed by the agent ## How to run this skill — MCP tools or venv CLI @@ -136,8 +136,7 @@ Read `/.work/translation_report.json`. It has this structure: "type": "ExecuteDataFlow", "strategy": "agentic", "status": "pending", - "raw_activity_json": { "...": "..." }, - "target_skill": "adf-to-databricks:adf-dataflow-converter" + "raw_activity_json": { "...": "..." } } ], "summary": { @@ -151,7 +150,7 @@ Read `/.work/translation_report.json`. It has this structure: ### Step 4 — Handle agentic gaps -For each translation with `"status": "pending"` and `"strategy": "agentic"`, invoke the appropriate skill from the `adf-to-databricks-plugin`. Route by activity type. +For each translation with `"status": "pending"` and `"strategy": "agentic"`, perform LLM-assisted translation from the activity's ARM JSON, routing by activity type. Every agentic gap in the translation report carries the activity's **full ADF/ARM JSON** under `raw_activity_json` (engine field `raw_definition`), and the generated placeholder notebook embeds the same JSON in a fenced `json` block. This holds for nested activities too — an `Until` inside an `IfCondition` / `Switch` / `ForEach` is reported as its own gap. Always translate from this ARM JSON. @@ -160,36 +159,36 @@ Databricks Lakeflow Jobs have no native repeat-until loop, so translate the `Unt - `typeProperties.expression` — the ADF exit condition (e.g. `@or(equals(variables('jobStatus'),'succeeded'), equals(variables('jobStatus'),'failed'))`); convert it into the Python `while not ():` guard. - `typeProperties.timeout` — wrap the loop in a wall-clock deadline (`time.monotonic()`), raising on timeout. - `typeProperties.activities` — the loop body (e.g. a `Wait`, a polling `WebActivity`, a `SetVariable` that captures the next status); translate each child inline so the whole loop runs in one notebook. -Read the loop variables from `dbutils.widgets`, surface the final state as a task value, and write the result over the placeholder notebook's `raise NotImplementedError` cell. If the external `adf-to-databricks:adf-pipeline-converter` skill is installed you may delegate to it with the same ARM JSON; otherwise perform the translation directly. +Read the loop variables from `dbutils.widgets`, surface the final state as a task value, and write the result over the placeholder notebook's `raise NotImplementedError` cell. Perform the translation directly from the same ARM JSON. **ExecuteDataFlow activities:** -Invoke `adf-to-databricks:adf-dataflow-converter` with the raw activity JSON and associated data flow definition. Provide context: +Translate the data flow directly from the raw activity JSON and associated data flow definition, using: - The raw `typeProperties` from the ADF activity - The data flow JSON definition (if available in the source directory under `dataflow/`) - The linked service configurations for source/sink connections - Target catalog and schema for the SDP pipeline or PySpark notebook output **Control flow activities (Switch, Until, Wait, Filter, AppendVariable):** -Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: +Translate the control-flow activity directly from the raw activity JSON, using: - The full pipeline JSON containing the activity - Any nested activities within the control flow - Variable definitions from the pipeline - The desired Databricks task type mapping **Stored procedures and external calls (SqlServerStoredProcedure, AzureFunction, WebHook, Custom):** -Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: +Translate the activity directly from the raw activity JSON, using: - The linked service configuration for the target system - Connection details and authentication method - Any parameters or request bodies **Complex expressions:** -If any activity (deterministic or agentic) contains ADF expressions that the deterministic translator could not resolve, invoke `adf-to-databricks:adf-expression-translator` with: +If any activity (deterministic or agentic) contains ADF expressions that the deterministic translator could not resolve, translate them directly, using: - The raw expression string (e.g., `@pipeline().parameters.inputPath`) - The expression context (pipeline parameters, variables, activity outputs) - The target format (Python f-string, Spark SQL, task parameter reference) **Trigger definitions:** -Invoke `adf-to-databricks:adf-trigger-converter` with: +Translate the trigger directly, using: - The trigger JSON definition - The associated pipeline references - Target: Databricks job schedule configuration (quartz_cron_expression, periodic, or file_arrival) diff --git a/skills/flowx-convert/references/activity-mapping.md b/skills/flowx-convert/references/activity-mapping.md index 9635220..5fb391b 100644 --- a/skills/flowx-convert/references/activity-mapping.md +++ b/skills/flowx-convert/references/activity-mapping.md @@ -5,7 +5,7 @@ This reference defines the mapping between Azure Data Factory activity types and ## Strategy Definitions - **Deterministic** — Handled by a built-in Python translator module. Fast, reliable, no LLM required. These mappings are well-defined and produce consistent output. -- **Agentic** — Handled by an LLM-assisted skill from the `adf-to-databricks-plugin`. Required when the ADF activity has complex semantics, requires interpretation, or lacks a direct Databricks equivalent. +- **Agentic** — Handled by agentic (LLM-assisted) translation performed by the agent. Required when the ADF activity has complex semantics, requires interpretation, or lacks a direct Databricks equivalent. - **Unsupported** — No automated translation path. Requires manual intervention. ## Activity Mapping Table @@ -26,19 +26,19 @@ This reference defines the mapping between Azure Data Factory activity types and | DatabricksJob | Deterministic | `databricks_job.py` | `run_job_task` | | Switch | Deterministic | `switch.py` | chained `condition_task`s | | Wait | Deterministic | `wait.py` | `notebook_task` (`time.sleep`) | -| ExecuteDataFlow | Agentic | `adf-to-databricks:adf-dataflow-converter` | DLT pipeline or PySpark notebook | -| Until | Agentic | `adf-to-databricks:adf-pipeline-converter` | while-loop notebook | +| ExecuteDataFlow | Agentic | Agentic (LLM-assisted) | DLT pipeline or PySpark notebook | +| Until | Agentic | Agentic (LLM-assisted) | while-loop notebook | | Filter | Deterministic | `filter.py` | `notebook_task` (filter array + task values) | | AppendVariable | Deterministic | `append_variable.py` | `notebook_task` (append to array task value) | -| SqlServerStoredProcedure | Agentic | `adf-to-databricks:adf-pipeline-converter` | SQL notebook | -| AzureFunction | Agentic | `adf-to-databricks:adf-pipeline-converter` | webhook/REST notebook | -| WebHook | Agentic | `adf-to-databricks:adf-pipeline-converter` | REST notebook | -| Custom | Agentic | `adf-to-databricks:adf-pipeline-converter` | custom notebook | -| ExecuteSSISPackage | Agentic | `adf-to-databricks:adf-pipeline-converter` | PySpark notebook | -| AzureMLExecutePipeline | Agentic | `adf-to-databricks:adf-pipeline-converter` | MLflow notebook | -| Triggers (Schedule) | Agentic | `adf-to-databricks:adf-trigger-converter` | `quartz_cron_expression` | -| Triggers (Tumbling Window) | Agentic | `adf-to-databricks:adf-trigger-converter` | periodic schedule | -| Triggers (Blob Event) | Agentic | `adf-to-databricks:adf-trigger-converter` | `file_arrival` trigger | +| SqlServerStoredProcedure | Agentic | Agentic (LLM-assisted) | SQL notebook | +| AzureFunction | Agentic | Agentic (LLM-assisted) | webhook/REST notebook | +| WebHook | Agentic | Agentic (LLM-assisted) | REST notebook | +| Custom | Agentic | Agentic (LLM-assisted) | custom notebook | +| ExecuteSSISPackage | Agentic | Agentic (LLM-assisted) | PySpark notebook | +| AzureMLExecutePipeline | Agentic | Agentic (LLM-assisted) | MLflow notebook | +| Triggers (Schedule) | Agentic | Agentic (LLM-assisted) | `quartz_cron_expression` | +| Triggers (Tumbling Window) | Agentic | Agentic (LLM-assisted) | periodic schedule | +| Triggers (Blob Event) | Agentic | Agentic (LLM-assisted) | `file_arrival` trigger | ## Deterministic Translator Details @@ -151,7 +151,7 @@ Maps to a `notebook_task` that appends a value to an array variable: ## Agentic Translation Notes -Agentic translations are handled by skills from the `adf-to-databricks-plugin` (`birbalin25/adf-to-databricks-plugin`). These skills use LLM reasoning to: +Agentic translations are performed by the agent, using LLM reasoning to: 1. Interpret complex ADF semantics that lack direct Databricks equivalents 2. Convert ADF expressions to Python/SQL equivalents diff --git a/skills/flowx-discover/SKILL.md b/skills/flowx-discover/SKILL.md index f33a992..6ed4187 100644 --- a/skills/flowx-discover/SKILL.md +++ b/skills/flowx-discover/SKILL.md @@ -22,7 +22,7 @@ Parse Azure Data Factory pipeline, dataset, linked service, and trigger JSON fil This is phase 1 of the flowx migration workflow. It takes raw ADF JSON exports and produces an `inventory.json` file that the `convert` skill consumes. The inventory classifies every ADF activity into one of three strategies: - **Deterministic** — a built-in translator exists (Copy, DatabricksNotebook, ForEach, IfCondition, etc.) -- **Agentic** — requires LLM-assisted translation via the `adf-to-databricks-plugin` skills (ExecuteDataFlow, Switch, Until, StoredProc, etc.) +- **Agentic** — requires agentic (LLM-assisted) translation by the agent (ExecuteDataFlow, Switch, Until, StoredProc, etc.) - **Unsupported** — no known translation path; requires manual intervention ## How to run this skill — MCP tool or venv CLI @@ -174,8 +174,7 @@ Read the generated `/metadata/inventory.json` file. It has this stru { "name": "RunDataFlow", "type": "ExecuteDataFlow", - "strategy": "agentic", - "skill": "adf-to-databricks:adf-dataflow-converter" + "strategy": "agentic" } ] } @@ -232,12 +231,12 @@ Coverage: 95.7% ### Step 6 — Detail agentic activities -For activities classified as `agentic`, explain which skill from the `adf-to-databricks-plugin` will handle each: +For activities classified as `agentic`, explain that each is translated by the agent using LLM-assisted reasoning from the activity's ARM JSON (no built-in deterministic translator exists for these types): -| Activity | Type | Handling Skill | +| Activity | Type | Handling | |---|---|---| -| RunDataFlow | ExecuteDataFlow | `adf-to-databricks:adf-dataflow-converter` | -| BranchLogic | Switch | `adf-to-databricks:adf-pipeline-converter` | +| RunDataFlow | ExecuteDataFlow | Agentic (LLM-assisted) | +| BranchLogic | Switch | Agentic (LLM-assisted) | | ... | ... | ... | ### Step 7 — Warn about unsupported activities diff --git a/skills/flowx-migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md index 379a958..db8ee26 100644 --- a/skills/flowx-migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -68,11 +68,11 @@ ADF JSON Exports **Process:** 1. Run deterministic translators for all activities classified as `deterministic` -2. For each `agentic` activity, invoke the appropriate skill from the `adf-to-databricks-plugin`: - - `adf-dataflow-converter` for ExecuteDataFlow activities - - `adf-pipeline-converter` for control flow and external call activities - - `adf-expression-translator` for complex ADF expression conversion - - `adf-trigger-converter` for trigger schedule translation +2. For each `agentic` activity, perform LLM-assisted translation from the activity's ARM JSON: + - ExecuteDataFlow activities → a purpose-built PySpark notebook + - control flow and external-call activities → a notebook implementing the activity's semantics + - complex ADF expressions → Python/SQL equivalents + - trigger schedules → Databricks schedule/trigger configuration 3. Merge deterministic and agentic results into a unified translation report 4. Generate Databricks IR (intermediate representation) for each activity diff --git a/skills/flowx-setup/SKILL.md b/skills/flowx-setup/SKILL.md index f319580..cd0ecf7 100644 --- a/skills/flowx-setup/SKILL.md +++ b/skills/flowx-setup/SKILL.md @@ -44,17 +44,18 @@ fi ## Path A — Databricks Genie Code (MCP, no virtual environment) In Genie Code the phases run on the deployed app, so **do not run `bootstrap.sh` and do not create a -venv** — it isn't needed. Deploy the MCP server instead: +venv** — it isn't needed. Deploy the MCP server instead. -```bash -bash /app/deploy.sh -``` +**Recommended (works on serverless): run the `app/deploy_app.py` notebook.** It uses the Databricks +SDK to stage a self-contained bundle (the app entrypoint plus a vendored copy of the flowx source) +to **`/Workspace/Shared/mcp-flowx`** and create/deploy the **`mcp-flowx`** Databricks App. Set its +`repo_root` widget to the flowx checkout (e.g. `/Workspace/Shared/flowx`). Because it uploads through +the SDK Workspace API, it runs directly in a serverless Genie Code session. The notebook prints the +app URL; the MCP endpoint is `/mcp`. -`app/deploy.sh` stages a self-contained bundle (the app entrypoint plus a vendored copy of the -flowx source), syncs it to **`/Workspace/Shared/mcp-flowx`**, and creates/deploys the -**`mcp-flowx`** Databricks App. The script prints the app URL; the MCP endpoint is -`/mcp`. It only needs the Databricks CLI and a system `python3` (for parsing CLI output) — -**not** an flowx venv. +**CLI alternative:** `bash /app/deploy.sh` does the same via the Databricks CLI +(`apps deploy` / `sync`), but those commands require a CLI session — run it from a workspace web +terminal or a local machine, **not** serverless notebook Python. > **Clone into a shared location.** The app's service principal cannot read private > `/Workspace/Users/` folders by default, so `deploy.sh` deploys the source from @@ -77,11 +78,11 @@ Once added, the `discover`, `convert`, `package`, and `migrate` skills run **ent `flowx` MCP tool** (`flowx(command="…", parameters={…})`) — there is no venv, no `bootstrap.sh`, and no `.migration-venv` marker on this path. -> **Note:** `databricks apps` deploy commands require a Databricks CLI session (workspace web -> terminal or a local machine), not serverless notebook Python. If the Genie session can't shell -> out to the CLI, run `app/deploy.sh` from the web terminal. (Same constraint as `databricks bundle -> deploy`.) If you see `Error: please specify target`, the CLI attached to a stray `databricks.yml`; -> `deploy.sh` already isolates against this, so re-run it as-is. +> **Note:** Prefer the `app/deploy_app.py` notebook in serverless Genie Code — the SDK works there, +> whereas `databricks apps deploy` / `sync` (used by `deploy.sh`) need a CLI session (web terminal +> or local machine). Only fall back to `deploy.sh` from a web terminal. If `deploy.sh` reports +> `Error: please specify target`, the CLI attached to a stray `databricks.yml`; it already isolates +> against this, so re-run it as-is. --- diff --git a/skills/setup/SKILL.md b/skills/setup/SKILL.md deleted file mode 100644 index 3d262c8..0000000 --- a/skills/setup/SKILL.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -name: setup -description: > - Setup the Python environment for the flowx plugin. Creates a .venv virtual environment - and installs the Python dependencies (from requirements.txt via pip) needed for each phase. - Run this once before any other flowx skill, or whenever dependencies are missing. -triggers: - - "setup flowx" - - "bootstrap flowx" - - "install flowx dependencies" - - "flowx environment" - - "create flowx venv" - - "ModuleNotFoundError flowx" ---- - -# Create the Python Environment - -Create a virtual environment (`.venv`) for the plugin and install its Python dependencies. This -is the prerequisite for the `ingest`, `translate`, `prepare`, and `migrate` skills which run Python -from this environment. - -## Context - -The flowx plugin ships Python code (in `src/flowx/`) that the skills invoke (e.g. -`python -m flowx.adapter ...`, `adf_loader.py`, `engine.py`, `dab_writer.py`). Some code depends -on third-party packages (`pyyaml`, `databricks-sdk`, `sqlglot`). Running it against a bare system -Python fails with `ModuleNotFoundError`. This step provisions an isolated `.venv` with the required -dependencies installed via `pip` from `requirements.txt`. - -The environment is created once and reused. Re-running the bootstrapscript confirms the venv exists -and ensures that dependencies are satisfied. - -## Workflow - -### Step 1 — Run the bootstrap script - -From the plugin root, run: - -```bash -bash /scripts/bootstrap.sh -``` - -Where `` is the root of the flowx plugin (the directory containing `src/`, -`skills/`, and `requirements.txt`). - -The script will: -1. Check that `python3`, `pip`, and the `venv` module are available. -2. Create `/.venv` if it does not already exist. -3. Install dependencies listed in `requirements.txt` into that venv using `pip`. - -### Step 2 — Handle a missing Python or pip - -If Python, pip, or the `venv` module are **not** available, the script prints a `WARNING:` block -explaining what to install and exits non-zero **without** creating anything. - -When this happens, **do not attempt to work around it**. Relay the warning to the user, ask them -to install, and stop: - -> ⚠️ Python must be installed before I can set up the flowx environment. -> -> * On macOS: `brew install python`. -> * On Debian/Ubuntu: `sudo apt-get install python3 python3-venv python3-pip`. -> -> Let me know once it's installed and I'll re-run setup. - -Re-run this setup skill after the user confirms Python and pip are installed. - -### Step 3 — Confirm success and how to run Python code - -On success, the script prints the interpreter path and a usage example. After this, every -Python command in the flowx skills **must** be run with the venv interpreter and `src/` -on `PYTHONPATH`: - -```bash -export PYTHONPATH="/src" -"/.venv/bin/python" -m flowx.adapter inputs ingest -``` - -(On Windows the interpreter is `\.venv\Scripts\python.exe`.) - -Use `/.venv/bin/python` anywhere the other skills show `python3`. - -## Output - -| Artifact | Description | -|---|---| -| `/.venv/` | Virtual environment containing the installed dependencies | -| `requirements.txt` | The dependency list installed into the venv | - -## Examples - -- "Set up the flowx environment" -- "Bootstrap flowx so I can run a migration" -- "I got a ModuleNotFoundError running ingest — fix the environment" diff --git a/src/flowx/models/adf_ast.py b/src/flowx/models/adf_ast.py index 3fa8d90..a91ef30 100644 --- a/src/flowx/models/adf_ast.py +++ b/src/flowx/models/adf_ast.py @@ -307,7 +307,6 @@ class InventoryItem: activity_name: Activity display name. activity_type: ADF activity type string. strategy: Determined translation strategy. - agentic_skill: Skill identifier when strategy is ``AGENTIC``. depends_on: Upstream activity names. """ @@ -315,7 +314,6 @@ class InventoryItem: activity_name: str activity_type: str strategy: TranslationStrategy - agentic_skill: str | None = None depends_on: list[str] | None = None diff --git a/src/flowx/models/ir.py b/src/flowx/models/ir.py index 1b272ba..aa1afe3 100644 --- a/src/flowx/models/ir.py +++ b/src/flowx/models/ir.py @@ -482,8 +482,6 @@ class PlaceholderActivity(Activity): original_type: str notebook_path: str = "/UNSUPPORTED_ADF_ACTIVITY" comment: str | None = None - # For an agentic gap (e.g. Until): the recommended skill the agent should translate from. - agentic_skill: str | None = None raw_definition: dict[str, Any] | None = None @@ -732,13 +730,11 @@ class AgenticGap: Attributes: activity_name: Display name of the activity. activity_type: ADF activity type string. - recommended_skill: Skill identifier to use for translation. raw_definition: Original ADF JSON definition for the activity. """ activity_name: str activity_type: str - recommended_skill: str | None = None raw_definition: dict[str, Any] | None = None diff --git a/src/flowx/parser/adf_loader.py b/src/flowx/parser/adf_loader.py index 30c7c6e..ab760b3 100644 --- a/src/flowx/parser/adf_loader.py +++ b/src/flowx/parser/adf_loader.py @@ -55,19 +55,19 @@ "AppendVariable", } -AGENTIC_TYPES: dict[str, str] = { - "ExecuteDataFlow": "adf-to-databricks:adf-dataflow-converter", - "Until": "adf-to-databricks:adf-pipeline-converter", - "SqlServerStoredProcedure": "adf-to-databricks:adf-pipeline-converter", - "AzureFunction": "adf-to-databricks:adf-pipeline-converter", - "WebHook": "adf-to-databricks:adf-pipeline-converter", - "Custom": "adf-to-databricks:adf-pipeline-converter", - "ExecuteSSISPackage": "adf-to-databricks:adf-pipeline-converter", - "AzureMLExecutePipeline": "adf-to-databricks:adf-pipeline-converter", - "GetMetadata": "adf-to-databricks:adf-pipeline-converter", - "Validation": "adf-to-databricks:adf-pipeline-converter", - "Fail": "adf-to-databricks:adf-pipeline-converter", - "Script": "adf-to-databricks:adf-pipeline-converter", +AGENTIC_TYPES: set[str] = { + "ExecuteDataFlow", + "Until", + "SqlServerStoredProcedure", + "AzureFunction", + "WebHook", + "Custom", + "ExecuteSSISPackage", + "AzureMLExecutePipeline", + "GetMetadata", + "Validation", + "Fail", + "Script", } # Activity complexity weights (easiest first): Databricks-native ~1:1 tasks, then control-flow, then @@ -195,21 +195,20 @@ def _parse_factory_global_parameters(data: dict[str, Any]) -> dict[str, Any]: return result -def classify_activity(activity_type: str) -> tuple[TranslationStrategy, str | None]: +def classify_activity(activity_type: str) -> TranslationStrategy: """Classify an ADF activity type into a translation strategy. Args: activity_type: ADF activity type string (e.g. ``"Copy"``). Returns: - A ``(strategy, agentic_skill_name)`` tuple. *agentic_skill_name* is - ``None`` for deterministic and unsupported strategies. + The :class:`TranslationStrategy` for the activity type. """ if activity_type in DETERMINISTIC_TYPES: - return TranslationStrategy.DETERMINISTIC, None + return TranslationStrategy.DETERMINISTIC if activity_type in AGENTIC_TYPES: - return TranslationStrategy.AGENTIC, AGENTIC_TYPES[activity_type] - return TranslationStrategy.UNSUPPORTED, None + return TranslationStrategy.AGENTIC + return TranslationStrategy.UNSUPPORTED def build_inventory(definitions: AdfDefinitions) -> Inventory: @@ -654,7 +653,7 @@ def _classify_activities( items: Accumulator list to append results to. """ for activity in activities: - strategy, skill = classify_activity(activity.type) + strategy = classify_activity(activity.type) dep_names = [dependency.activity for dependency in activity.depends_on] if activity.depends_on else None items.append( @@ -663,7 +662,6 @@ def _classify_activities( activity_name=activity.name, activity_type=activity.type, strategy=strategy, - agentic_skill=skill, depends_on=dep_names, ) ) @@ -698,8 +696,6 @@ def _inventory_to_dict(inventory: Inventory, source_dir: str) -> dict[str, Any]: "type": item.activity_type, "strategy": item.strategy.value, } - if item.agentic_skill: - entry["skill"] = item.agentic_skill if item.depends_on: entry["depends_on"] = item.depends_on pipeline_map.setdefault(item.pipeline_name, []).append(entry) @@ -901,7 +897,7 @@ def write_pipeline_arm(definitions: AdfDefinitions, metadata_dir: Path) -> list[ # CLI entry point # --------------------------------------------------------------------------- -# Flowx-managed entries under the shared output_dir, cleared at the start of each fresh run. +# flowx-managed entries under the shared output_dir, cleared at the start of each fresh run. _MANAGED_OUTPUT_DIRS: tuple[str, ...] = ("metadata", ".work", "resources", "src", "setup") _MANAGED_OUTPUT_FILES: tuple[str, ...] = ("databricks.yml", "SETUP.md", "WARNINGS.md") diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py index 8b460b3..6148cbf 100644 --- a/src/flowx/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -214,12 +214,10 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: """Returns a PreparedActivity with a stub notebook for an unsupported activity.""" task = build_common_task_fields(activity) - agentic_skill: str | None = None raw_definition: dict[str, Any] | None = None if isinstance(activity, PlaceholderActivity): comment = activity.comment or "This activity requires manual implementation." original_type = activity.original_type - agentic_skill = activity.agentic_skill raw_definition = activity.raw_definition elif isinstance(activity, UnsupportedActivity): comment = activity.reason or "This activity type is not supported." @@ -237,11 +235,10 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: if raw_definition is not None: import json as _json - skill_hint = f" using `{agentic_skill}`" if agentic_skill else "" arm_lines = _json.dumps(raw_definition, indent=2).splitlines() arm_block = ( "# MAGIC\n" - f"# MAGIC An agent should translate this activity{skill_hint} from the ADF/ARM JSON below,\n" + "# MAGIC An agent should translate this activity from the ADF/ARM JSON below,\n" "# MAGIC then replace the `raise NotImplementedError` cell with the generated code.\n" "# MAGIC\n" "# MAGIC ```json\n" + "".join(f"# MAGIC {line}\n" for line in arm_lines) + "# MAGIC ```\n" diff --git a/src/flowx/translator/engine.py b/src/flowx/translator/engine.py index b99c318..b45e30e 100644 --- a/src/flowx/translator/engine.py +++ b/src/flowx/translator/engine.py @@ -159,7 +159,7 @@ def translate_pipeline( activity_ir, context = _dispatch_activity(adf_activity, context, definitions) translated_activities.append(activity_ir) - strategy, skill = classify_activity(adf_activity.type) + strategy = classify_activity(adf_activity.type) if strategy is TranslationStrategy.DETERMINISTIC: deterministic_count += 1 elif strategy is TranslationStrategy.AGENTIC: @@ -267,14 +267,13 @@ def _collect_agentic_gaps(activities: list[AdfActivity], warnings: list[str]) -> def _walk(acts: list[AdfActivity] | None) -> None: for act in acts or []: - strategy, skill = classify_activity(act.type) + strategy = classify_activity(act.type) if strategy is not TranslationStrategy.DETERMINISTIC and act.name not in seen: seen.add(act.name) gaps.append( AgenticGap( activity_name=act.name, activity_type=act.type, - recommended_skill=skill, raw_definition=act.raw if act.raw is not None else act.type_properties, ) ) @@ -366,13 +365,16 @@ def _dispatch_activity( return result, context case _: - strategy, skill = classify_activity(activity.type) - reason = f"Agentic skill: {skill}" if skill else f"No translator for type '{activity.type}'" + strategy = classify_activity(activity.type) + reason = ( + "Requires agentic (LLM-assisted) translation from the ADF/ARM JSON" + if strategy is TranslationStrategy.AGENTIC + else f"No translator for type '{activity.type}'" + ) placeholder = PlaceholderActivity( **base_kwargs, original_type=activity.type, comment=reason, - agentic_skill=skill, raw_definition=activity.raw, ) context = context.with_activity(activity.name, placeholder) diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py index a89456f..f729c3a 100644 --- a/tests/integration/test_end_to_end.py +++ b/tests/integration/test_end_to_end.py @@ -313,14 +313,10 @@ def test_deterministic_count(self, adf_definitions): assert len(det_items) == inv.deterministic_count def test_agentic_activities_identified(self, adf_definitions): - """Agentic activities are identified with correct skill mapping.""" + """Agentic activities are identified and counted.""" inv = build_inventory(adf_definitions) agentic_items = [i for i in inv.items if i.strategy is TranslationStrategy.AGENTIC] assert len(agentic_items) == inv.agentic_count - for item in agentic_items: - assert item.agentic_skill is not None - # All agentic skills should reference a known skill - assert "adf-to-databricks" in item.agentic_skill def test_mixed_pipeline_classification(self, adf_definitions): """Mixed pipeline has both deterministic and agentic items.""" diff --git a/tests/unit/test_adf_loader.py b/tests/unit/test_adf_loader.py index f56b4d8..8213742 100644 --- a/tests/unit/test_adf_loader.py +++ b/tests/unit/test_adf_loader.py @@ -83,23 +83,20 @@ def test_classify_deterministic_types(self): assert DETERMINISTIC_TYPES == expected for atype in expected: - strategy, skill = classify_activity(atype) + strategy = classify_activity(atype) assert strategy is TranslationStrategy.DETERMINISTIC, f"{atype} should be DETERMINISTIC" - assert skill is None, f"{atype} should have no agentic skill" def test_classify_agentic_types(self): - """All agentic types are classified with correct skill names.""" - for atype, expected_skill in AGENTIC_TYPES.items(): - strategy, skill = classify_activity(atype) + """All agentic types are classified as AGENTIC.""" + for atype in AGENTIC_TYPES: + strategy = classify_activity(atype) assert strategy is TranslationStrategy.AGENTIC, f"{atype} should be AGENTIC" - assert skill == expected_skill, f"{atype} skill should be {expected_skill}" def test_classify_unknown_types(self): """Unknown activity types are classified as UNSUPPORTED.""" for unknown_type in ("Bogus", "SomeFutureActivity", "MagicTransform", ""): - strategy, skill = classify_activity(unknown_type) + strategy = classify_activity(unknown_type) assert strategy is TranslationStrategy.UNSUPPORTED - assert skill is None # --------------------------------------------------------------------------- @@ -124,19 +121,11 @@ def test_build_inventory_has_pipeline_names(self, adf_definitions): for item in inv.items: assert item.pipeline_name in pipeline_names - def test_build_inventory_agentic_items_have_skills(self, adf_definitions): - """Agentic inventory items have a non-None skill.""" + def test_build_inventory_agentic_count_matches_items(self, adf_definitions): + """The agentic count matches the number of items classified AGENTIC.""" inv = build_inventory(adf_definitions) - for item in inv.items: - if item.strategy is TranslationStrategy.AGENTIC: - assert item.agentic_skill is not None - - def test_build_inventory_deterministic_items_no_skill(self, adf_definitions): - """Deterministic inventory items have no agentic skill.""" - inv = build_inventory(adf_definitions) - for item in inv.items: - if item.strategy is TranslationStrategy.DETERMINISTIC: - assert item.agentic_skill is None + agentic_items = [i for i in inv.items if i.strategy is TranslationStrategy.AGENTIC] + assert len(agentic_items) == inv.agentic_count # --------------------------------------------------------------------------- diff --git a/tests/unit/test_until_agentic_handler.py b/tests/unit/test_until_agentic_handler.py index aa97a05..64d02e8 100644 --- a/tests/unit/test_until_agentic_handler.py +++ b/tests/unit/test_until_agentic_handler.py @@ -48,7 +48,6 @@ def test_nested_until_gap_carries_full_arm_json(): # full ARM JSON, not just typeProperties: name + nested loop body present assert raw.get("name") == "Poll Until Ready" assert raw["typeProperties"]["activities"][0]["name"] == "Wait A Bit" - assert until_gaps[0].recommended_skill == "adf-to-databricks:adf-pipeline-converter" def test_until_placeholder_ir_node_carries_arm_json(): @@ -68,4 +67,3 @@ def _find(tasks): ph = _find(report.pipeline.tasks) assert ph is not None and ph.raw_definition is not None assert ph.raw_definition.get("type") == "Until" - assert ph.agentic_skill == "adf-to-databricks:adf-pipeline-converter" From c9c764fd7f879e0992854f4c66a17e1014ea9c26 Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:58:23 -0400 Subject: [PATCH 20/77] Update MCP deployment to use Databricks SDK (#5) * Refactor expression parsing (#1) * Improve control flow conversion (#2) Improves conversion of control flow, expression-based parameters, and schedule triggers. Co-authored-by: Isaac * Merge external (#3) * Revert "Update GitHub actions (#10)" This reverts commit 3c9cb71f274c6db55a3bfb4cd06a3b7768f2fc12. * Improve expression resolution, control flow parsing, and setup (#11) * Refactor expression parsing (#1) * Improve control flow conversion (#2) * Update repo structure * Format modules * Initial commit * Initial release * Improve preparer coverage for managed ingestion pipelines (#2) * Lakeflow connect ingestion pipeline handling * Add required connection parameters * Add marketplace file (#3) --------- Co-authored-by: Greg Hansen <163584195+ghanse@users.noreply.github.com> Co-authored-by: service-jira-pub-repo-auto * Add SDK-based installation --------- Co-authored-by: service-jira-pub-repo-auto --- README.md | 148 +++++++----- app/README.md | 34 ++- app/deploy_app.py | 115 +++++++++ app/deploy_helpers.py | 120 ++++++++++ docs/README.md | 2 +- docs/app/(home)/page.tsx | 2 +- docs/app/layout.config.tsx | 4 +- docs/content/docs/architecture.mdx | 2 +- docs/content/docs/installation.mdx | 224 ++++++++---------- scripts/bootstrap.sh | 6 +- skills/flowx-convert/SKILL.md | 23 +- .../references/activity-mapping.md | 26 +- skills/flowx-discover/SKILL.md | 13 +- skills/flowx-migrate/references/workflow.md | 10 +- skills/flowx-setup/SKILL.md | 29 +-- src/flowx/models/adf_ast.py | 2 - src/flowx/models/ir.py | 4 - src/flowx/parser/adf_loader.py | 44 ++-- src/flowx/preparer/workflow_preparer.py | 5 +- src/flowx/translator/engine.py | 14 +- tests/integration/test_end_to_end.py | 6 +- tests/unit/test_adf_loader.py | 29 +-- tests/unit/test_until_agentic_handler.py | 2 - 23 files changed, 554 insertions(+), 310 deletions(-) create mode 100644 app/deploy_app.py create mode 100644 app/deploy_helpers.py diff --git a/README.md b/README.md index 356fa27..ea4888e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # flowx -ADF to Databricks Lakeflow Jobs translator via Declarative Automation Bundles. +ADF to Databricks Lakeflow Jobs translator, delivered as agent skills. -flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). It deterministically translates known activity types and falls back to agentic LLM-assisted translation for complex or rare types. +flowx converts Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). It deterministically translates known activity types and falls back to agentic (LLM-assisted) translation for complex or rare types. flowx runs as a set of [agent skills](skills/) usable from Databricks Genie Code, Claude Code, or any tool that supports the Agent Skills standard. ## Architecture @@ -10,23 +10,23 @@ flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline de flowx Pipeline ================== - ADF JSON (UC Volumes) + ADF JSON (UC Volumes / Workspace) | v +------------------+ - | 1. PROFILE | Parse ADF ARM/JSON exports + | 1. DISCOVER | Parse ADF ARM/JSON exports | adf_loader.py | -> Typed AST -> metadata/inventory.json +------------------+ | v +------------------+ - | 2. TRANSLATE | Registry dispatch + topological sort + | 2. CONVERT | Registry dispatch + topological sort | engine.py | -> Pipeline IR (deterministic + agentic gaps) +------------------+ | v +------------------+ - | 3. PREPARE | IR -> DAB YAML + notebooks + setup scripts + | 3. PACKAGE | IR -> DAB YAML + notebooks + setup scripts | dab_writer.py | -> Deployable DABs project +------------------+ | @@ -34,34 +34,66 @@ flowx is a Claude Code plugin that converts Azure Data Factory (ADF) pipeline de databricks bundle validate / deploy ``` -## Quick Start - -1. Add the flowx marketplace and install the plugin in Claude Code: - ``` - /plugin marketplace add databricks-solutions/flowx - /plugin install flowx@flowx - ``` - Then run `/reload-plugins` to activate it. - -2. Set up the runtime (run once): - ``` - /flowx:flowx-setup - ``` - This auto-detects your environment and prepares the right execution path — - a local Python virtual environment for Claude Code, or a deployed MCP server - for Databricks Genie Code. See [Setup](#setup) for details. - -3. Run the end-to-end migration: - ``` - /flowx:flowx-migrate - ``` - - Or run individual phases: - ``` - /flowx:flowx-discover # Parse ADF JSON, produce inventory + complexity report - /flowx:flowx-convert # Deterministic + agentic translation - /flowx:flowx-package # Generate DABs project - ``` +The phases are exposed two ways: as **skills** the agent runs directly (via a Python virtual +environment locally), and as a single **MCP tool** hosted on a Databricks App (for Genie Code). See +[Running flowx as an MCP server](#running-flowx-as-an-mcp-server). + +## Installation + +flowx installs in one of two shapes depending on where your agent runs. Full, step-by-step +instructions for both are in the [installation docs](docs/content/docs/installation.mdx); the +summary: + +### Databricks Genie Code + +The phases run as an MCP server (a Databricks App). Clone the repo into **`/Workspace/Shared`** +(so the app's service principal can read the source), copy `skills/` into your skills folder, then +run the setup skill: + +``` +@flowx-setup +``` + +On Databricks, `flowx-setup` deploys the `mcp-flowx` app for you. You can also deploy it directly by +running the **`app/deploy_app.py`** notebook (SDK-based, works on serverless), or `app/deploy.sh` +from a workspace web terminal. Then add the app under Genie Code **Settings → MCP Servers → Add +Server → Custom MCP server**. + +### Claude Code (and other local agent harnesses) + +flowx is a Claude Code plugin distributed through its marketplace: + +``` +/plugin marketplace add databricks-solutions/flowx +/plugin install flowx@flowx +``` + +Run `/reload-plugins`, then set up the local runtime once: + +``` +/flowx:flowx-setup +``` + +This provisions a Python virtual environment (via `scripts/bootstrap.sh`) and writes a +`.migration-venv` marker the phase skills read. No `uv` is required for plugin users. + +## Usage + +Run the end-to-end migration: + +``` +/flowx:flowx-migrate +``` + +Or run individual phases: + +``` +/flowx:flowx-discover # Parse ADF JSON, produce inventory + complexity report +/flowx:flowx-convert # Deterministic + agentic translation +/flowx:flowx-package # Generate DABs project +``` + +(In Genie Code, invoke the same skills with the `@` prefix, e.g. `@flowx-migrate`.) ## Setup @@ -110,25 +142,28 @@ missing. ### Agentic Fallback (12 types) +Activities with complex semantics, or without a direct Databricks equivalent, are translated by the +agent using LLM-assisted reasoning from the activity's ARM JSON. + | ADF Activity | Strategy | |---|---| -| ExecuteDataFlow | LLM-assisted via adf-to-databricks-plugin | -| SqlServerStoredProcedure | LLM-assisted via adf-to-databricks-plugin | -| AzureFunction | LLM-assisted via adf-to-databricks-plugin | -| WebHook | LLM-assisted via adf-to-databricks-plugin | -| Custom | LLM-assisted via adf-to-databricks-plugin | -| ExecuteSSISPackage | LLM-assisted via adf-to-databricks-plugin | -| AzureMLExecutePipeline | LLM-assisted via adf-to-databricks-plugin | -| GetMetadata | LLM-assisted via adf-to-databricks-plugin | -| Validation | LLM-assisted via adf-to-databricks-plugin | -| Fail | LLM-assisted via adf-to-databricks-plugin | -| Script | LLM-assisted via adf-to-databricks-plugin | -| Until | LLM-assisted via adf-to-databricks-plugin | +| ExecuteDataFlow | LLM-assisted (agentic) | +| SqlServerStoredProcedure | LLM-assisted (agentic) | +| AzureFunction | LLM-assisted (agentic) | +| WebHook | LLM-assisted (agentic) | +| Custom | LLM-assisted (agentic) | +| ExecuteSSISPackage | LLM-assisted (agentic) | +| AzureMLExecutePipeline | LLM-assisted (agentic) | +| GetMetadata | LLM-assisted (agentic) | +| Validation | LLM-assisted (agentic) | +| Fail | LLM-assisted (agentic) | +| Script | LLM-assisted (agentic) | +| Until | LLM-assisted (agentic) | ## How It Works ### Phase 1: Discover -Reads ADF JSON definitions from Unity Catalog volumes, normalizes ARM template format, parses into typed AST nodes, and classifies each activity as deterministic, agentic, or unsupported. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`. +Reads ADF JSON definitions from Unity Catalog volumes (or a `/Workspace` Git folder), normalizes ARM template format, parses into typed AST nodes, and classifies each activity as deterministic, agentic, or unsupported. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`. ### Phase 2: Convert Applies deterministic translators via registry dispatch, resolves dependencies through topological sort, and threads immutable `TranslationContext` through control-flow visitors. Agentic gaps are flagged for LLM-assisted translation. Produces Pipeline IR. @@ -157,16 +192,24 @@ flowx_output/ SETUP.md # Setup instructions (package) metadata/ inventory.json # discover: activity inventory - profile_report.csv # profile: per-pipeline complexity report + profile_report.csv # discover: per-pipeline complexity report .arm.json # discover: verbatim original ADF/ARM source configuration.json # modify: collected configuration answers - .work/ # transient intermediates (translation report, IR, gaps.json); pruned by prepare + .work/ # transient intermediates (translation report, IR, gaps.json); pruned by package ``` +## Running flowx as an MCP server + +The phases are also packaged as [Model Context Protocol](https://modelcontextprotocol.io) tools (in +[`src/flowx/mcp/`](src/flowx/mcp)) so an agent can invoke them directly instead of shelling out to +the CLI. The server exposes a single `flowx(command, parameters)` tool to stay under host tool +limits. For Databricks Genie Code it runs as a Databricks App; see the [app README](app/README.md) +for deployment (SDK notebook or CLI script) and Genie Code registration. + ## Development ```bash -make dev # Install dependencies +make dev # Install dependencies (uses uv) make test # Run unit tests make integration # Run integration tests make fmt # Format + lint (ruff + mypy) @@ -177,9 +220,8 @@ make clean # Remove build artifacts - Python 3.12+ - [uv](https://docs.astral.sh/uv/) package manager -These prerequisites are for contributing to the flowx project. Plugin *users* do not need -`uv` — `/flowx:flowx-setup` provisions the runtime (a pip-based `.venv` locally, or -the MCP server on Databricks). +These prerequisites are for contributing to the flowx project. Plugin *users* do not need `uv` — +`flowx-setup` provisions the runtime (a pip-based `.venv` locally, or the MCP server on Databricks). ## Contributing diff --git a/app/README.md b/app/README.md index 6f52aa3..c60cbf7 100644 --- a/app/README.md +++ b/app/README.md @@ -57,14 +57,25 @@ Register the stdio server with a local MCP client, e.g.: ## Deploy as a Databricks App (for Genie Code) +**Recommended — the `deploy_app.py` notebook (SDK, runs on serverless).** Open `app/deploy_app.py` +in the workspace and run it, setting the `repo_root` widget to the flowx checkout +(e.g. `/Workspace/Shared/flowx`). It uses the Databricks SDK to stage a self-contained bundle (the +app entrypoint plus a vendored copy of the pure-Python `flowx` package) to `/Workspace/Shared/mcp-flowx` +and create/deploy the app (default name **`mcp-flowx`**). Because it uploads through the SDK Workspace +API, it runs directly in a serverless Genie Code session — no CLI needed. The deployment logic lives in +`app/deploy_helpers.py` (imported by the notebook, so it is also unit-testable). + +**CLI alternative — `deploy.sh`:** + ```bash -# Authenticated Databricks CLI (v0.230+) required. +# Authenticated Databricks CLI (v0.230+) required — run from a web terminal or local machine. ./app/deploy.sh ``` -`deploy.sh` stages a self-contained bundle in a temporary directory outside the repo (the app entrypoint -plus a vendored copy of the pure-Python `flowx` package), syncs it to your -workspace, and creates/deploys the app (default name **`mcp-flowx`**). +`deploy.sh` stages a self-contained bundle in a temporary directory outside the repo, syncs it to your +workspace via `databricks sync`, and creates/deploys the app. `databricks apps deploy` / `sync` are +not available from serverless notebook Python, which is why the notebook above is preferred inside +Genie Code. > **Clone into `/Workspace/Shared`.** `deploy.sh` deploys the app source from > `/Workspace/Shared/` because the app's service principal **cannot read @@ -75,9 +86,9 @@ workspace, and creates/deploys the app (default name **`mcp-flowx`**). End-to-end, to use it from Genie Code: -1. **Deploy** with `./app/deploy.sh`. The app is named `mcp-flowx` and deploys the - source from `/Workspace/Shared/mcp-flowx` (override with `APP_SOURCE_PATH`). The - script prints the app URL; the MCP endpoint is `/mcp`. +1. **Deploy** by running the `app/deploy_app.py` notebook (or `./app/deploy.sh` from a CLI session). + The app is named `mcp-flowx` and deploys the source from `/Workspace/Shared/mcp-flowx`. The + deployer prints the app URL; the MCP endpoint is `/mcp`. 2. **Grant app access:** give **Can use** on `mcp-flowx` to the users / service principals that will call it (Apps UI → *Permissions*, or `databricks apps set-permissions mcp-flowx ...`). @@ -100,10 +111,11 @@ set the app env var `FLOWX_ALLOWED_ORIGINS` to your workspace URL and redeploy. MCP access is capped at **20 tools** across all servers; flowx exposes just **one** tool (`flowx`, with 12 commands), so it uses a single slot. -> Run `./app/deploy.sh` from a Databricks CLI session (workspace web terminal or a -> local machine) — `databricks apps` deploy is not available from serverless -> notebook Python. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp) -> and [host a custom MCP server](https://docs.databricks.com/aws/en/generative-ai/mcp/custom-mcp). +> In serverless Genie Code, deploy with the `app/deploy_app.py` notebook (SDK-based). `./app/deploy.sh` +> must run from a Databricks CLI session (workspace web terminal or a local machine), since +> `databricks apps` deploy is not available from serverless notebook Python. See [Connect Genie Code +> to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp) and [host a +> custom MCP server](https://docs.databricks.com/aws/en/generative-ai/mcp/custom-mcp). ## Troubleshooting diff --git a/app/deploy_app.py b/app/deploy_app.py new file mode 100644 index 0000000..335993f --- /dev/null +++ b/app/deploy_app.py @@ -0,0 +1,115 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC # Deploy the flowx MCP server (SDK, no CLI) +# MAGIC +# MAGIC Deploys the flowx MCP server as a Databricks App using the **Python SDK** only, so it runs +# MAGIC directly from a **serverless** Databricks / Genie Code session — where `app/deploy.sh` cannot, +# MAGIC because `databricks apps deploy` / `databricks sync` require a CLI session (web terminal or a +# MAGIC local machine). +# MAGIC +# MAGIC The deployment logic lives in `deploy_helpers.py` (imported below) so it stays importable and +# MAGIC unit-testable. Because the bundle is uploaded through the **Workspace API** (`ImportFormat.RAW`), +# MAGIC there is no `databricks sync` and none of its footguns: no `.gitignore` dropping staged files, +# MAGIC and no stray `databricks.yml` making the CLI abort with "please specify target". + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Configuration +# MAGIC +# MAGIC - **repo_root** *(required)* — the flowx checkout: the directory containing `app/` and +# MAGIC `src/flowx` (e.g. `/Workspace/Shared/flowx`). +# MAGIC - **app_name** — the app to create/deploy (default `mcp-flowx`; the `mcp-` prefix auto-lists it +# MAGIC in the AI Playground). +# MAGIC - **source_code_path** — workspace dir the app deploys from. Must be readable by the app's +# MAGIC service principal, so it defaults under `/Workspace/Shared` — **not** a private +# MAGIC `/Workspace/Users/` home, which the app SP cannot read. + +# COMMAND ---------- + +dbutils.widgets.text("repo_root", "/Workspace/Shared/flowx", "flowx repo root (contains app/ and src/flowx)") +dbutils.widgets.text("app_name", "mcp-flowx", "App name") +dbutils.widgets.text("source_code_path", "", "Workspace source path (blank = /Workspace/Shared/)") + +repo_root = dbutils.widgets.get("repo_root").strip() +app_name = dbutils.widgets.get("app_name").strip() +source_code_path = dbutils.widgets.get("source_code_path").strip() or f"/Workspace/Shared/{app_name}" +app_description = "flowx MCP server — ADF to Databricks Lakeflow migration tools" + +# COMMAND ---------- + +import os +import sys + +# Make the sibling deploy_helpers module importable from the user-specified checkout. +sys.path.insert(0, os.path.join(repo_root, "app")) + +from databricks.sdk import WorkspaceClient + +import deploy_helpers + +deploy_helpers.validate_repo_root(repo_root) +workspace_client = WorkspaceClient() + +print(f"repo_root : {repo_root}") +print(f"app_name : {app_name}") +print(f"source_code_path : {source_code_path}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Stage the self-contained bundle at the workspace source path +# MAGIC +# MAGIC Uploads the app files and a fresh copy of the `flowx` package to `source_code_path`. +# MAGIC `ImportFormat.RAW` lands each file verbatim (not as a notebook) and `overwrite=True` makes +# MAGIC redeploys idempotent. + +# COMMAND ---------- + +flowx_module_count = deploy_helpers.stage_app_bundle(workspace_client, repo_root, source_code_path) +print(f"Staged app files + {flowx_module_count} flowx module files to {source_code_path}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Create the app if needed, then deploy +# MAGIC +# MAGIC `create_and_wait` / `deploy_and_wait` block until the operation reaches a terminal state +# MAGIC (default 20-minute timeout each). + +# COMMAND ---------- + +was_created = deploy_helpers.ensure_app_exists(workspace_client, app_name, app_description) +print(f"App '{app_name}' {'created' if was_created else 'already existed — reusing it'}.") + +print(f"Deploying '{app_name}' from {source_code_path} ...") +deployment = deploy_helpers.deploy_app_source(workspace_client, app_name, source_code_path) + +deployed_app = workspace_client.apps.get(name=app_name) +app_url = deployed_app.url or "" +deployment_state = deployment.status.state if deployment.status else "unknown" +print(f"Deployment status: {deployment_state}") +print(f"App URL : {app_url or '(pending — re-run workspace_client.apps.get)'}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Next steps to use it in Genie Code + +# COMMAND ---------- + +print(f""" +Deployed app: {app_name} +MCP endpoint: {app_url or ''}/mcp +Health check: {app_url or ''}/ + +Next steps: + 1. App access — grant 'Can use' on '{app_name}' to the users / service principals that will + call it (Apps UI > Permissions). + 2. Data access — grant the app's service principal read/write on the catalogs, schemas, and + UC volumes the migration touches (and any SQL warehouse used by the reporting tools). + 3. Add it in Genie Code (Agent mode): Settings > MCP Servers > Add Server > Custom MCP server + > select '{app_name}' > Save. The single 'flowx' tool appears immediately. + 4. If a browser CORS error appears, set the app env var FLOWX_ALLOWED_ORIGINS to your + workspace URL and redeploy (re-run this notebook). +""") diff --git a/app/deploy_helpers.py b/app/deploy_helpers.py new file mode 100644 index 0000000..fedcdbf --- /dev/null +++ b/app/deploy_helpers.py @@ -0,0 +1,120 @@ +"""Helpers for deploying the flowx MCP server as a Databricks App via the Python SDK. + +Kept separate from ``deploy_app`` (the deployment notebook) so the deployment logic is +importable and unit-testable outside a notebook session. Every function takes an explicit +``WorkspaceClient`` and plain paths, so nothing here depends on ``dbutils`` or a notebook +runtime. + +Mirrors what ``deploy.sh`` does, minus the CLI: assemble a self-contained source bundle +(``app.py``, ``app.yaml``, ``requirements.txt``, and a vendored copy of the ``flowx`` package) +at a workspace path, then create/deploy the app from it. +""" + +import os + +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.apps import App, AppDeployment, AppDeploymentMode +from databricks.sdk.service.workspace import ImportFormat + +# Top-level files the app needs alongside the vendored flowx package. +APP_BUNDLE_FILES = ("app.py", "app.yaml", "requirements.txt") + + +def validate_repo_root(repo_root: str) -> None: + """Raise if ``repo_root`` is not a flowx checkout (missing app/app.py or src/flowx).""" + if not repo_root: + raise ValueError( + "repo_root is required: set it to the flowx checkout — the directory " + "containing app/ and src/flowx." + ) + has_app_entrypoint = os.path.isfile(os.path.join(repo_root, "app", "app.py")) + has_flowx_package = os.path.isdir(os.path.join(repo_root, "src", "flowx")) + if not (has_app_entrypoint and has_flowx_package): + raise ValueError( + f"{repo_root!r} does not look like a flowx checkout " + "(expected app/app.py and src/flowx)." + ) + + +def upload_file(workspace_client: WorkspaceClient, local_path: str, workspace_path: str) -> None: + """Upload one local file to a workspace path verbatim (RAW), creating parent dirs.""" + workspace_client.workspace.mkdirs(os.path.dirname(workspace_path)) + with open(local_path, "rb") as file_handle: + workspace_client.workspace.upload( + workspace_path, file_handle, format=ImportFormat.RAW, overwrite=True + ) + + +def upload_directory( + workspace_client: WorkspaceClient, local_directory: str, workspace_directory: str +) -> int: + """Recursively upload a directory tree, skipping ``__pycache__`` and ``.pyc`` files. + + Returns the number of files uploaded. + """ + uploaded_file_count = 0 + for current_directory, subdirectories, file_names in os.walk(local_directory): + subdirectories[:] = [name for name in subdirectories if name != "__pycache__"] + for file_name in file_names: + if file_name.endswith(".pyc"): + continue + local_path = os.path.join(current_directory, file_name) + relative_path = os.path.relpath(local_path, local_directory) + workspace_path = f"{workspace_directory}/{relative_path.replace(os.sep, '/')}" + upload_file(workspace_client, local_path, workspace_path) + uploaded_file_count += 1 + return uploaded_file_count + + +def stage_app_bundle( + workspace_client: WorkspaceClient, repo_root: str, source_code_path: str +) -> int: + """Assemble the self-contained app bundle at ``source_code_path`` in the workspace. + + Uploads the top-level app files and a fresh copy of the ``flowx`` package (the prior + copy is deleted first so removed modules do not linger). Returns the flowx module count. + """ + app_directory = os.path.join(repo_root, "app") + workspace_client.workspace.mkdirs(source_code_path) + + for file_name in APP_BUNDLE_FILES: + upload_file( + workspace_client, + os.path.join(app_directory, file_name), + f"{source_code_path}/{file_name}", + ) + + package_source_directory = os.path.join(repo_root, "src", "flowx") + package_workspace_directory = f"{source_code_path}/flowx" + try: + workspace_client.workspace.delete(package_workspace_directory, recursive=True) + except Exception: + pass # first deploy: nothing to delete + return upload_directory( + workspace_client, package_source_directory, package_workspace_directory + ) + + +def ensure_app_exists( + workspace_client: WorkspaceClient, app_name: str, app_description: str +) -> bool: + """Create the app if it does not already exist. Returns True if it was created.""" + try: + workspace_client.apps.get(name=app_name) + return False + except Exception: + workspace_client.apps.create_and_wait(App(name=app_name, description=app_description)) + return True + + +def deploy_app_source( + workspace_client: WorkspaceClient, app_name: str, source_code_path: str +) -> AppDeployment: + """Deploy the staged source to the app and block until deployment reaches terminal state.""" + return workspace_client.apps.deploy_and_wait( + app_name=app_name, + app_deployment=AppDeployment( + source_code_path=source_code_path, + mode=AppDeploymentMode.SNAPSHOT, + ), + ) diff --git a/docs/README.md b/docs/README.md index f151cad..1434dba 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # flowx docs -Documentation site for [flowx](https://github.com/ghanse/flowx), built with [fumadocs](https://fumadocs.dev) and deployed to GitHub Pages. +Documentation site for [flowx](https://github.com/databricks-solutions/flowx), built with [fumadocs](https://fumadocs.dev) and deployed to GitHub Pages. ## Local development diff --git a/docs/app/(home)/page.tsx b/docs/app/(home)/page.tsx index 021cb6b..9365553 100644 --- a/docs/app/(home)/page.tsx +++ b/docs/app/(home)/page.tsx @@ -16,7 +16,7 @@ export default function HomePage() { Read the docs View on GitHub diff --git a/docs/app/layout.config.tsx b/docs/app/layout.config.tsx index 9741c3e..39c5777 100644 --- a/docs/app/layout.config.tsx +++ b/docs/app/layout.config.tsx @@ -14,9 +14,9 @@ export const baseOptions: BaseLayoutProps = { }, { text: 'GitHub', - url: 'https://github.com/ghanse/flowx', + url: 'https://github.com/databricks-solutions/flowx', external: true, }, ], - githubUrl: 'https://github.com/ghanse/flowx', + githubUrl: 'https://github.com/databricks-solutions/flowx', }; diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx index bed4d9e..fbdb0f8 100644 --- a/docs/content/docs/architecture.mdx +++ b/docs/content/docs/architecture.mdx @@ -78,7 +78,7 @@ The MCP server runs in whichever transport fits the calling tool. This is chosen own service principal ``` -See [Installation](/docs/installation#running-flowx-as-an-mcp-server) for the exact commands and the [app README](https://github.com/ghanse/flowx/tree/main/app) for deployment details. +See [Installation](/docs/installation#running-flowx-as-an-mcp-server) for the exact commands and the [app README](https://github.com/databricks-solutions/flowx/tree/main/app) for deployment details. A Databricks App can't read the user's workspace / UC Volume files (`/Volumes/...` is **not** auto-mounted). Two ways to get data in/out of the `flowx` tool: diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index 619c98d..cb9f5d9 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -1,147 +1,151 @@ --- title: Installation -description: Install flowx in Databricks Genie Code, Claude Code, or other agentic tools. +description: Install flowx in Databricks Genie Code or a local agent harness. --- -import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; import { Callout } from 'fumadocs-ui/components/callout'; import { Steps, Step } from 'fumadocs-ui/components/steps'; -flowx is a set of [agent skills](https://github.com/ghanse/flowx/tree/main/skills) that can be installed and used with AI coding assistants. -To use these skills, install flowx as a plugin using your AI assistant's preferred installation method. +flowx is a set of [agent skills](https://github.com/databricks-solutions/flowx/tree/main/skills) that run from an AI coding assistant. How you install them depends on where your agent runs: - - -Clone the flowx repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos). Clone it under **`/Workspace/Shared`** (e.g. `/Workspace/Shared/flowx`) rather than your private `/Workspace/Users/` home — when you later deploy the MCP server, the app's service principal must be able to read the source, and it has no access to private user folders by default. Then copy the `skills/` directory into a user-level skills folder: +- **Databricks Genie Code** runs the phases as a hosted **MCP server** (a Databricks App). No local Python environment is involved — the app vendors flowx's code and dependencies. +- **A local agent harness (Claude Code, or any Agent Skills tool)** runs the phases from a local **Python virtual environment**, optionally exposing them over a local MCP server too. -```bash -databricks workspace import-dir skills /Users//.assistant/skills -``` +Pick the matching section below and follow it end to end. -To make flowx available for other workspace users, copy `skills/` into a workspace-level skills folder: +## Installing flowx for Databricks Genie Code -```bash -databricks workspace import-dir skills /Workspace/.assistant/skills -``` +In Genie Code the phases run as the single `flowx` tool on a Databricks App you deploy and then register as a custom MCP server. -Genie Code picks up skills from these directories automatically. Skills fire automatically when their description matches your request. -To invoke a specific skill, use the `@` prefix (e.g. `@flowx-migrate translate the ADF pipelines I exported to /Volumes/main/default/adf_export`). + + +### Clone flowx into a shared workspace location -See the [Databricks Genie Code Skills documentation](https://docs.databricks.com/aws/en/genie-code/skills) for more details. - +Clone the repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos), under **`/Workspace/Shared`** (e.g. `/Workspace/Shared/flowx`). - -flowx is packaged as a Claude Code plugin. The plugin manifest lives at [`.claude-plugin/plugin.json`](https://github.com/ghanse/flowx/blob/main/.claude-plugin/plugin.json). To install -flowx, run the following command from a Claude Code session: + +The MCP app's service principal cannot read private `/Workspace/Users/` folders by default. Cloning into `/Workspace/Shared` keeps the repo, the deployed app source, and team access all in a location every user and the app's service principal can reach. If your workspace restricts `/Workspace/Shared`, use any other all-users location and pass it to the deployer. + + -```bash -/plugin marketplace add ghanse/flowx -/plugin install flowx -``` + +### Copy the skills into your skills folder -You can also copy the skill folders into your local `/.claude/skills` folder: +Genie Code picks up skills from your `.assistant/skills` folder automatically. Copy `skills/` into a user-level folder: ```bash -cp -R skills/{flowx-setup,flowx-discover,flowx-convert,flowx-package,flowx-migrate} ~/.claude/skills/ +databricks workspace import-dir skills /Users//.assistant/skills ``` -Once installed, the skills can be invoked using `/flowx:flowx-migrate`, `/flowx:flowx-discover`, etc. Claude Code can also invoke the skills when prompted. See the [Claude Code Plugins documentation](https://docs.claude.com/en/docs/claude-code/plugins) for more details. - - - -Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install the flowx skills. The general pattern: - -1. Copy each skill folder (`skills/flowx-setup`, `skills/flowx-discover`, `skills/flowx-convert`, `skills/flowx-package`, `skills/flowx-migrate`) into the tool's configured skills directory. -2. Make sure the path contains `SKILL.md` directly, -3. Restart the tool if it caches skill metadata at startup. - - -If your tool expects a single Markdown file instead of a directory tree, use the following command to flatten flowx's skills files: +Or make flowx available to all workspace users with a workspace-level folder: ```bash -cat skills/*/SKILL.md > flowx-skills.md +databricks workspace import-dir skills /Workspace/.assistant/skills ``` - - - - -## Running flowx as an MCP server -flowx's phases are also packaged as [Model Context Protocol](https://modelcontextprotocol.io) tools (in [`src/flowx/mcp/`](https://github.com/ghanse/flowx/tree/main/src/flowx/mcp)) so an agent can invoke them directly instead of shelling out to the CLI. The `setup` skill wires this up automatically based on your environment; you can also do it manually. See [Architecture](/docs/architecture) for how the tool layer maps onto the phases. - -### Configuring the MCP server for Databricks Genie Code - -Genie Code connects to a **hosted** MCP endpoint, so the flowx tools run as a Databricks App that you add in Genie Code's **Custom MCP server** picker. End to end: - - - -#### Clone flowx - -Follow the **Databricks Genie Code** install steps above to clone the repo and copy `skills/` into your skills folder. - - -The MCP app's service principal cannot read private `/Workspace/Users/` folders by default, and `deploy.sh` deploys the source from `/Workspace/Shared/`. Cloning into `/Workspace/Shared` keeps the repo, the deployed source, and team access all in a location every user and the app's service principal can reach. If your workspace restricts `/Workspace/Shared`, use any other folder all users (and the app service principal) can read and pass it via `APP_SOURCE_PATH`. - +Skills fire automatically when their description matches your request. To invoke one explicitly, use the `@` prefix (e.g. `@flowx-migrate translate the ADF pipelines I exported to /Volumes/main/default/adf_export`). See the [Genie Code Skills docs](https://docs.databricks.com/aws/en/genie-code/skills) for more. -#### Run the setup skill +### Run the setup skill -Ask your agent to *"set up the flowx environment"* (or run `bash /scripts/bootstrap.sh`). On Databricks the `setup` skill detects the environment and, after creating the venv, runs the app deployment in the next step for you. Run it from a workspace web terminal if your Genie session can't shell out to the Databricks CLI. +Ask your agent to *"set up the flowx environment"* (or run `@flowx-setup`). On Databricks, setup detects the environment and prepares the MCP path — it does **not** create a virtual environment, because the phases run through the deployed app rather than a local interpreter. -#### Deploy the MCP server +### Deploy the MCP server -```bash -bash /app/deploy.sh -``` +Deploy the `mcp-flowx` Databricks App from the flowx checkout. The recommended way runs entirely in the workspace, including on **serverless** compute: -`deploy.sh` stages a self-contained bundle (the app entrypoint plus a vendored copy of the flowx source), syncs it to **`/Workspace/Shared/mcp-flowx`** (a location the app's service principal can read — override with `APP_SOURCE_PATH`), and creates/deploys the **`mcp-flowx`** app. The script prints the app URL; the MCP endpoint is **`/mcp`**. +Open and run the **`app/deploy_app.py`** notebook. Set its `repo_root` widget to your checkout (e.g. `/Workspace/Shared/flowx`); it uses the Databricks SDK to stage a self-contained source bundle (app entrypoint plus a vendored copy of the flowx package) to `/Workspace/Shared/mcp-flowx` and create/deploy the app. The notebook prints the app URL; the MCP endpoint is **`/mcp`**. - -`databricks apps` deploy commands require a Databricks CLI session and must be run from the workspace web terminal or a local machine. + +You can instead run `bash app/deploy.sh` from a **workspace web terminal or a local machine**. The `databricks apps deploy` / `databricks sync` commands it uses require a CLI session and are **not** available from serverless notebook Python — which is why `deploy_app.py` (SDK-based) is preferred inside Genie Code. -#### Grant access +### Grant access - **App access:** grant **Can use** on the `mcp-flowx` app to the users or service principals that will call it (Apps UI → *Permissions*, or `databricks apps set-permissions`). - **Data access:** grant the app's own service principal access to the catalogs, schemas, and Unity Catalog volumes the migration reads from and writes to, plus any SQL warehouse used by the reporting commands (`flowx(command="record_results")` / `flowx(command="install_dashboard")`). -#### Register the MCP server +### Register the MCP server in Genie Code -MCP servers are available in Genie Code [Agent mode](https://learn.microsoft.com/en-us/azure/databricks/genie-code/use-genie-code#modes). To add the flowx MCP server: +MCP servers are available in Genie Code [Agent mode](https://learn.microsoft.com/en-us/azure/databricks/genie-code/use-genie-code#modes): 1. In the Genie Code panel, click **⚙ Settings**. 2. Under **MCP Servers**, click **+ Add Server**. 3. Choose **Custom MCP server** and select the **`mcp-flowx`** Databricks App. 4. Click **Save**. -The single `flowx` tool will be available when you use Genie Code in Agent mode. +The single `flowx` tool is available when you use Genie Code in Agent mode. -Databricks requires a custom MCP app to be: -* Deployed in the same workspace -* Reachable at `https:///mcp` - -If Genie Code cannot connect to the flowx MCP server, set the app's `FLOWX_ALLOWED_ORIGINS` environment variable to your workspace URL and redeploy. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp). +A custom MCP app must be deployed in the **same workspace** and reachable at `https:///mcp`. If Genie Code cannot connect, set the app's `FLOWX_ALLOWED_ORIGINS` environment variable to your workspace URL and redeploy. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp). -#### Verify the MCP Server +### Verify -Open the health endpoint `/` (returns `{"status":"ok"}`), or ask Genie Code *"what flowx MCP tools are available?"*. You should see the single `flowx` tool. +Open the health endpoint `/` (returns `{"status":"ok"}`), or ask Genie Code *"what flowx MCP tools are available?"*. You should see the single `flowx` tool. You can now run `@flowx-migrate` (or the individual phase skills). -### Configuring the MCP server for other agent tools +## Installing flowx for a local agent harness (Claude Code) -Deploy the MCP server locally to use flowx with other agent tools. Install the MCP server stack into a local Python virtual environment and run it over stdio: +Locally, flowx installs as a Claude Code plugin and runs its phases from a Python virtual environment. + + + +### Install the plugin + +flowx is distributed through its Claude Code marketplace. From a Claude Code session: + +```bash +/plugin marketplace add databricks-solutions/flowx +/plugin install flowx@flowx +``` + +Then run `/reload-plugins` to activate it. + + +You can also copy the skill folders straight into your local skills directory: + +```bash +cp -R skills/{flowx-setup,flowx-discover,flowx-convert,flowx-package,flowx-migrate} ~/.claude/skills/ +``` + + + + +### Run the setup skill + +Run `/flowx:flowx-setup` (or ask *"set up the flowx environment"*) **once** before any phase. flowx's Python modules depend on third-party packages (`pyyaml`, `databricks-sdk`, `sqlglot`), so setup provisions an isolated virtual environment via `scripts/bootstrap.sh`. It will: + +1. Check that `python3`, `pip`, and the `venv` module are available. +2. Create the virtual environment at `/.venv`. +3. Install `requirements.txt` into it with `pip`. +4. Write the resolved interpreter path to the marker file `/.migration-venv`, which the phase skills read. + +The environment is created once and reused. No `uv` is required for plugin users. + + +If `python3`, `pip`, or the `venv` module are missing, the script prints a warning and exits **without** creating anything. Install Python, then re-run setup: + +* **macOS:** `brew install python` +* **Debian/Ubuntu:** `sudo apt-get install python3 python3-venv python3-pip` +* **Windows:** [python.org/downloads](https://www.python.org/downloads/) (enable "Add python.exe to PATH") + + + + +### (Optional) Run the phases over a local MCP server + +The phase skills call the venv CLI directly, so this step is optional. To instead drive the phases through MCP tools locally, install the MCP server stack into the venv and register the stdio server with your MCP client: ```bash PY="$(cat /.migration-venv)" @@ -149,8 +153,6 @@ PY="$(cat /.migration-venv)" PYTHONPATH="/src" "$PY" -m flowx.mcp ``` -Register the server with your MCP client (use the interpreter path from the marker file for `command`): - ```json { "mcpServers": { @@ -166,53 +168,35 @@ Register the server with your MCP client (use the interpreter path from the mark If you prefer an installed package over `PYTHONPATH`, run `pip install -e ".[mcp]"` from the plugin root; then `python -m flowx.mcp` works without setting `PYTHONPATH`. + -## Running flowx as a Python process - -flowx's skills invoke Python modules that may depend on third-party packages. The `setup` skill provisions an isolated virtual environment with the required dependencies. - -Run it **once** after installing the skills, before `discover`, `convert`, `package`, or `migrate`. Just ask your agent: + +### Verify -> Set up the flowx environment +Open Claude Code and ask *"What flowx skills do you have available?"*. You should see all five skills (`flowx-setup`, `flowx-discover`, `flowx-convert`, `flowx-package`, `flowx-migrate`). Invoke them with `/flowx:flowx-migrate`, `/flowx:flowx-discover`, etc. -The setup script can also be run directly from the plugin root: - -```bash -bash /scripts/bootstrap.sh -``` - -Running the setup process will: - -1. Check that `python3`, `pip`, and `venv` are available. -2. Create the virtual environment if it doesn't already exist. When running under Databricks (Genie Code or notebooks, detected via `DATABRICKS_RUNTIME_VERSION`), the venv is created at `/Workspace/Users//.migration-skills`; everywhere else it is created at `/.venv`. -3. Install the `requirements.txt` dependencies into your virtual environment using `pip`. -4. Write the resolved interpreter path to the marker file `/.migration-venv`. - -The environment is created once and reused. Re-running the script simply confirms the venv exists and its dependencies are satisfied. - - -If `python3`, `pip`, or the `venv` module are missing, the script will print a warning and exit **without** creating anything. -To install Python in your environment, run one of the following commands: - -* **macOS:** `brew install python` -* **Debian/Ubuntu:** `sudo apt-get install python3 python3-venv python3-pip` -* **Windows:** [python.org/downloads](https://www.python.org/downloads/) (enable "Add python.exe to PATH") - - -After the venv exists, every Python command the skills run uses the interpreter recorded in the marker file, with `src/` on `PYTHONPATH`. Read the interpreter path from `/.migration-venv` rather than hardcoding it: +If you hit a `ModuleNotFoundError` while running a phase, the venv is missing or incomplete — re-run `/flowx:flowx-setup`. Every Python command the skills run uses the interpreter recorded in `/.migration-venv`, with `src/` on `PYTHONPATH`: ```bash export PYTHONPATH="/src" PY="$(cat /.migration-venv)" "$PY" -m flowx.adapter inputs discover ``` + + -The marker file points at `/.venv` locally or `/Workspace/Users//.migration-skills` on Databricks. The agent normally runs these commands for you; they are handy for troubleshooting a `ModuleNotFoundError`. +### Other AI tools -## Verifying the installation +Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install the flowx skills: -Open your agent and ask: +1. Copy each skill folder (`skills/flowx-setup`, `skills/flowx-discover`, `skills/flowx-convert`, `skills/flowx-package`, `skills/flowx-migrate`) into the tool's configured skills directory, so the path contains `SKILL.md` directly. +2. Restart the tool if it caches skill metadata at startup. +3. Follow the **local agent harness** setup above to provision the Python environment (`scripts/bootstrap.sh`). -> What flowx skills do you have available? + +If your tool expects a single Markdown file instead of a directory tree, concatenate the skills: -You should see all five skills listed with their descriptions. If only some appear, double-check the install path your tool watches for skills. +```bash +cat skills/*/SKILL.md > flowx-skills.md +``` + diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index ead7187..50cb58e 100644 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -137,9 +137,9 @@ if [ ! -x "$VENV_DIR/bin/python" ]; then rm -rf "$VENV_DIR" "$PYTHON_BIN" -m venv --without-pip "$VENV_DIR" echo "Bootstrapping pip via get-pip.py ..." - curl -sSL https://bootstrap.pypa.io/get-pip.py -o /tmp/_orchestra_get_pip.py - "$VENV_DIR/bin/python" /tmp/_orchestra_get_pip.py --quiet - rm -f /tmp/_orchestra_get_pip.py + curl -sSL https://bootstrap.pypa.io/get-pip.py -o /tmp/_flowx_get_pip.py + "$VENV_DIR/bin/python" /tmp/_flowx_get_pip.py --quiet + rm -f /tmp/_flowx_get_pip.py fi else echo "Using existing virtual environment at $VENV_DIR ..." diff --git a/skills/flowx-convert/SKILL.md b/skills/flowx-convert/SKILL.md index 5a84d32..5cd2b43 100644 --- a/skills/flowx-convert/SKILL.md +++ b/skills/flowx-convert/SKILL.md @@ -2,8 +2,8 @@ name: flowx-convert description: > Translate parsed ADF pipeline AST into Databricks IR (intermediate representation). - Runs deterministic translators for known activity types, then invokes agentic skills - from adf-to-databricks-plugin for gaps. + Runs deterministic translators for known activity types, then performs agentic + (LLM-assisted) translation for the remaining gaps. triggers: - "translate ADF" - "convert ADF" @@ -22,7 +22,7 @@ This is phase 2 of the flowx migration workflow. It consumes the ADF source (pro The translation follows a **deterministic-first** strategy: 1. Activities with known, well-defined mappings are translated by built-in Python translators -2. Activities that require interpretation, expression conversion, or lack a Python translator are handled by agent skills from the `adf-to-databricks-plugin` +2. Activities that require interpretation, expression conversion, or lack a Python translator are handled by agentic (LLM-assisted) translation performed by the agent ## How to run this skill — MCP tools or venv CLI @@ -136,8 +136,7 @@ Read `/.work/translation_report.json`. It has this structure: "type": "ExecuteDataFlow", "strategy": "agentic", "status": "pending", - "raw_activity_json": { "...": "..." }, - "target_skill": "adf-to-databricks:adf-dataflow-converter" + "raw_activity_json": { "...": "..." } } ], "summary": { @@ -151,7 +150,7 @@ Read `/.work/translation_report.json`. It has this structure: ### Step 4 — Handle agentic gaps -For each translation with `"status": "pending"` and `"strategy": "agentic"`, invoke the appropriate skill from the `adf-to-databricks-plugin`. Route by activity type. +For each translation with `"status": "pending"` and `"strategy": "agentic"`, perform LLM-assisted translation from the activity's ARM JSON, routing by activity type. Every agentic gap in the translation report carries the activity's **full ADF/ARM JSON** under `raw_activity_json` (engine field `raw_definition`), and the generated placeholder notebook embeds the same JSON in a fenced `json` block. This holds for nested activities too — an `Until` inside an `IfCondition` / `Switch` / `ForEach` is reported as its own gap. Always translate from this ARM JSON. @@ -160,36 +159,36 @@ Databricks Lakeflow Jobs have no native repeat-until loop, so translate the `Unt - `typeProperties.expression` — the ADF exit condition (e.g. `@or(equals(variables('jobStatus'),'succeeded'), equals(variables('jobStatus'),'failed'))`); convert it into the Python `while not ():` guard. - `typeProperties.timeout` — wrap the loop in a wall-clock deadline (`time.monotonic()`), raising on timeout. - `typeProperties.activities` — the loop body (e.g. a `Wait`, a polling `WebActivity`, a `SetVariable` that captures the next status); translate each child inline so the whole loop runs in one notebook. -Read the loop variables from `dbutils.widgets`, surface the final state as a task value, and write the result over the placeholder notebook's `raise NotImplementedError` cell. If the external `adf-to-databricks:adf-pipeline-converter` skill is installed you may delegate to it with the same ARM JSON; otherwise perform the translation directly. +Read the loop variables from `dbutils.widgets`, surface the final state as a task value, and write the result over the placeholder notebook's `raise NotImplementedError` cell. Perform the translation directly from the same ARM JSON. **ExecuteDataFlow activities:** -Invoke `adf-to-databricks:adf-dataflow-converter` with the raw activity JSON and associated data flow definition. Provide context: +Translate the data flow directly from the raw activity JSON and associated data flow definition, using: - The raw `typeProperties` from the ADF activity - The data flow JSON definition (if available in the source directory under `dataflow/`) - The linked service configurations for source/sink connections - Target catalog and schema for the SDP pipeline or PySpark notebook output **Control flow activities (Switch, Until, Wait, Filter, AppendVariable):** -Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: +Translate the control-flow activity directly from the raw activity JSON, using: - The full pipeline JSON containing the activity - Any nested activities within the control flow - Variable definitions from the pipeline - The desired Databricks task type mapping **Stored procedures and external calls (SqlServerStoredProcedure, AzureFunction, WebHook, Custom):** -Invoke `adf-to-databricks:adf-pipeline-converter` with the raw activity JSON. Provide context: +Translate the activity directly from the raw activity JSON, using: - The linked service configuration for the target system - Connection details and authentication method - Any parameters or request bodies **Complex expressions:** -If any activity (deterministic or agentic) contains ADF expressions that the deterministic translator could not resolve, invoke `adf-to-databricks:adf-expression-translator` with: +If any activity (deterministic or agentic) contains ADF expressions that the deterministic translator could not resolve, translate them directly, using: - The raw expression string (e.g., `@pipeline().parameters.inputPath`) - The expression context (pipeline parameters, variables, activity outputs) - The target format (Python f-string, Spark SQL, task parameter reference) **Trigger definitions:** -Invoke `adf-to-databricks:adf-trigger-converter` with: +Translate the trigger directly, using: - The trigger JSON definition - The associated pipeline references - Target: Databricks job schedule configuration (quartz_cron_expression, periodic, or file_arrival) diff --git a/skills/flowx-convert/references/activity-mapping.md b/skills/flowx-convert/references/activity-mapping.md index 9635220..5fb391b 100644 --- a/skills/flowx-convert/references/activity-mapping.md +++ b/skills/flowx-convert/references/activity-mapping.md @@ -5,7 +5,7 @@ This reference defines the mapping between Azure Data Factory activity types and ## Strategy Definitions - **Deterministic** — Handled by a built-in Python translator module. Fast, reliable, no LLM required. These mappings are well-defined and produce consistent output. -- **Agentic** — Handled by an LLM-assisted skill from the `adf-to-databricks-plugin`. Required when the ADF activity has complex semantics, requires interpretation, or lacks a direct Databricks equivalent. +- **Agentic** — Handled by agentic (LLM-assisted) translation performed by the agent. Required when the ADF activity has complex semantics, requires interpretation, or lacks a direct Databricks equivalent. - **Unsupported** — No automated translation path. Requires manual intervention. ## Activity Mapping Table @@ -26,19 +26,19 @@ This reference defines the mapping between Azure Data Factory activity types and | DatabricksJob | Deterministic | `databricks_job.py` | `run_job_task` | | Switch | Deterministic | `switch.py` | chained `condition_task`s | | Wait | Deterministic | `wait.py` | `notebook_task` (`time.sleep`) | -| ExecuteDataFlow | Agentic | `adf-to-databricks:adf-dataflow-converter` | DLT pipeline or PySpark notebook | -| Until | Agentic | `adf-to-databricks:adf-pipeline-converter` | while-loop notebook | +| ExecuteDataFlow | Agentic | Agentic (LLM-assisted) | DLT pipeline or PySpark notebook | +| Until | Agentic | Agentic (LLM-assisted) | while-loop notebook | | Filter | Deterministic | `filter.py` | `notebook_task` (filter array + task values) | | AppendVariable | Deterministic | `append_variable.py` | `notebook_task` (append to array task value) | -| SqlServerStoredProcedure | Agentic | `adf-to-databricks:adf-pipeline-converter` | SQL notebook | -| AzureFunction | Agentic | `adf-to-databricks:adf-pipeline-converter` | webhook/REST notebook | -| WebHook | Agentic | `adf-to-databricks:adf-pipeline-converter` | REST notebook | -| Custom | Agentic | `adf-to-databricks:adf-pipeline-converter` | custom notebook | -| ExecuteSSISPackage | Agentic | `adf-to-databricks:adf-pipeline-converter` | PySpark notebook | -| AzureMLExecutePipeline | Agentic | `adf-to-databricks:adf-pipeline-converter` | MLflow notebook | -| Triggers (Schedule) | Agentic | `adf-to-databricks:adf-trigger-converter` | `quartz_cron_expression` | -| Triggers (Tumbling Window) | Agentic | `adf-to-databricks:adf-trigger-converter` | periodic schedule | -| Triggers (Blob Event) | Agentic | `adf-to-databricks:adf-trigger-converter` | `file_arrival` trigger | +| SqlServerStoredProcedure | Agentic | Agentic (LLM-assisted) | SQL notebook | +| AzureFunction | Agentic | Agentic (LLM-assisted) | webhook/REST notebook | +| WebHook | Agentic | Agentic (LLM-assisted) | REST notebook | +| Custom | Agentic | Agentic (LLM-assisted) | custom notebook | +| ExecuteSSISPackage | Agentic | Agentic (LLM-assisted) | PySpark notebook | +| AzureMLExecutePipeline | Agentic | Agentic (LLM-assisted) | MLflow notebook | +| Triggers (Schedule) | Agentic | Agentic (LLM-assisted) | `quartz_cron_expression` | +| Triggers (Tumbling Window) | Agentic | Agentic (LLM-assisted) | periodic schedule | +| Triggers (Blob Event) | Agentic | Agentic (LLM-assisted) | `file_arrival` trigger | ## Deterministic Translator Details @@ -151,7 +151,7 @@ Maps to a `notebook_task` that appends a value to an array variable: ## Agentic Translation Notes -Agentic translations are handled by skills from the `adf-to-databricks-plugin` (`birbalin25/adf-to-databricks-plugin`). These skills use LLM reasoning to: +Agentic translations are performed by the agent, using LLM reasoning to: 1. Interpret complex ADF semantics that lack direct Databricks equivalents 2. Convert ADF expressions to Python/SQL equivalents diff --git a/skills/flowx-discover/SKILL.md b/skills/flowx-discover/SKILL.md index f33a992..6ed4187 100644 --- a/skills/flowx-discover/SKILL.md +++ b/skills/flowx-discover/SKILL.md @@ -22,7 +22,7 @@ Parse Azure Data Factory pipeline, dataset, linked service, and trigger JSON fil This is phase 1 of the flowx migration workflow. It takes raw ADF JSON exports and produces an `inventory.json` file that the `convert` skill consumes. The inventory classifies every ADF activity into one of three strategies: - **Deterministic** — a built-in translator exists (Copy, DatabricksNotebook, ForEach, IfCondition, etc.) -- **Agentic** — requires LLM-assisted translation via the `adf-to-databricks-plugin` skills (ExecuteDataFlow, Switch, Until, StoredProc, etc.) +- **Agentic** — requires agentic (LLM-assisted) translation by the agent (ExecuteDataFlow, Switch, Until, StoredProc, etc.) - **Unsupported** — no known translation path; requires manual intervention ## How to run this skill — MCP tool or venv CLI @@ -174,8 +174,7 @@ Read the generated `/metadata/inventory.json` file. It has this stru { "name": "RunDataFlow", "type": "ExecuteDataFlow", - "strategy": "agentic", - "skill": "adf-to-databricks:adf-dataflow-converter" + "strategy": "agentic" } ] } @@ -232,12 +231,12 @@ Coverage: 95.7% ### Step 6 — Detail agentic activities -For activities classified as `agentic`, explain which skill from the `adf-to-databricks-plugin` will handle each: +For activities classified as `agentic`, explain that each is translated by the agent using LLM-assisted reasoning from the activity's ARM JSON (no built-in deterministic translator exists for these types): -| Activity | Type | Handling Skill | +| Activity | Type | Handling | |---|---|---| -| RunDataFlow | ExecuteDataFlow | `adf-to-databricks:adf-dataflow-converter` | -| BranchLogic | Switch | `adf-to-databricks:adf-pipeline-converter` | +| RunDataFlow | ExecuteDataFlow | Agentic (LLM-assisted) | +| BranchLogic | Switch | Agentic (LLM-assisted) | | ... | ... | ... | ### Step 7 — Warn about unsupported activities diff --git a/skills/flowx-migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md index 379a958..db8ee26 100644 --- a/skills/flowx-migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -68,11 +68,11 @@ ADF JSON Exports **Process:** 1. Run deterministic translators for all activities classified as `deterministic` -2. For each `agentic` activity, invoke the appropriate skill from the `adf-to-databricks-plugin`: - - `adf-dataflow-converter` for ExecuteDataFlow activities - - `adf-pipeline-converter` for control flow and external call activities - - `adf-expression-translator` for complex ADF expression conversion - - `adf-trigger-converter` for trigger schedule translation +2. For each `agentic` activity, perform LLM-assisted translation from the activity's ARM JSON: + - ExecuteDataFlow activities → a purpose-built PySpark notebook + - control flow and external-call activities → a notebook implementing the activity's semantics + - complex ADF expressions → Python/SQL equivalents + - trigger schedules → Databricks schedule/trigger configuration 3. Merge deterministic and agentic results into a unified translation report 4. Generate Databricks IR (intermediate representation) for each activity diff --git a/skills/flowx-setup/SKILL.md b/skills/flowx-setup/SKILL.md index f319580..cd0ecf7 100644 --- a/skills/flowx-setup/SKILL.md +++ b/skills/flowx-setup/SKILL.md @@ -44,17 +44,18 @@ fi ## Path A — Databricks Genie Code (MCP, no virtual environment) In Genie Code the phases run on the deployed app, so **do not run `bootstrap.sh` and do not create a -venv** — it isn't needed. Deploy the MCP server instead: +venv** — it isn't needed. Deploy the MCP server instead. -```bash -bash /app/deploy.sh -``` +**Recommended (works on serverless): run the `app/deploy_app.py` notebook.** It uses the Databricks +SDK to stage a self-contained bundle (the app entrypoint plus a vendored copy of the flowx source) +to **`/Workspace/Shared/mcp-flowx`** and create/deploy the **`mcp-flowx`** Databricks App. Set its +`repo_root` widget to the flowx checkout (e.g. `/Workspace/Shared/flowx`). Because it uploads through +the SDK Workspace API, it runs directly in a serverless Genie Code session. The notebook prints the +app URL; the MCP endpoint is `/mcp`. -`app/deploy.sh` stages a self-contained bundle (the app entrypoint plus a vendored copy of the -flowx source), syncs it to **`/Workspace/Shared/mcp-flowx`**, and creates/deploys the -**`mcp-flowx`** Databricks App. The script prints the app URL; the MCP endpoint is -`/mcp`. It only needs the Databricks CLI and a system `python3` (for parsing CLI output) — -**not** an flowx venv. +**CLI alternative:** `bash /app/deploy.sh` does the same via the Databricks CLI +(`apps deploy` / `sync`), but those commands require a CLI session — run it from a workspace web +terminal or a local machine, **not** serverless notebook Python. > **Clone into a shared location.** The app's service principal cannot read private > `/Workspace/Users/` folders by default, so `deploy.sh` deploys the source from @@ -77,11 +78,11 @@ Once added, the `discover`, `convert`, `package`, and `migrate` skills run **ent `flowx` MCP tool** (`flowx(command="…", parameters={…})`) — there is no venv, no `bootstrap.sh`, and no `.migration-venv` marker on this path. -> **Note:** `databricks apps` deploy commands require a Databricks CLI session (workspace web -> terminal or a local machine), not serverless notebook Python. If the Genie session can't shell -> out to the CLI, run `app/deploy.sh` from the web terminal. (Same constraint as `databricks bundle -> deploy`.) If you see `Error: please specify target`, the CLI attached to a stray `databricks.yml`; -> `deploy.sh` already isolates against this, so re-run it as-is. +> **Note:** Prefer the `app/deploy_app.py` notebook in serverless Genie Code — the SDK works there, +> whereas `databricks apps deploy` / `sync` (used by `deploy.sh`) need a CLI session (web terminal +> or local machine). Only fall back to `deploy.sh` from a web terminal. If `deploy.sh` reports +> `Error: please specify target`, the CLI attached to a stray `databricks.yml`; it already isolates +> against this, so re-run it as-is. --- diff --git a/src/flowx/models/adf_ast.py b/src/flowx/models/adf_ast.py index 3fa8d90..a91ef30 100644 --- a/src/flowx/models/adf_ast.py +++ b/src/flowx/models/adf_ast.py @@ -307,7 +307,6 @@ class InventoryItem: activity_name: Activity display name. activity_type: ADF activity type string. strategy: Determined translation strategy. - agentic_skill: Skill identifier when strategy is ``AGENTIC``. depends_on: Upstream activity names. """ @@ -315,7 +314,6 @@ class InventoryItem: activity_name: str activity_type: str strategy: TranslationStrategy - agentic_skill: str | None = None depends_on: list[str] | None = None diff --git a/src/flowx/models/ir.py b/src/flowx/models/ir.py index 1b272ba..aa1afe3 100644 --- a/src/flowx/models/ir.py +++ b/src/flowx/models/ir.py @@ -482,8 +482,6 @@ class PlaceholderActivity(Activity): original_type: str notebook_path: str = "/UNSUPPORTED_ADF_ACTIVITY" comment: str | None = None - # For an agentic gap (e.g. Until): the recommended skill the agent should translate from. - agentic_skill: str | None = None raw_definition: dict[str, Any] | None = None @@ -732,13 +730,11 @@ class AgenticGap: Attributes: activity_name: Display name of the activity. activity_type: ADF activity type string. - recommended_skill: Skill identifier to use for translation. raw_definition: Original ADF JSON definition for the activity. """ activity_name: str activity_type: str - recommended_skill: str | None = None raw_definition: dict[str, Any] | None = None diff --git a/src/flowx/parser/adf_loader.py b/src/flowx/parser/adf_loader.py index 30c7c6e..ab760b3 100644 --- a/src/flowx/parser/adf_loader.py +++ b/src/flowx/parser/adf_loader.py @@ -55,19 +55,19 @@ "AppendVariable", } -AGENTIC_TYPES: dict[str, str] = { - "ExecuteDataFlow": "adf-to-databricks:adf-dataflow-converter", - "Until": "adf-to-databricks:adf-pipeline-converter", - "SqlServerStoredProcedure": "adf-to-databricks:adf-pipeline-converter", - "AzureFunction": "adf-to-databricks:adf-pipeline-converter", - "WebHook": "adf-to-databricks:adf-pipeline-converter", - "Custom": "adf-to-databricks:adf-pipeline-converter", - "ExecuteSSISPackage": "adf-to-databricks:adf-pipeline-converter", - "AzureMLExecutePipeline": "adf-to-databricks:adf-pipeline-converter", - "GetMetadata": "adf-to-databricks:adf-pipeline-converter", - "Validation": "adf-to-databricks:adf-pipeline-converter", - "Fail": "adf-to-databricks:adf-pipeline-converter", - "Script": "adf-to-databricks:adf-pipeline-converter", +AGENTIC_TYPES: set[str] = { + "ExecuteDataFlow", + "Until", + "SqlServerStoredProcedure", + "AzureFunction", + "WebHook", + "Custom", + "ExecuteSSISPackage", + "AzureMLExecutePipeline", + "GetMetadata", + "Validation", + "Fail", + "Script", } # Activity complexity weights (easiest first): Databricks-native ~1:1 tasks, then control-flow, then @@ -195,21 +195,20 @@ def _parse_factory_global_parameters(data: dict[str, Any]) -> dict[str, Any]: return result -def classify_activity(activity_type: str) -> tuple[TranslationStrategy, str | None]: +def classify_activity(activity_type: str) -> TranslationStrategy: """Classify an ADF activity type into a translation strategy. Args: activity_type: ADF activity type string (e.g. ``"Copy"``). Returns: - A ``(strategy, agentic_skill_name)`` tuple. *agentic_skill_name* is - ``None`` for deterministic and unsupported strategies. + The :class:`TranslationStrategy` for the activity type. """ if activity_type in DETERMINISTIC_TYPES: - return TranslationStrategy.DETERMINISTIC, None + return TranslationStrategy.DETERMINISTIC if activity_type in AGENTIC_TYPES: - return TranslationStrategy.AGENTIC, AGENTIC_TYPES[activity_type] - return TranslationStrategy.UNSUPPORTED, None + return TranslationStrategy.AGENTIC + return TranslationStrategy.UNSUPPORTED def build_inventory(definitions: AdfDefinitions) -> Inventory: @@ -654,7 +653,7 @@ def _classify_activities( items: Accumulator list to append results to. """ for activity in activities: - strategy, skill = classify_activity(activity.type) + strategy = classify_activity(activity.type) dep_names = [dependency.activity for dependency in activity.depends_on] if activity.depends_on else None items.append( @@ -663,7 +662,6 @@ def _classify_activities( activity_name=activity.name, activity_type=activity.type, strategy=strategy, - agentic_skill=skill, depends_on=dep_names, ) ) @@ -698,8 +696,6 @@ def _inventory_to_dict(inventory: Inventory, source_dir: str) -> dict[str, Any]: "type": item.activity_type, "strategy": item.strategy.value, } - if item.agentic_skill: - entry["skill"] = item.agentic_skill if item.depends_on: entry["depends_on"] = item.depends_on pipeline_map.setdefault(item.pipeline_name, []).append(entry) @@ -901,7 +897,7 @@ def write_pipeline_arm(definitions: AdfDefinitions, metadata_dir: Path) -> list[ # CLI entry point # --------------------------------------------------------------------------- -# Flowx-managed entries under the shared output_dir, cleared at the start of each fresh run. +# flowx-managed entries under the shared output_dir, cleared at the start of each fresh run. _MANAGED_OUTPUT_DIRS: tuple[str, ...] = ("metadata", ".work", "resources", "src", "setup") _MANAGED_OUTPUT_FILES: tuple[str, ...] = ("databricks.yml", "SETUP.md", "WARNINGS.md") diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py index 8b460b3..6148cbf 100644 --- a/src/flowx/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -214,12 +214,10 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: """Returns a PreparedActivity with a stub notebook for an unsupported activity.""" task = build_common_task_fields(activity) - agentic_skill: str | None = None raw_definition: dict[str, Any] | None = None if isinstance(activity, PlaceholderActivity): comment = activity.comment or "This activity requires manual implementation." original_type = activity.original_type - agentic_skill = activity.agentic_skill raw_definition = activity.raw_definition elif isinstance(activity, UnsupportedActivity): comment = activity.reason or "This activity type is not supported." @@ -237,11 +235,10 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: if raw_definition is not None: import json as _json - skill_hint = f" using `{agentic_skill}`" if agentic_skill else "" arm_lines = _json.dumps(raw_definition, indent=2).splitlines() arm_block = ( "# MAGIC\n" - f"# MAGIC An agent should translate this activity{skill_hint} from the ADF/ARM JSON below,\n" + "# MAGIC An agent should translate this activity from the ADF/ARM JSON below,\n" "# MAGIC then replace the `raise NotImplementedError` cell with the generated code.\n" "# MAGIC\n" "# MAGIC ```json\n" + "".join(f"# MAGIC {line}\n" for line in arm_lines) + "# MAGIC ```\n" diff --git a/src/flowx/translator/engine.py b/src/flowx/translator/engine.py index b99c318..b45e30e 100644 --- a/src/flowx/translator/engine.py +++ b/src/flowx/translator/engine.py @@ -159,7 +159,7 @@ def translate_pipeline( activity_ir, context = _dispatch_activity(adf_activity, context, definitions) translated_activities.append(activity_ir) - strategy, skill = classify_activity(adf_activity.type) + strategy = classify_activity(adf_activity.type) if strategy is TranslationStrategy.DETERMINISTIC: deterministic_count += 1 elif strategy is TranslationStrategy.AGENTIC: @@ -267,14 +267,13 @@ def _collect_agentic_gaps(activities: list[AdfActivity], warnings: list[str]) -> def _walk(acts: list[AdfActivity] | None) -> None: for act in acts or []: - strategy, skill = classify_activity(act.type) + strategy = classify_activity(act.type) if strategy is not TranslationStrategy.DETERMINISTIC and act.name not in seen: seen.add(act.name) gaps.append( AgenticGap( activity_name=act.name, activity_type=act.type, - recommended_skill=skill, raw_definition=act.raw if act.raw is not None else act.type_properties, ) ) @@ -366,13 +365,16 @@ def _dispatch_activity( return result, context case _: - strategy, skill = classify_activity(activity.type) - reason = f"Agentic skill: {skill}" if skill else f"No translator for type '{activity.type}'" + strategy = classify_activity(activity.type) + reason = ( + "Requires agentic (LLM-assisted) translation from the ADF/ARM JSON" + if strategy is TranslationStrategy.AGENTIC + else f"No translator for type '{activity.type}'" + ) placeholder = PlaceholderActivity( **base_kwargs, original_type=activity.type, comment=reason, - agentic_skill=skill, raw_definition=activity.raw, ) context = context.with_activity(activity.name, placeholder) diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py index a89456f..f729c3a 100644 --- a/tests/integration/test_end_to_end.py +++ b/tests/integration/test_end_to_end.py @@ -313,14 +313,10 @@ def test_deterministic_count(self, adf_definitions): assert len(det_items) == inv.deterministic_count def test_agentic_activities_identified(self, adf_definitions): - """Agentic activities are identified with correct skill mapping.""" + """Agentic activities are identified and counted.""" inv = build_inventory(adf_definitions) agentic_items = [i for i in inv.items if i.strategy is TranslationStrategy.AGENTIC] assert len(agentic_items) == inv.agentic_count - for item in agentic_items: - assert item.agentic_skill is not None - # All agentic skills should reference a known skill - assert "adf-to-databricks" in item.agentic_skill def test_mixed_pipeline_classification(self, adf_definitions): """Mixed pipeline has both deterministic and agentic items.""" diff --git a/tests/unit/test_adf_loader.py b/tests/unit/test_adf_loader.py index f56b4d8..8213742 100644 --- a/tests/unit/test_adf_loader.py +++ b/tests/unit/test_adf_loader.py @@ -83,23 +83,20 @@ def test_classify_deterministic_types(self): assert DETERMINISTIC_TYPES == expected for atype in expected: - strategy, skill = classify_activity(atype) + strategy = classify_activity(atype) assert strategy is TranslationStrategy.DETERMINISTIC, f"{atype} should be DETERMINISTIC" - assert skill is None, f"{atype} should have no agentic skill" def test_classify_agentic_types(self): - """All agentic types are classified with correct skill names.""" - for atype, expected_skill in AGENTIC_TYPES.items(): - strategy, skill = classify_activity(atype) + """All agentic types are classified as AGENTIC.""" + for atype in AGENTIC_TYPES: + strategy = classify_activity(atype) assert strategy is TranslationStrategy.AGENTIC, f"{atype} should be AGENTIC" - assert skill == expected_skill, f"{atype} skill should be {expected_skill}" def test_classify_unknown_types(self): """Unknown activity types are classified as UNSUPPORTED.""" for unknown_type in ("Bogus", "SomeFutureActivity", "MagicTransform", ""): - strategy, skill = classify_activity(unknown_type) + strategy = classify_activity(unknown_type) assert strategy is TranslationStrategy.UNSUPPORTED - assert skill is None # --------------------------------------------------------------------------- @@ -124,19 +121,11 @@ def test_build_inventory_has_pipeline_names(self, adf_definitions): for item in inv.items: assert item.pipeline_name in pipeline_names - def test_build_inventory_agentic_items_have_skills(self, adf_definitions): - """Agentic inventory items have a non-None skill.""" + def test_build_inventory_agentic_count_matches_items(self, adf_definitions): + """The agentic count matches the number of items classified AGENTIC.""" inv = build_inventory(adf_definitions) - for item in inv.items: - if item.strategy is TranslationStrategy.AGENTIC: - assert item.agentic_skill is not None - - def test_build_inventory_deterministic_items_no_skill(self, adf_definitions): - """Deterministic inventory items have no agentic skill.""" - inv = build_inventory(adf_definitions) - for item in inv.items: - if item.strategy is TranslationStrategy.DETERMINISTIC: - assert item.agentic_skill is None + agentic_items = [i for i in inv.items if i.strategy is TranslationStrategy.AGENTIC] + assert len(agentic_items) == inv.agentic_count # --------------------------------------------------------------------------- diff --git a/tests/unit/test_until_agentic_handler.py b/tests/unit/test_until_agentic_handler.py index aa97a05..64d02e8 100644 --- a/tests/unit/test_until_agentic_handler.py +++ b/tests/unit/test_until_agentic_handler.py @@ -48,7 +48,6 @@ def test_nested_until_gap_carries_full_arm_json(): # full ARM JSON, not just typeProperties: name + nested loop body present assert raw.get("name") == "Poll Until Ready" assert raw["typeProperties"]["activities"][0]["name"] == "Wait A Bit" - assert until_gaps[0].recommended_skill == "adf-to-databricks:adf-pipeline-converter" def test_until_placeholder_ir_node_carries_arm_json(): @@ -68,4 +67,3 @@ def _find(tasks): ph = _find(report.pipeline.tasks) assert ph is not None and ph.raw_definition is not None assert ph.raw_definition.get("type") == "Until" - assert ph.agentic_skill == "adf-to-databricks:adf-pipeline-converter" From 974a2c9df7770d94282ae982b14ef36903bd0e46 Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Wed, 15 Jul 2026 08:55:58 -0400 Subject: [PATCH 21/77] Fix issue templates (#6) ## Changes This PR fixes issue templates for flowx. ### Linked issues N/A ### Tests - [x] manually tested - [ ] added unit tests - [ ] added integration tests --- .build-constraints.txt | 6 +++--- .github/ISSUE_TEMPLATE/{bug_report.yml => bug.yml} | 0 .github/ISSUE_TEMPLATE/{feature_request.yml => feature.yml} | 0 Makefile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename .github/ISSUE_TEMPLATE/{bug_report.yml => bug.yml} (100%) rename .github/ISSUE_TEMPLATE/{feature_request.yml => feature.yml} (100%) diff --git a/.build-constraints.txt b/.build-constraints.txt index ee7a59e..3a6ff79 100644 --- a/.build-constraints.txt +++ b/.build-constraints.txt @@ -1,6 +1,6 @@ -hatchling==1.30.1 \ - --hash=sha256:161eacafb3c6f91526e92116d21426369f2c36e98c36a864f11a96345ad4ee31 \ - --hash=sha256:eee4fd45357f72ebb3d7a42e5d72cfb5e29ed426d79e8836288926c4258d5f2e +hatchling==1.31.0 \ + --hash=sha256:6b48ad4068a482ed7239b3a8215bc55b47aad3345d58dfc94e553c5d2d46211b \ + --hash=sha256:aac80bec8b6fe35e8480f1c335be8910fa210a0e6f735a139be205dadcacb544 packaging==26.2 \ --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.yml rename to .github/ISSUE_TEMPLATE/bug.yml diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.yml rename to .github/ISSUE_TEMPLATE/feature.yml diff --git a/Makefile b/Makefile index a99a435..1a82a25 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ lock-dependencies: uv run --exact --all-extras --group yq tomlq -r '.["build-system"].requires[]' pyproject.toml | \ uv pip compile --generate-hashes --universal --no-header - > build-constraints-new.txt mv build-constraints-new.txt .build-constraints.txt - perl -pi -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple"|g' uv.lock + perl -pi -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple"|g; s|url = "https://[^/"]+/packages/|url = "https://files.pythonhosted.org/packages/|g; s|, size = \d+||g' uv.lock $(MAKE) requirements requirements: From 657f31484e502478cc775eaf654f6e7d247b547f Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:24:42 -0400 Subject: [PATCH 22/77] Add dependabot and codecov (#7) ## Changes This PR adds dependabot and codecov configuration and updates CI to improve security. ### Linked issues N/A ### Tests - [x] manually tested - [ ] added unit tests - [ ] added integration tests --- .github/codecov.yml | 10 ++++++++++ .github/dependabot.yml | 10 ++++++++++ .github/workflows/docs-release.yml | 2 +- .github/workflows/push.yml | 14 ++++++++++++++ CODEOWNERS | 1 + CODEOWNERS.txt | 0 pyproject.toml | 4 +++- 7 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 .github/codecov.yml create mode 100644 .github/dependabot.yml create mode 100644 CODEOWNERS delete mode 100644 CODEOWNERS.txt diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000..8ad3f44 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,10 @@ +coverage: + status: + project: + default: + target: auto + threshold: 0.5% # The minimum coverage threshold for the project + patch: + default: + target: auto + threshold: 0.5% # The minimum coverage threshold for the patch diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..fd193b1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "uv" + directory: "/" + cooldown: + default-days: 7 + exclude: + - "databricks*" + schedule: + interval: "daily" diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml index d71e6ab..4c06abf 100644 --- a/.github/workflows/docs-release.yml +++ b/.github/workflows/docs-release.yml @@ -29,7 +29,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: - bun-version: latest + bun-version: 1.3.14 - name: Scrub internal proxy URLs from bun.lock # The Databricks-internal npm proxy is unreachable from public runners; diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5063847..24baca4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -5,8 +5,21 @@ on: push: branches: [main] +permissions: + contents: read + jobs: + # Gate all downstream jobs behind a single check so PRs from forks (no access to the + # tool environment) and draft PRs do not trigger the expensive acceptance suite. + # PRs from forks are to be tested by the reviewer(s) / maintainer(s) before merging. + not-a-fork: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork + steps: + - run: echo "Not a fork PR, proceeding" + ci: + needs: not-a-fork runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -26,6 +39,7 @@ jobs: || { echo "requirements.txt is stale. Run 'make requirements' (or 'make precommit') and commit it."; exit 1; } fmt: + needs: not-a-fork runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..d11692b --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @databricks-solutions/flowx-maintainers \ No newline at end of file diff --git a/CODEOWNERS.txt b/CODEOWNERS.txt deleted file mode 100644 index e69de29..0000000 diff --git a/pyproject.toml b/pyproject.toml index 057e8fb..2792f4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,9 @@ yq = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "hatchling>=1.90,<2.0" +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] From b613600e8503bbed512c6b93aaa264bee08d82fe Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 20 Jul 2026 12:53:58 -0700 Subject: [PATCH 23/77] Add source-neutral Airflow and dbt migration foundations --- pyproject.toml | 5 + skills/flowx-convert/SKILL.md | 453 ++-------------- skills/flowx-convert/sources/adf.md | 159 ++++++ skills/flowx-convert/sources/airflow.md | 43 ++ skills/flowx-discover/SKILL.md | 284 ++--------- skills/flowx-discover/sources/adf.md | 100 ++++ skills/flowx-discover/sources/airflow.md | 64 +++ skills/flowx-migrate/SKILL.md | 36 +- skills/flowx-migrate/references/workflow.md | 2 +- src/flowx/adapter/__main__.py | 108 +++- src/flowx/bundler/dab_writer.py | 13 + src/flowx/dbt/__init__.py | 1 + src/flowx/dbt/manifest.py | 155 ++++++ src/flowx/ir_serde.py | 482 ++++++++++++++++++ src/flowx/models/ir.py | 47 ++ .../activity_preparers/dbt_factory.py | 188 +++++++ .../preparer/activity_preparers/notebook.py | 8 +- src/flowx/preparer/workflow_preparer.py | 3 + src/flowx/sources/__init__.py | 71 +++ .../{translator => sources/adf}/__init__.py | 0 .../{parser => sources/adf}/ir_rewriter.py | 0 .../adf_loader.py => sources/adf/loader.py} | 0 .../adf}/query_analysis.py | 0 .../engine.py => sources/adf/translate.py} | 457 +---------------- .../adf/translators}/__init__.py | 0 .../adf/translators}/append_variable.py | 0 .../adf/translators}/copy.py | 2 +- .../adf/translators}/databricks_job.py | 2 +- .../adf/translators}/delete.py | 2 +- .../adf/translators}/execute_pipeline.py | 2 +- .../adf/translators}/filter.py | 0 .../adf/translators}/for_each.py | 2 +- .../adf/translators}/if_condition.py | 2 +- .../adf/translators}/lookup.py | 2 +- .../adf/translators}/notebook.py | 2 +- .../adf/translators}/resolve.py | 0 .../adf/translators}/set_variable.py | 0 .../adf/translators}/spark_jar.py | 2 +- .../adf/translators}/spark_python.py | 2 +- .../adf/translators}/switch.py | 4 +- .../adf/translators}/wait.py | 2 +- .../adf/translators}/web_activity.py | 2 +- src/flowx/sources/airflow/__init__.py | 0 src/flowx/sources/airflow/convert.py | 57 +++ src/flowx/sources/airflow/discover.py | 114 +++++ src/flowx/sources/airflow/loader.py | 295 +++++++++++ tests/conftest.py | 2 +- tests/integration/test_adf_live.py | 4 +- tests/integration/test_end_to_end.py | 4 +- tests/integration/test_golden_output.py | 4 +- tests/integration/test_path_equivalence.py | 9 +- .../resources/airflow/orders_analytics_dag.py | 46 ++ tests/unit/test_adapter.py | 50 +- tests/unit/test_adf_loader.py | 2 +- tests/unit/test_dbt_factory_preparer.py | 130 +++++ tests/unit/test_dbt_manifest.py | 124 +++++ tests/unit/test_ir_rewriter.py | 2 +- tests/unit/test_merge_agentic.py | 2 +- tests/unit/test_preparers.py | 6 +- tests/unit/test_profile_report.py | 2 +- tests/unit/test_query_analysis.py | 2 +- tests/unit/test_resolve_field.py | 2 +- tests/unit/test_source_router.py | 98 ++++ tests/unit/test_translators.py | 130 ++--- tests/unit/test_until_agentic_handler.py | 4 +- .../unit/test_web_body_and_param_defaults.py | 4 +- 66 files changed, 2548 insertions(+), 1252 deletions(-) create mode 100644 skills/flowx-convert/sources/adf.md create mode 100644 skills/flowx-convert/sources/airflow.md create mode 100644 skills/flowx-discover/sources/adf.md create mode 100644 skills/flowx-discover/sources/airflow.md create mode 100644 src/flowx/dbt/__init__.py create mode 100644 src/flowx/dbt/manifest.py create mode 100644 src/flowx/ir_serde.py create mode 100644 src/flowx/preparer/activity_preparers/dbt_factory.py create mode 100644 src/flowx/sources/__init__.py rename src/flowx/{translator => sources/adf}/__init__.py (100%) rename src/flowx/{parser => sources/adf}/ir_rewriter.py (100%) rename src/flowx/{parser/adf_loader.py => sources/adf/loader.py} (100%) rename src/flowx/{translator => sources/adf}/query_analysis.py (100%) rename src/flowx/{translator/engine.py => sources/adf/translate.py} (73%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/__init__.py (100%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/append_variable.py (100%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/copy.py (99%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/databricks_job.py (93%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/delete.py (96%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/execute_pipeline.py (98%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/filter.py (100%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/for_each.py (98%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/if_condition.py (99%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/lookup.py (99%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/notebook.py (99%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/resolve.py (100%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/set_variable.py (100%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/spark_jar.py (96%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/spark_python.py (96%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/switch.py (97%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/wait.py (93%) rename src/flowx/{translator/activity_translators => sources/adf/translators}/web_activity.py (98%) create mode 100644 src/flowx/sources/airflow/__init__.py create mode 100644 src/flowx/sources/airflow/convert.py create mode 100644 src/flowx/sources/airflow/discover.py create mode 100644 src/flowx/sources/airflow/loader.py create mode 100644 tests/resources/airflow/orders_analytics_dag.py create mode 100644 tests/unit/test_dbt_factory_preparer.py create mode 100644 tests/unit/test_dbt_manifest.py create mode 100644 tests/unit/test_source_router.py diff --git a/pyproject.toml b/pyproject.toml index 057e8fb..9b0c2ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,5 +84,10 @@ exclude = ["templates/*"] [tool.ruff.lint] select = ["E", "F", "I"] +[tool.ruff.lint.per-file-ignores] +# Sample Airflow DAG fixtures are parsed statically, never executed; they +# reference runtime globals (spark, dbutils) and Airflow imports by design. +"tests/resources/airflow/*" = ["F821", "F401"] + [tool.ruff.lint.isort] known-first-party = ["flowx"] diff --git a/skills/flowx-convert/SKILL.md b/skills/flowx-convert/SKILL.md index 5cd2b43..4a1221e 100644 --- a/skills/flowx-convert/SKILL.md +++ b/skills/flowx-convert/SKILL.md @@ -1,433 +1,90 @@ --- name: flowx-convert description: > - Translate parsed ADF pipeline AST into Databricks IR (intermediate representation). - Runs deterministic translators for known activity types, then performs agentic - (LLM-assisted) translation for the remaining gaps. + Translate a source's parsed inventory into Databricks IR (intermediate representation): run + deterministic translators for known types, then agentic (LLM-assisted) translation for the gaps. + Phase 2 of the flowx migration workflow; routes to a source-specific guide. triggers: - - "translate ADF" - - "convert ADF" - - "translate pipelines" - "convert pipelines" + - "translate pipelines" + - "convert ADF" + - "convert airflow" - "run translation" --- -# Convert ADF to Databricks IR +# Convert Source to Databricks IR -Convert the parsed ADF inventory into Databricks intermediate representation (IR) using deterministic translators for known types and agentic fallback for unknown types. +Convert a source's discovered inventory into Databricks intermediate representation (IR). This is +phase 2 of the flowx migration workflow; it produces a transient translation report under +`/.work/` that the `flowx-package` skill turns into a Databricks Asset Bundle. -## Context +Translation is **source-specific** (ADF activity translators vs. Airflow operator mapping), so this +skill routes to the right source guide. The shared mechanics — how to run the phase, the report +contract, and the `inspect`/`modify`/`merge_agentic` machinery — live here. -This is phase 2 of the flowx migration workflow. It consumes the ADF source (profiled by the `discover` skill) and produces a translation report — a transient intermediate under `/.work/` — that the `package` skill uses to generate Databricks Declarative Automation Bundles. It shares the single migration `` with the other phases. +## Step 1 — Identify the source (required) -The translation follows a **deterministic-first** strategy: -1. Activities with known, well-defined mappings are translated by built-in Python translators -2. Activities that require interpretation, expression conversion, or lack a Python translator are handled by agentic (LLM-assisted) translation performed by the agent +Use the same source the discover phase used (it is recorded as `"source"` in +`/metadata/inventory.json`): -## How to run this skill — MCP tools or venv CLI +- **Azure Data Factory / Fabric Data Factory** → source `adf` → read `sources/adf.md` +- **Apache Airflow** → source `airflow` → read `sources/airflow.md` -This phase runs one of two ways; run the **`setup`** skill first if you haven't. +There is no default source. Every phase invocation passes `--source ` explicitly. -- **MCP tool (Databricks Genie Code, or a local stdio registration) — the only path in Genie Code:** - call the single **`flowx`** tool (one command per step) and run **no** `python3`/`$PY`/`bash` - commands. The `"$PY" -m …` snippets in the steps below are the **local-CLI fallback only** — ignore - them on this path. Map the steps to: +## Step 2 — Follow the source guide - ``` - flowx(command="convert", parameters={"output_dir": "", "pipeline": ""}) - # convert reuses the discovered output_dir on the server; only pass "adf_definitions" (inline ARM - # JSON) if you are converting without a prior discover on this server. - flowx(command="inspect", parameters={"report_path": "/.work/translation_report.json", "answers": [...]}) - flowx(command="apply_answers", parameters={"report_path": "...", "answers": ["id=value", ...], "output_dir": "", "lookup_csv": ""}) - flowx(command="merge_agentic", parameters={"report_path": "...", "agentic_results_dir": "", "output_path": ""}) - ``` +Read the matching `sources/.md` and follow it. ADF has a rich deterministic-first + +agentic-gap flow with just-in-time configuration; Airflow is currently deterministic-only. + +## How to run this phase — MCP tool or venv CLI + +Run the **`setup`** skill first if you haven't. - Use the tool results in place of reading the files directly. `command="merge_agentic"` covers the - agentic `--merge-agentic` step shown later in this skill. +- **MCP tool (Genie Code, or local stdio):** call the single **`flowx`** tool with + `command="convert"` and `parameters` including `"source": ""` and `"output_dir": ""`. -- **venv CLI (local, no MCP server):** ensure the venv exists (`setup` Path B / `bootstrap.sh`), then - run the commands below with the venv interpreter (from the marker file `/.migration-venv`) - and `src/` on `PYTHONPATH` (use `$PY` anywhere a command shows `python3`): +- **venv CLI (local):** ```bash export PYTHONPATH="/src" PY="$(cat /.migration-venv)" - "$PY" -m flowx.adapter convert --output-dir + "$PY" -m flowx.adapter convert --source --source-path --output-dir [--pipeline ] ``` - If Python or pip is missing, `bootstrap.sh` prints a warning telling the user what to install — - relay it and stop until they have Python 3.12+ and pip. - -## Workflow - -Follow these steps in order: - -### Step 0 — Gather phase inputs - -Run the adapter inputs subcommand so the agent surfaces the free-text -options the phase needs (inventory path, ADF source dir, output -directory): - -```bash -"$PY" -m flowx.adapter inputs convert -``` - -The JSON response carries the prompts and defaults; collect answers from the user -(or fall back to the defaults). Keep them in conversation context — the same shared -`` is used by every phase. - -### Step 1 — Locate the inventory - -The discover phase wrote `/metadata/inventory.json` (and `profile_report.csv`). If the -shared `` is not already in conversation context, ask the user: - -> Which migration output directory did the discover phase use? (default: `./flowx_output`) - -Validate `/metadata/inventory.json` exists and is well-formed. - -### Step 2 — Run deterministic translation - -Execute the translation engine on all deterministic activities: - -```bash -# Unified runner (recommended): `"$PY" -m flowx.adapter convert ...` -# forwards to the engine below; --adf-source-path aliases --source-dir. -"$PY" -m flowx.translator.engine \ - --source-dir \ - --output-dir \ - [--pipeline ] -``` - -Where: -- `` is the original ADF JSON directory (the same `--source-dir` used by discover) -- `` is the **shared migration output directory** (default: `./flowx_output`) — the - same one discover used -- `` (optional) — when provided, translates only the named pipeline. **Always pass `--pipeline` when the user has specified a specific pipeline to migrate**, matching the value passed to the discover phase. - -The translation report and intermediate IR are written to the **transient** `/.work/` -folder (`translation_report.json`, per-pipeline IR, `gaps.json`). These are consumed by the steps -below and the package phase, then pruned — they are not kept artifacts. - -### Step 3 — Read the translation report - -Read `/.work/translation_report.json`. It has this structure: - -```json -{ - "inventory_path": "/path/to/inventory.json", - "generated_at": "2026-04-07T12:30:00Z", - "translations": [ - { - "pipeline": "ETL_Main", - "activity": "CopyFromBlob", - "type": "Copy", - "strategy": "deterministic", - "status": "translated", - "ir": { - "task_key": "copy_from_blob", - "task_type": "notebook_task", - "notebook_path": "notebooks/copy_from_blob.py", - "parameters": { "source": "abfss://...", "target": "..." } - } - }, - { - "pipeline": "ETL_Main", - "activity": "TransformData", - "type": "ExecuteDataFlow", - "strategy": "agentic", - "status": "pending", - "raw_activity_json": { "...": "..." } - } - ], - "summary": { - "total": 47, - "deterministic_translated": 35, - "agentic_pending": 10, - "failed": 2 - } -} -``` - -### Step 4 — Handle agentic gaps - -For each translation with `"status": "pending"` and `"strategy": "agentic"`, perform LLM-assisted translation from the activity's ARM JSON, routing by activity type. - -Every agentic gap in the translation report carries the activity's **full ADF/ARM JSON** under `raw_activity_json` (engine field `raw_definition`), and the generated placeholder notebook embeds the same JSON in a fenced `json` block. This holds for nested activities too — an `Until` inside an `IfCondition` / `Switch` / `ForEach` is reported as its own gap. Always translate from this ARM JSON. - -**Until activities (agent-based handler):** -Databricks Lakeflow Jobs have no native repeat-until loop, so translate the `Until` from its ARM JSON into a single Python notebook task implementing a bounded polling loop. From the embedded JSON, read: -- `typeProperties.expression` — the ADF exit condition (e.g. `@or(equals(variables('jobStatus'),'succeeded'), equals(variables('jobStatus'),'failed'))`); convert it into the Python `while not ():` guard. -- `typeProperties.timeout` — wrap the loop in a wall-clock deadline (`time.monotonic()`), raising on timeout. -- `typeProperties.activities` — the loop body (e.g. a `Wait`, a polling `WebActivity`, a `SetVariable` that captures the next status); translate each child inline so the whole loop runs in one notebook. -Read the loop variables from `dbutils.widgets`, surface the final state as a task value, and write the result over the placeholder notebook's `raise NotImplementedError` cell. Perform the translation directly from the same ARM JSON. - -**ExecuteDataFlow activities:** -Translate the data flow directly from the raw activity JSON and associated data flow definition, using: -- The raw `typeProperties` from the ADF activity -- The data flow JSON definition (if available in the source directory under `dataflow/`) -- The linked service configurations for source/sink connections -- Target catalog and schema for the SDP pipeline or PySpark notebook output - -**Control flow activities (Switch, Until, Wait, Filter, AppendVariable):** -Translate the control-flow activity directly from the raw activity JSON, using: -- The full pipeline JSON containing the activity -- Any nested activities within the control flow -- Variable definitions from the pipeline -- The desired Databricks task type mapping - -**Stored procedures and external calls (SqlServerStoredProcedure, AzureFunction, WebHook, Custom):** -Translate the activity directly from the raw activity JSON, using: -- The linked service configuration for the target system -- Connection details and authentication method -- Any parameters or request bodies - -**Complex expressions:** -If any activity (deterministic or agentic) contains ADF expressions that the deterministic translator could not resolve, translate them directly, using: -- The raw expression string (e.g., `@pipeline().parameters.inputPath`) -- The expression context (pipeline parameters, variables, activity outputs) -- The target format (Python f-string, Spark SQL, task parameter reference) - -**Trigger definitions:** -Translate the trigger directly, using: -- The trigger JSON definition -- The associated pipeline references -- Target: Databricks job schedule configuration (quartz_cron_expression, periodic, or file_arrival) - -### Step 5 — Collect agentic results - -Each resolved agentic gap produces one translation result. Write them into -`/agentic_results/` as one JSON file per activity (the filename is -arbitrary, e.g. `__.json`). Each file MUST use this schema: - -```json -{ - "activity_name": "", - "pipeline": "", - "task": { - "type": "NotebookActivity", - "name": "", - "task_key": "", - "notebook_path": "/Workspace/.../your_translated_notebook" - } -} -``` - -- `activity_name` (required) — matches the `name` of the placeholder task in the - report (the merge locates it by name, recursing into IfCondition / ForEach / - Switch containers, so nested gaps like an `Until` are found). -- `pipeline` (optional) — only needed to disambiguate multi-pipeline reports. -- `task` (required) — the replacement IR task. The most portable form is a - `NotebookActivity` whose `notebook_path` points at a notebook you have written - to the workspace; the package phase references it directly. `task_key` and - `depends_on` are inherited from the placeholder when omitted, so dependency - edges are preserved. + `--source` is required. The convert phase writes `/.work/translation_report.json`. -### Step 6 — Merge agentic results +## The translation report contract (shared) -Fold the results into the translation report (placeholders are replaced in place): +Every source's convert phase writes `/.work/translation_report.json` in the same shape: +a single pipeline IR dict (keys `name`, `tasks`, optional `schedule`/`parameters`), or a +`{"pipelines": [...]}` wrapper for many. The `flowx-package` phase consumes this regardless of +source. IR serialization is source-neutral (`flowx.ir_serde`), so the report format is identical +across ADF and Airflow. -```bash -"$PY" -m flowx.translator.engine \ - --merge-agentic \ - --report /.work/translation_report.json \ - --agentic-results -``` +## Shared adapter commands -Equivalently via the unified runner: `"$PY" -m flowx.adapter convert --merge-agentic --report /.work/translation_report.json --agentic-results `. Add `--output ` to write a copy instead of overwriting the report. The command exits non-zero if any result could not be matched to a placeholder. +These operate on the report, not on a source's raw definitions, so they are the same for every +source (the ADF guide uses them heavily; Airflow currently needs only the base convert): -This updates `/.work/translation_report.json` with the agentic results merged in, changing their status from `pending` to `translated` (or `failed` if the agentic skill could not produce a result). +- `inspect ` — emit the full just-in-time option schema (each option annotated with a + `show_when` condition). Walk it locally; ask an option only when its `show_when` is satisfied. +- `modify --output-dir --answer OPTION_ID=VALUE ...` — validate and apply collected + answers, writing `.work/translation_report.stamped.json` + `metadata/configuration.json`. +- `merge_agentic --report --agentic-results ` — fold agent-produced per-activity + translations into the report (placeholders replaced in place). -### Step 6.1 — Gather just-in-time translation configuration - -Run `inspect` **once** to get the full option schema, then drive the whole question chain yourself — -do **not** re-run `inspect` per follow-up: - -```bash -"$PY" -m flowx.adapter inspect /.work/translation_report.json -``` - -It returns every option the report can raise, each annotated with a `show_when` condition: - -```json -{"pipelines": [{"pipeline_name": "...", "options": [ - {"option_id": "notify_destination", "prompt": "...", "rationale": "...", - "choices": [{"value": "...", "label": "...", "description": "..."}], - "free_text": false, "default": "keep", "show_when": []}, - {"option_id": "notify_slack_url", "prompt": "...", "free_text": true, "default": "", - "show_when": [{"option_id": "notify_destination", "in": ["slack"]}]} -]}]} -``` - -Walk it locally: - -1. **Ask an option only when its `show_when` is satisfied** — every clause `{option_id, in:[values]}` - must match an answer you've already collected (empty `show_when` = always ask). So `notify_slack_url` - surfaces only after `notify_destination=slack`; the metadata-driven `access`/`size`/`lookup_tool` - chain surfaces only after `metadata_driven_consolidate=consolidate`, etc. Present each option's - `prompt`/`rationale` and `choices`; honor the `default`. -2. **Validate each answer** against `choices` (a `free_text` option — empty `choices` — accepts any - value; blank skips an optional one). -3. **Perform data actions inline** when an answer calls for it — e.g. when - `metadata_driven_lookup_tool=have`, run the lookup query with your database tool to get the rows. -4. When every applicable option is answered, apply them **in one `modify` call** (Step 6.2) with all - answers as `--answer OPTION_ID=VALUE` flags. `modify` validates every answer server-side. - -**Activity→Notify (`activity_and_notify`) motifs.** When **any** activity (Copy, -Notebook, Lookup, stored procedure, …) is followed by notification Web -activities, the adapter raises `notify_destination`: -`keep` (default) leaves the Web activities to translate directly — nothing is -collapsed. Any other value (`email`, `slack`, `teams`, `pagerduty`, `webhook`) -collapses the pattern: the upstream activity becomes the task and the -notifications become Databricks job-task `on_success`/`on_failure` notifications -routed to that destination (the ADF Web activity URL/body is not used). The schema includes **one -follow-up per Databricks-SDK field** of each destination, each gated by -`show_when: [{notify_destination, in:[]}]`; ask the chosen destination's fields (required -first) once the user picks it: - -| Destination | Chained field options (SDK arg) | -|-------------|---------------------------------| -| `email` | `notify_email_recipients` (`addresses`, comma-separated) | -| `slack` | `notify_slack_url` (`url`), `notify_slack_channel_id` (`channel_id`, optional), `notify_slack_oauth_token` (`oauth_token`, optional) | -| `teams` | `notify_teams_url` (`url`) | -| `pagerduty` | `notify_pagerduty_integration_key` (`integration_key`) | -| `webhook` | `notify_webhook_url` (`url`), `notify_webhook_username` (`username`, optional), `notify_webhook_password` (`password`, optional) | - -All destinations also take an optional `notify_destination_name` and -`notify_events` (both/on_failure/on_success). Optional fields left blank are -omitted so the SDK applies its defaults. For **non-email** destinations, the -`modify` phase creates (or reuses by display name) the Databricks notification -destination via the SDK **as soon as you submit the answers** — it validates the -config immediately and bakes the resolved destination id into the modified report, -so package just wires `webhook_notifications` to that id (no further SDK call). -This requires workspace auth at `modify` time; if creation fails there, the id is -left unresolved and package retries or emits a `notification_destination` setup task. -**Email** needs no destination — it uses raw `email_notifications` and is never -created via the SDK. - -When the metadata-driven flow ends with `metadata_driven_lookup_tool=have` -and the agent has a database tool (Genie, MCP SQL, or a workspace SDK), -run the lookup query directly to obtain the rows as CSV. When the answer is -`none`, ask the user for a CSV file path or a literal CSV string. Pass it inline -to `modify` via `--lookup-csv` (no intermediate JSON file): - -```bash -"$PY" -m flowx.adapter modify \ - /.work/translation_report.json \ - --output-dir \ - --answer metadata_driven_consolidate=consolidate \ - --answer metadata_driven_access=yes \ - --lookup-csv "" -``` - -When no metadata-driven motif is consolidated, `--lookup-csv` is omitted. In that default -(non-consolidated) case the motif becomes a Databricks **for-each task** that runs one Spark JDBC -read per source table — its iteration inputs are the resolved lookup rows when available, otherwise a -control-table lookup task seeds them at run time. (Consolidating instead emits one managed Lakeflow -Connect ingestion pipeline.) - -#### Legacy flow details - -Before writing the final report, surface any pipeline-modifier options the -IR raises (Copy Data paradigm, non-Databricks task compute, Lakeflow Connect -opt-in, Databricks task compute). Use the adapter CLI bridge: - -```bash -"$PY" -m flowx.adapter inspect /.work/translation_report.json -``` - -The command emits JSON: - -```json -{ - "pipelines": [ - { - "pipeline_name": "ETL_Main", - "options": [ - { - "option_id": "copy_activity_paradigm", - "prompt": "How should Copy Data activities targeting Delta be implemented?", - "rationale": "...", - "options": [{"value": "notebook", "label": "...", "description": "..."}, ...], - "affected_task_keys": ["copy_orders", "copy_customers"], - "default": "notebook" - }, - ... - ] - } - ] -} -``` - -For each option, prompt the user with the rationale, options, and the task keys it -affects. Use the default when the user defers. Then apply the collected answers as -`--answer OPTION_ID=VALUE` flags: - -```bash -"$PY" -m flowx.adapter modify \ - /.work/translation_report.json \ - --output-dir \ - --answer copy_activity_paradigm=sdp \ - --answer non_databricks_task_compute=serverless \ - --answer use_lakeflow_connectors=lakeflow_connect -``` - -`modify` writes two things under the shared ``: -- `.work/translation_report.stamped.json` — the configuration-stamped IR the package phase consumes -- `metadata/configuration.json` — the collected answers, kept as the migration's configuration record - -The package phase (next skill) reads the stamped report from `.work/` automatically. -When no options are raised, the inspect output is `{"pipelines": [{"pipeline_name": "...", "options": []},...]}` — skip `modify`; package falls back to the un-stamped report. - -### Step 7 — Present translation summary - -Display a summary to the user: - -``` -Translation Summary -=================== -Total activities: 47 -Deterministic translated: 35 (74.5%) -Agentic translated: 8 (17.0%) -Failed: 4 ( 8.5%) - -Overall coverage: 91.5% - -Failed translations: - - ETL_Main / RunSSIS (ExecuteSSISPackage) — no translator available - - ETL_Main / CustomTask (Custom) — agentic skill returned error - ... - -Generated artifacts (transient, under /.work/): - - translation_report.json - - per-pipeline IR (43 files) - - gaps.json -``` - -If coverage is below 100%, explain the options for failed translations: -1. Manual notebook creation for unsupported types -2. Retry agentic translation with additional context -3. Skip the activity and add a placeholder task in the DAB - -## Reference - -See `references/activity-mapping.md` for the complete mapping between ADF activity types and translation strategies. - -## Examples - -- "Convert the ADF pipelines" -- "Convert ADF to Databricks" -- "Run the translation on the inventory from the profile step" -- "Convert the parsed pipelines using deterministic + agentic" -- "Convert only the pl_demo_01 pipeline" - -## Output Artifacts - -The convert phase writes only **transient** intermediates, under `/.work/` (consumed -by `modify`/`package`, then pruned — not kept): +## Output artifacts (shared, transient under `/.work/`) | File | Description | |---|---| -| `.work/translation_report.json` | Full translation report with IR for all activities | +| `.work/translation_report.json` | Full translation report with IR for all tasks | | `.work/.json` | Per-pipeline Databricks IR | -| `.work/gaps.json` | Agentic gaps awaiting skill conversion | +| `.work/gaps.json` | Agentic gaps awaiting LLM-assisted conversion (ADF) | | `.work/translation_report.stamped.json` | Configuration-stamped report (written by `modify`) | + +## Reference + +- `sources/adf.md` — ADF translation: deterministic engine, agentic gaps, just-in-time config, + notify motifs, metadata-driven consolidation. See also `references/activity-mapping.md`. +- `sources/airflow.md` — Airflow translation (operator → IR, deterministic-only today). diff --git a/skills/flowx-convert/sources/adf.md b/skills/flowx-convert/sources/adf.md new file mode 100644 index 0000000..2a605f3 --- /dev/null +++ b/skills/flowx-convert/sources/adf.md @@ -0,0 +1,159 @@ +# Convert — Azure Data Factory + +Source guide for `--source adf`. Translate the discovered ADF inventory into Databricks IR using +deterministic translators for known activity types and agentic (LLM-assisted) fallback for the +rest. See the parent `SKILL.md` for how to run the phase, the report contract, and the shared +`inspect`/`modify`/`merge_agentic` commands. + +Translation is **deterministic-first**: +1. Activities with known mappings are translated by built-in Python translators. +2. Activities needing interpretation, expression conversion, or lacking a translator are handled by + agentic translation the agent performs from the ARM JSON. + +## Step 1 — Locate the inventory + +The discover phase wrote `/metadata/inventory.json`. Confirm the shared `` +(default `./flowx_output`) and that the inventory exists. + +## Step 2 — Run deterministic translation + +```bash +"$PY" -m flowx.adapter convert --source adf \ + --adf-source-path \ + --output-dir \ + [--pipeline ] +``` + +`` is the same ADF JSON directory discover used. Always pass `--pipeline` when the +user scoped to a single pipeline. The report and intermediate IR are written to the transient +`/.work/` folder (`translation_report.json`, per-pipeline IR, `gaps.json`). + +## Step 3 — Read the translation report + +Read `/.work/translation_report.json`. Each translation entry carries `pipeline`, +`activity`, `type`, `strategy` (`deterministic`/`agentic`), `status`, and either the translated +`ir` or a `raw_activity_json` for pending agentic gaps. + +## Step 4 — Handle agentic gaps + +For each translation with `"status": "pending"` and `"strategy": "agentic"`, perform LLM-assisted +translation from the activity's **full ADF/ARM JSON** (under `raw_activity_json`; also embedded in +the placeholder notebook's fenced `json` block). Nested gaps (an `Until` inside `IfCondition` / +`Switch` / `ForEach`) are reported individually. + +**Until activities:** Lakeflow Jobs have no native repeat-until, so translate the `Until` into one +Python notebook task implementing a bounded polling loop — convert `typeProperties.expression` into +the `while not ():` guard, wrap in a `time.monotonic()` deadline from +`typeProperties.timeout`, and translate `typeProperties.activities` (the loop body) inline. Read +loop variables from `dbutils.widgets`, surface final state as a task value. + +**ExecuteDataFlow:** translate from the raw `typeProperties` + the `dataflow/` JSON definition + +linked-service source/sink connections, into an SDP pipeline or PySpark notebook. + +**Control flow (Switch, Until, Wait, Filter, AppendVariable):** translate from the raw activity +JSON, the containing pipeline JSON, nested activities, and variable definitions. + +**Stored procedures / external calls (SqlServerStoredProcedure, AzureFunction, WebHook, Custom):** +translate from the raw JSON + the linked-service configuration + connection/auth details. + +**Complex expressions:** translate unresolved ADF expressions (e.g. `@pipeline().parameters.x`) +into the target form (Python f-string, Spark SQL, or task-parameter reference). + +## Step 5 — Collect agentic results + +Write one JSON file per resolved gap into `/agentic_results/`: + +```json +{ + "activity_name": "", + "pipeline": "", + "task": {"type": "NotebookActivity", "name": "", "task_key": "", + "notebook_path": "/Workspace/.../your_translated_notebook"} +} +``` + +`activity_name` (required) is matched by name, recursing into containers. `task_key`/`depends_on` +are inherited from the placeholder when omitted, preserving dependency edges. + +## Step 6 — Merge agentic results + +```bash +"$PY" -m flowx.adapter convert --merge-agentic \ + --report /.work/translation_report.json \ + --agentic-results +``` + +Placeholders are replaced in place, status → `translated`. Exits non-zero if any result can't be +matched. Add `--output ` to write a copy instead of overwriting. + +## Step 6.1 — Just-in-time translation configuration + +Run `inspect` **once** for the full option schema, then drive the whole question chain locally +(don't re-run `inspect` per follow-up): + +```bash +"$PY" -m flowx.adapter inspect /.work/translation_report.json +``` + +Each option carries a `show_when` (a conjunction of `{option_id, in:[values]}` clauses; empty = +always ask). Ask an option only when its `show_when` is satisfied by answers already collected; +present its `prompt`/`rationale`/`choices`; honor the `default`; validate each answer (a `free_text` +option accepts any value). Perform data actions inline when an answer calls for it. Apply everything +in one `modify` call. + +**Activity→Notify motifs.** When any activity is followed by notification Web activities, `inspect` +raises `notify_destination`: `keep` (default) translates the Web activities directly; any other +value (`email`/`slack`/`teams`/`pagerduty`/`webhook`) collapses the pattern into Databricks job-task +`on_success`/`on_failure` notifications. Each destination chains follow-up field options (gated by +`show_when`): + +| Destination | Chained field options (SDK arg) | +|-------------|---------------------------------| +| `email` | `notify_email_recipients` (`addresses`, comma-separated) | +| `slack` | `notify_slack_url` (`url`), `notify_slack_channel_id` (optional), `notify_slack_oauth_token` (optional) | +| `teams` | `notify_teams_url` (`url`) | +| `pagerduty` | `notify_pagerduty_integration_key` (`integration_key`) | +| `webhook` | `notify_webhook_url` (`url`), `notify_webhook_username` (optional), `notify_webhook_password` (optional) | + +All destinations also take optional `notify_destination_name` and `notify_events` +(both/on_failure/on_success). For non-email destinations, `modify` creates (or reuses by name) the +Databricks notification destination via the SDK when you submit answers, baking the id into the +report; requires workspace auth at `modify` time. Email uses raw `email_notifications`, no SDK call. + +**Metadata-driven consolidation.** When the flow ends with `metadata_driven_lookup_tool=have` and +the agent has a database tool (Genie, MCP SQL, workspace SDK), run the lookup query to get the rows +as CSV; when `none`, ask the user for a CSV file/string. Pass it inline to `modify` via +`--lookup-csv`: + +```bash +"$PY" -m flowx.adapter modify \ + /.work/translation_report.json \ + --output-dir \ + --answer metadata_driven_consolidate=consolidate \ + --answer metadata_driven_access=yes \ + --lookup-csv "" +``` + +When no metadata-driven motif is consolidated, `--lookup-csv` is omitted and the motif becomes a +Databricks for-each task running one Spark JDBC read per source table. Consolidating instead emits +one managed Lakeflow Connect ingestion pipeline. + +`modify` writes `.work/translation_report.stamped.json` (consumed by package) and +`metadata/configuration.json` (the kept configuration record). When `inspect` raises no options, +skip `modify` — package falls back to the un-stamped report. + +## Step 7 — Present translation summary + +``` +Translation Summary +=================== +Total activities: 47 +Deterministic translated: 35 (74.5%) +Agentic translated: 8 (17.0%) +Failed: 4 ( 8.5%) +Overall coverage: 91.5% +``` + +If coverage is below 100%, explain options for failed translations: manual notebook creation, retry +agentic translation with more context, or skip with a placeholder task. See +`references/activity-mapping.md` for the full ADF activity → strategy mapping. diff --git a/skills/flowx-convert/sources/airflow.md b/skills/flowx-convert/sources/airflow.md new file mode 100644 index 0000000..4dfa5e0 --- /dev/null +++ b/skills/flowx-convert/sources/airflow.md @@ -0,0 +1,43 @@ +# Convert — Apache Airflow + +Source guide for `--source airflow`. Translate parsed Airflow DAGs into Databricks IR. See the +parent `SKILL.md` for how to run the phase and the report contract. + +Airflow translation is **deterministic** today: the same static parse the discover phase uses +produces the Pipeline IR directly. There is no separate agentic-gap round for Airflow yet — +operators without a mapping are emitted as placeholder tasks the user fills in manually (or via a +future agentic pass), not as pending gaps in the report. + +## Step 1 — Run the translation + +```bash +"$PY" -m flowx.adapter convert --source airflow \ + --airflow-source-path \ + --output-dir \ + [--pipeline ] +``` + +Use the same source path and `--pipeline` scoping the discover phase used. This writes +`/.work/translation_report.json` in the shared report shape (one pipeline dict, or a +`{"pipelines": [...]}` wrapper for a folder of DAGs). + +## Step 2 — Review the report + +Read `/.work/translation_report.json`. Each task is a `NotebookActivity` (from a +PythonOperator callable or BashOperator command, carrying `generated_source`) or a +`PlaceholderActivity` (an unmapped operator). Dependencies come from `>>` / `<<`; the DAG's cron +`schedule_interval` is carried as the pipeline `schedule`. + +## Step 3 — Handle placeholders (optional) + +For any `PlaceholderActivity`, decide whether to hand-write the notebook body now or leave the +placeholder for the package phase to emit (it ships a notebook with a clear TODO). The shared +`inspect`/`modify`/`merge_agentic` commands from the parent `SKILL.md` are available if you want to +apply agent-translated results, but the ADF just-in-time option chain (notify motifs, +metadata-driven consolidation) does not apply to Airflow. + +## Step 4 — Proceed to package + +Run `flowx-package` with the same ``. Package is source-independent — it consumes the +translation report and emits the DABs bundle (databricks.yml, resources/, src/ notebooks, SETUP.md) +identically for every source. diff --git a/skills/flowx-discover/SKILL.md b/skills/flowx-discover/SKILL.md index 6ed4187..b27e6f0 100644 --- a/skills/flowx-discover/SKILL.md +++ b/skills/flowx-discover/SKILL.md @@ -1,267 +1,64 @@ --- name: flowx-discover description: > - Load and parse Azure Data Factory pipeline definitions from Unity Catalog volumes or local directories. - Produces a typed inventory that classifies every activity as deterministic, agentic, or unsupported. + Parse a source orchestrator's pipeline definitions (Azure Data Factory, Apache Airflow) into a + typed inventory that classifies every task as deterministic, agentic, or unsupported. Phase 1 of + the flowx migration workflow; routes to a source-specific guide. triggers: + - "discover pipelines" - "discover ADF" - - "load ADF" - - "parse ADF" - - "import pipelines" + - "discover airflow" - "load pipelines" - "parse pipelines" - - "inventory ADF" + - "import pipelines" + - "inventory source" --- -# Discover ADF Pipeline Definitions +# Discover Source Pipeline Definitions -Parse Azure Data Factory pipeline, dataset, linked service, and trigger JSON files into a typed AST and produce a classified inventory. +Parse a source orchestrator's definitions into a typed inventory. This is phase 1 of the flowx +migration workflow; it produces `metadata/inventory.json` (consumed by `flowx-convert`) plus a +per-pipeline complexity report at `metadata/profile_report.csv`. -## Context +flowx supports more than one **source**, and discovery is source-specific — ADF ships ARM JSON, +Airflow ships Python DAG modules, and the two share no parser. This skill routes to the right +source guide; the shared mechanics (output layout, inventory shape, how to run a phase) live here. -This is phase 1 of the flowx migration workflow. It takes raw ADF JSON exports and produces an `inventory.json` file that the `convert` skill consumes. The inventory classifies every ADF activity into one of three strategies: +## Step 1 — Identify the source (required) -- **Deterministic** — a built-in translator exists (Copy, DatabricksNotebook, ForEach, IfCondition, etc.) -- **Agentic** — requires agentic (LLM-assisted) translation by the agent (ExecuteDataFlow, Switch, Until, StoredProc, etc.) -- **Unsupported** — no known translation path; requires manual intervention +Ask the user which orchestrator they are migrating **from**, or infer it from the input: -## How to run this skill — MCP tool or venv CLI +- **Azure Data Factory / Fabric Data Factory** → source `adf` → read `sources/adf.md` +- **Apache Airflow** → source `airflow` → read `sources/airflow.md` -This phase runs one of two ways; run the **`setup`** skill first if you haven't. +There is no default source. Every phase invocation passes `--source ` explicitly. -- **MCP tool (Databricks Genie Code, or a local stdio registration) — the only path in Genie Code:** - call the single **`flowx`** tool with `command="discover"` and run **no** `python3`/`$PY`/`bash` - commands. The `"$PY" -m …` snippets in the steps below are the **local-CLI fallback only** — ignore - them on this path. +## Step 2 — Follow the source guide - The hosted server **cannot read your workspace/volume files**, so pass the ADF JSON **inline** as - `adf_definitions` — a mapping of relative path → JSON content mirroring the ADF Git-export layout. - You (the agent) read the ARM JSON files from the source and supply them: +Read the matching `sources/.md` in this skill directory and follow it. Each guide covers +the source's input layout, the exact discover command, and how to read its inventory. - ``` - flowx(command="discover", parameters={ - "adf_definitions": { - "pipeline/Foo.json": { ...ARM JSON... }, - "dataset/Bar.json": { ... }, - "linkedService/Baz.json": { ... }, - "trigger/Qux.json": { ... } - }, - "output_dir": "", "pipeline": ""}) - ``` +## How to run a phase — MCP tool or venv CLI - For **large factories** (hundreds–thousands of pipelines), don't inline — reference the source - instead (inline `adf_definitions` is capped at ~5 MB): pass `"adf_volume_path": - "/Volumes/cat/sch/adf_export"` for a UC Volume (read via the SDK Files API) or - `"adf_workspace_path": "/Workspace/Shared/adf_export"` for an ADF Git folder in the workspace (read - via the SDK Workspace API). Locally, where the server can read - the path, you may instead pass `adf_source_path`. The tool returns the inventory summary - (pipeline/activity counts by strategy and coverage); use it in place of reading the files directly. +Both paths are the same across sources; only `--source` and the source path differ. Run the +**`setup`** skill first if you haven't. -- **venv CLI (local, no MCP server):** ensure the venv exists (`setup` Path B / `bootstrap.sh`), then - run the commands below with the venv interpreter and `src/` on `PYTHONPATH`. The interpreter path is - in the marker file `/.migration-venv` (use `$PY` anywhere a command shows `python3`): +- **MCP tool (Databricks Genie Code, or a local stdio registration):** call the single **`flowx`** + tool with `command="discover"` and `parameters` including `"source": ""`. Run **no** + `python3`/`$PY` commands on this path. + +- **venv CLI (local, no MCP server):** ensure the venv exists (`setup` / `bootstrap.sh`), then: ```bash export PYTHONPATH="/src" PY="$(cat /.migration-venv)" - "$PY" -m flowx.adapter discover --adf-source-path --output-dir + "$PY" -m flowx.adapter discover --source --source-path --output-dir [--pipeline ] ``` - If Python or pip is missing, `bootstrap.sh` prints a warning telling the user what to install — - relay it and stop until they have Python 3.12+ and pip. - -## Workflow - -Follow these steps in order: - -### Step 1 — Determine the ADF source path - -Ask the user for the location of their ADF JSON exports. Accept either: -- A Unity Catalog volume path (e.g., `/Volumes/main/default/adf_export`) -- A local directory path (e.g., `./adf_export/` or `/tmp/adf_json/`) - -The directory should contain subdirectories or files for: -- `pipeline/` or `pipelines/` — pipeline definition JSON files -- `dataset/` or `datasets/` — dataset definition JSON files (optional) -- `linkedService/` or `linked_services/` — linked service JSON files (optional) -- `trigger/` or `triggers/` — trigger definition JSON files (optional) - -### Step 2 — Download from UC volumes if needed - -If the source path starts with `/Volumes/`, the files live in a Unity Catalog volume and must be downloaded to a local temp directory first. - -Use the `databricks-execution-compute` skill to run the following on the Databricks workspace: - -```python -import os, json, shutil, tempfile - -volume_path = "" -local_dir = tempfile.mkdtemp(prefix="adf_ingest_") - -# Copy from volume to local -for root, dirs, files in os.walk(volume_path): - for f in files: - if f.endswith(".json"): - src = os.path.join(root, f) - rel = os.path.relpath(src, volume_path) - dst = os.path.join(local_dir, rel) - os.makedirs(os.path.dirname(dst), exist_ok=True) - shutil.copy2(src, dst) - -print(f"Downloaded ADF files to: {local_dir}") -``` - -Alternatively, use the Databricks CLI: -```bash -databricks fs cp -r "dbfs:" "" --overwrite -``` - -Set the working source directory to the local temp path for subsequent steps. - -### Step 3 — Run the deterministic parser - -Run the discover phase via the adapter's unified phase runner (recommended): - -```bash -"$PY" -m flowx.adapter discover \ - --adf-source-path \ - --output-dir \ - [--pipeline ] -``` - -`--adf-source-path` is accepted as an alias of `--source-dir` (it matches the -`adf_source_path` input option). This forwards to, and is equivalent to, running -the loader directly: - -```bash -"$PY" -m flowx.parser.adf_loader \ - --source-dir --output-dir [--pipeline ] -``` - -Where: -- `` is the root of the flowx plugin (the directory containing `src/`) -- `` is the local directory containing ADF JSON files -- `` is the **single shared migration output directory** used by all three phases - (default: `./flowx_output`). Discover writes its artifacts into the `metadata/` subfolder. -- `` (optional) — when provided, filters to only the named pipeline. When omitted, all pipelines in the source directory are included. - -**Always pass `--pipeline` when the user has specified a specific pipeline to migrate.** This ensures the inventory and all downstream phases are scoped to only that pipeline. - -This produces, under `/metadata/`: -- `inventory.json` — the classified activity inventory -- `profile_report.csv` — one row per pipeline with a complexity assessment (see Step 4b) -- `.arm.json` — the verbatim original ADF/ARM source for each pipeline (provenance) + `--source-path` is the generic flag (each source also accepts its own alias, e.g. + `--adf-source-path`); both normalise to the phase's `--source-dir`. `--source` is required. -### Step 4 — Read and validate the inventory - -Read the generated `/metadata/inventory.json` file. It has this structure: - -```json -{ - "source_dir": "/path/to/adf/json", - "generated_at": "2026-04-07T12:00:00Z", - "pipelines": [ - { - "name": "PipelineName", - "file": "pipeline/PipelineName.json", - "activities": [ - { - "name": "CopyFromBlob", - "type": "Copy", - "strategy": "deterministic", - "translator": "copy.py" - }, - { - "name": "RunDataFlow", - "type": "ExecuteDataFlow", - "strategy": "agentic" - } - ] - } - ], - "summary": { - "pipeline_count": 12, - "activity_count": 47, - "deterministic_count": 35, - "agentic_count": 10, - "unsupported_count": 2, - "coverage_pct": 95.7 - } -} -``` - -### Step 4b — Review the complexity report - -`/metadata/profile_report.csv` carries one row per pipeline with a migration-complexity -assessment. Columns: - -| Column | Meaning | -|---|---| -| `pipeline` | Pipeline name | -| `activities` | Total activities (including nested ForEach/If/Switch children) | -| `datasets` | Distinct datasets the pipeline references | -| `linked_services` | Distinct linked services (activity-level + via referenced datasets) | -| `collapsible_patterns` | Number of motif patterns detected (auto-collapsible during convert) | -| `databricks_native_activities` | Notebook / SparkJar / SparkPython / Job activities (simplest) | -| `control_flow_activities` | ForEach / If / Switch / SetVariable / AppendVariable / Filter / Wait / Until | -| `other_activities` | Everything else — Copy, Web, Lookup, agentic types (hardest) | -| `complexity_score` | Weighted score: native×1 + control×2 + other×3 + datasets + linked_services + collapsible_patterns | -| `complexity_size` | T-shirt size from the score: **S** ≤5, **M** ≤15, **L** ≤30, **XL** >30 | - -Use it to set expectations: S/M pipelines are largely deterministic; L/XL pipelines (many "other" -activities, datasets, or linked services) warrant closer review and more agentic translation. - -### Step 5 — Present the summary - -Display a summary table to the user: - -``` -ADF Ingestion Summary -===================== -Pipelines parsed: 12 -Total activities: 47 - -Strategy Breakdown: - Deterministic: 35 (74.5%) - Agentic: 10 (21.3%) - Unsupported: 2 ( 4.3%) - -Coverage: 95.7% -``` - -### Step 6 — Detail agentic activities - -For activities classified as `agentic`, explain that each is translated by the agent using LLM-assisted reasoning from the activity's ARM JSON (no built-in deterministic translator exists for these types): - -| Activity | Type | Handling | -|---|---|---| -| RunDataFlow | ExecuteDataFlow | Agentic (LLM-assisted) | -| BranchLogic | Switch | Agentic (LLM-assisted) | -| ... | ... | ... | - -### Step 7 — Warn about unsupported activities - -For activities classified as `unsupported`, warn the user clearly: - -``` -WARNING: The following activities have no automated translation path: - - Pipeline "ETL_Main" / Activity "RunSSIS" (ExecuteSSISPackage) - Recommendation: Manual conversion to PySpark notebook required. -``` - -### Step 8 — Confirm output location - -Tell the user where the metadata files were written (`/metadata/`: inventory.json, profile_report.csv, and the per-pipeline `.arm.json`), summarise the complexity sizes, and confirm they can proceed to the `convert` phase using the same ``. - -## Examples - -- "Discover my ADF pipelines from /Volumes/main/default/adf_export" -- "Parse ADF definitions from ./tests/resources/json/" -- "Load the ADF pipeline JSON files and show me the inventory" -- "Import pipelines from /tmp/customer_adf_export" -- "Discover only the pl_demo_01 pipeline from /Volumes/main/default/adf_export" - -## Output Artifacts +## Output artifacts (shared across sources) All under the shared `/metadata/` folder: @@ -269,4 +66,15 @@ All under the shared `/metadata/` folder: |---|---| | `metadata/inventory.json` | Classified activity inventory for the convert phase | | `metadata/profile_report.csv` | Per-pipeline complexity report (counts + T-shirt size) | -| `metadata/.arm.json` | Verbatim original ADF/ARM source for each pipeline | +| `metadata/.arm.json` | (ADF) Verbatim original source for each pipeline (provenance) | + +The inventory classifies every task into one of three strategies: + +- **Deterministic** — a built-in translator exists; converted without an LLM. +- **Agentic** — requires LLM-assisted translation from the source definition. +- **Unsupported** — no known translation path; needs manual intervention. + +## Reference + +- `sources/adf.md` — Azure Data Factory discovery (ARM JSON, UC-volume download, complexity report) +- `sources/airflow.md` — Apache Airflow discovery (DAG `.py` parsing, operator classification) diff --git a/skills/flowx-discover/sources/adf.md b/skills/flowx-discover/sources/adf.md new file mode 100644 index 0000000..9997e27 --- /dev/null +++ b/skills/flowx-discover/sources/adf.md @@ -0,0 +1,100 @@ +# Discover — Azure Data Factory + +Source guide for `--source adf`. Parse Azure Data Factory pipeline, dataset, linked service, and +trigger JSON files into a typed AST and produce a classified inventory. See the parent `SKILL.md` +for the shared output layout, inventory shape, and how to run a phase. + +## Step 1 — Determine the ADF source path + +Ask the user for the location of their ADF JSON exports. Accept either: +- A Unity Catalog volume path (e.g. `/Volumes/main/default/adf_export`) +- A local directory path (e.g. `./adf_export/`) + +The directory should contain subdirectories or files for: +- `pipeline/` or `pipelines/` — pipeline definition JSON files +- `dataset/` or `datasets/` — dataset definition JSON files (optional) +- `linkedService/` or `linked_services/` — linked service JSON files (optional) +- `trigger/` or `triggers/` — trigger definition JSON files (optional) + +On the MCP path the hosted server cannot read your workspace/volume files, so pass the ADF JSON +inline as `adf_definitions` (a mapping of relative path → JSON content), or for large factories +reference the source via `adf_volume_path` / `adf_workspace_path`. + +## Step 2 — Download from UC volumes if needed + +If the source path starts with `/Volumes/`, copy the files to a local temp directory first (e.g. via +the `databricks-execution-compute` skill or `databricks fs cp -r`), then point discover at the local +path. + +## Step 3 — Run the deterministic parser + +```bash +"$PY" -m flowx.adapter discover --source adf \ + --adf-source-path \ + --output-dir \ + [--pipeline ] +``` + +`--adf-source-path` is the ADF alias of `--source-path`; both normalise to `--source-dir`. Always +pass `--pipeline` when the user specified a single pipeline to migrate, so all downstream phases are +scoped to it. + +## Step 4 — Read and validate the inventory + +Read `/metadata/inventory.json`: + +```json +{ + "source": "adf", + "source_dir": "/path/to/adf/json", + "pipelines": [ + { + "name": "PipelineName", + "activities": [ + {"name": "CopyFromBlob", "type": "Copy", "strategy": "deterministic", "translator": "copy.py"}, + {"name": "RunDataFlow", "type": "ExecuteDataFlow", "strategy": "agentic"} + ] + } + ], + "summary": {"pipeline_count": 12, "activity_count": 47, "deterministic_count": 35, + "agentic_count": 10, "unsupported_count": 2, "coverage_pct": 95.7} +} +``` + +## Step 4b — Review the complexity report + +`/metadata/profile_report.csv` has one row per pipeline: `pipeline`, `activities`, +`datasets`, `linked_services`, `collapsible_patterns`, `databricks_native_activities`, +`control_flow_activities`, `other_activities`, `complexity_score`, `complexity_size` (S ≤5, M ≤15, +L ≤30, XL >30). Use it to set expectations: S/M are largely deterministic; L/XL warrant closer +review and more agentic translation. + +## Step 5 — Present the summary + +``` +ADF Profile Summary +=================== +Pipelines parsed: 12 +Total activities: 47 +Strategy Breakdown: + Deterministic: 35 (74.5%) + Agentic: 10 (21.3%) + Unsupported: 2 ( 4.3%) +Coverage: 95.7% +``` + +## Step 6 — Detail agentic activities + +For `agentic` activities, explain that each is translated by the agent using LLM-assisted reasoning +from the activity's ARM JSON (no built-in deterministic translator exists), e.g. `ExecuteDataFlow`, +`Switch`, `Until`, stored procedures. + +## Step 7 — Warn about unsupported activities + +For `unsupported` activities, warn clearly, e.g. `ExecuteSSISPackage` — recommend manual conversion +to a PySpark notebook. + +## Step 8 — Confirm output location + +Tell the user where the metadata files were written (`/metadata/`), summarise the +complexity sizes, and confirm they can proceed to `flowx-convert` with the same ``. diff --git a/skills/flowx-discover/sources/airflow.md b/skills/flowx-discover/sources/airflow.md new file mode 100644 index 0000000..d347ab8 --- /dev/null +++ b/skills/flowx-discover/sources/airflow.md @@ -0,0 +1,64 @@ +# Discover — Apache Airflow + +Source guide for `--source airflow`. Parse Airflow DAG `.py` modules into a classified inventory. +See the parent `SKILL.md` for the shared output layout, inventory shape, and how to run a phase. + +## How it works + +flowx reads DAG modules **statically** with Python's `ast` — no Airflow install, and the DAGs are +never executed. It extracts operators, `>>` / `<<` task dependencies, the DAG's +`schedule_interval`, and inline PythonOperator callables / BashOperator commands. Each task is +classified: + +- **Deterministic** — a mapped operator (PythonOperator, BashOperator) that becomes a generated + notebook task. +- **Agentic** — an operator with no deterministic mapping yet; emitted as a placeholder for + LLM-assisted translation. + +## Step 1 — Determine the Airflow source path + +Ask the user for either a single DAG `.py` file or a directory of DAGs (scanned recursively; files +with no `DAG(` / `@dag` construct are skipped). Local paths only — the parser reads source text. + +## Step 2 — Run the parser + +```bash +"$PY" -m flowx.adapter discover --source airflow \ + --airflow-source-path \ + --output-dir \ + [--pipeline ] +``` + +`--airflow-source-path` is the Airflow alias of `--source-path`; both normalise to `--source-dir`. +Pass `--pipeline ` to scope to a single DAG. + +## Step 3 — Read and validate the inventory + +Read `/metadata/inventory.json` (`"source": "airflow"`). Each pipeline entry lists its +tasks with a `strategy`. `metadata/profile_report.csv` carries one row per DAG (`pipeline`, +`activities`, `complexity_size`). + +## Step 4 — Present the summary + +``` +Airflow Discover Summary +======================== +DAGs parsed: 3 +Total tasks: 8 + Deterministic: 7 + Agentic: 1 +Coverage: 87.5% +``` + +## Step 5 — Detail agentic tasks + +For `agentic` tasks, name the operator that has no deterministic mapping yet (e.g. a custom or +provider operator) and note it will be emitted as a placeholder notebook for the convert phase to +fill via LLM-assisted translation. + +## Coverage notes + +Current deterministic coverage: `PythonOperator` (callable body → generated PySpark notebook) and +`BashOperator` (command → `%sh` notebook). Dependencies (`>>` / `<<`) and cron +`schedule_interval` → Quartz are handled. Other operators become placeholders. Confirm the output +location and proceed to `flowx-convert` with the same `` and `--source airflow`. diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index 9f895bb..f707de5 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -1,32 +1,43 @@ --- name: flowx-migrate description: > - End-to-end migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs. - Orchestrates discover, convert, and package phases in sequence. + End-to-end migration of a source orchestrator's pipelines (Azure Data Factory, Apache Airflow) + to Databricks Lakeflow Jobs. Orchestrates discover, convert, and package phases in sequence. triggers: - - "migrate ADF" - "migrate pipelines" + - "migrate ADF" + - "migrate airflow" - "ADF to Databricks" + - "airflow to Databricks" - "migrate to Lakeflow" - - "ADF migration" - - "convert ADF to Lakeflow" - "migrate data factory" --- -# End-to-End ADF to Databricks Migration +# End-to-End Source to Databricks Migration -Orchestrate the complete migration of Azure Data Factory pipelines to Databricks Lakeflow Jobs via Declarative Automation Bundles. This skill runs all three phases in sequence: discover, convert, package. +Orchestrate the complete migration of a source orchestrator's pipelines to Databricks Lakeflow Jobs +via Declarative Automation Bundles. This skill runs all three phases in sequence: discover, convert, +package. ## Context This is the top-level orchestration skill. It runs the full migration pipeline: -1. **Discover** — Parse ADF JSON exports into a typed inventory -2. **Convert** — Convert ADF activities to Databricks IR (deterministic + agentic) +1. **Discover** — Parse the source's definitions into a typed inventory +2. **Convert** — Convert the source's tasks to Databricks IR (deterministic + agentic) 3. **Package** — Generate Databricks Declarative Automation Bundles for deployment Each phase builds on the output of the previous phase. The user is shown a summary and asked to confirm before proceeding to the next phase. +## Step 0 — Identify the source (required) + +Ask which orchestrator the user is migrating **from**, or infer it from the input: **Azure Data +Factory / Fabric DF** (`--source adf`) or **Apache Airflow** (`--source airflow`). There is no +default. Pass `--source ` to the discover and convert phases (package is source-independent). +The discover/convert skills route to the matching `sources/.md` guide for source-specific +detail; the invocations below show ADF but apply to any source by swapping `--source` and the +source path (`--adf-source-path` / `--airflow-source-path`, both aliases of `--source-path`). + ## How to run this skill — MCP tools or venv CLI This skill orchestrates all three phases. Run the **`setup`** skill first if you haven't. There are @@ -164,7 +175,9 @@ Example prompt: ### Step 2 — Phase 1: Discover -Invoke the `flowx:flowx-discover` skill with the ADF source path and `--output-dir ` (the shared migration dir). Profile writes `/metadata/{inventory.json, profile_report.csv, .arm.json}`. +Invoke the `flowx:flowx-discover` skill with `--source `, the source path, and +`--output-dir ` (the shared migration dir). Discover routes to its `sources/.md` +guide and writes `/metadata/{inventory.json, profile_report.csv}` (plus `.arm.json` for ADF). Wait for discover to complete and present the inventory summary: @@ -197,7 +210,8 @@ If the user says yes, proceed to step 4. ### Step 4 — Phase 2: Convert Invoke the `flowx:flowx-convert` skill with: -- ADF source dir: the original ADF source path (same `--source-dir` as discover) +- `--source `: the same source discover used +- Source path: the original source path (same one discover used) - Output dir: the same shared `` (convert writes its report to `/.work/`) Wait for the translation to complete and present the summary: diff --git a/skills/flowx-migrate/references/workflow.md b/skills/flowx-migrate/references/workflow.md index db8ee26..faf083b 100644 --- a/skills/flowx-migrate/references/workflow.md +++ b/skills/flowx-migrate/references/workflow.md @@ -85,7 +85,7 @@ ADF JSON Exports **Key decisions:** - Deterministic translators run first because they are fast and reliable. Agentic skills are only invoked for gaps. - The IR is an intermediate format that decouples translation from DABs generation. This allows the package phase to target different output formats in the future. -- Each deterministic translator is a standalone Python module in `src/flowx/translator/activity_translators/`. Adding support for a new activity type means adding a new module. +- Each source's deterministic translators are standalone Python modules under `src/flowx/sources//` (e.g. ADF's live in `src/flowx/sources/adf/translators/`). Adding support for a new activity type means adding a new module there. - Agentic results are saved separately before merging, so they can be inspected, retried, or manually overridden. ## Phase 3: Package diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py index 1d214aa..c45e1b0 100644 --- a/src/flowx/adapter/__main__.py +++ b/src/flowx/adapter/__main__.py @@ -44,16 +44,13 @@ # the cheap commands (inputs, phase pass-throughs, materialize-lookup, workspace-paths) skip ~0.15s of # unused import cost on every adapter subprocess. -# Maps the unified phase runner subcommands to the module CLI they forward to. -_PHASE_MODULES: dict[str, str] = { - "discover": "flowx.parser.adf_loader", - "convert": "flowx.translator.engine", - "package": "flowx.bundler.dab_writer", -} -# Aliases so the inputs option ids double as CLI flags on the phase runners. -_PHASE_FLAG_ALIASES: dict[str, str] = { - "--adf-source-path": "--source-dir", -} +# The package phase is source-independent: it consumes the shared Pipeline IR every +# source produces, so it routes to one module regardless of --source. +_PACKAGE_MODULE = "flowx.bundler.dab_writer" + +# Generic source-path flag the phase runners accept; each source also accepts its own +# alias (e.g. --adf-source-path). Both normalise to the phase CLI's --source-dir. +_SOURCE_PATH_FLAG = "--source-path" def main(argv: list[str] | None = None) -> int: @@ -67,7 +64,7 @@ def main(argv: list[str] | None = None) -> int: Exit code (0 on success, non-zero on usage or runtime errors). """ raw_args = list(sys.argv[1:]) if argv is None else list(argv) - if raw_args and raw_args[0] in _PHASE_MODULES: + if raw_args and raw_args[0] in ("discover", "convert", "package"): # Phase runners are pure pass-through to the underlying phase CLI; # bypass argparse so forwarded --flags aren't misparsed at this level. return _run_phase(raw_args[0], raw_args[1:]) @@ -390,43 +387,100 @@ def _build_parser() -> argparse.ArgumentParser: help="Workspace folder for the dashboard (defaults to the current user's home).", ) - # Unified phase runners: `adapter -- ` forwards to the phase CLI (one entry point); - # --adf-source-path is accepted as an alias of the loader/translator --source-dir flag. + # Unified phase runners: `adapter --source -- ` routes discover/convert + # to the named source's phase module (default: adf, for back-compat). package is source-independent. + # --source-path (and each source's own alias, e.g. --adf-source-path) normalise to --source-dir. for _phase in ("discover", "convert", "package"): _runner = subparsers.add_parser( _phase, - help=f"Run the {_phase} phase (forwards flags to the underlying phase CLI).", + help=f"Run the {_phase} phase (routes to the --source's phase module; forwards remaining flags).", ) _runner.add_argument( "forward", nargs=argparse.REMAINDER, - help="Flags forwarded to the phase CLI (e.g. --adf-source-path/--source-dir, --output-dir, --pipeline).", + help=( + "Flags forwarded to the phase CLI (e.g. --source adf|airflow, " + "--source-path/--source-dir, --output-dir, --pipeline)." + ), ) return parser +def _split_source(forward: list[str]) -> tuple[str | None, list[str]]: + """Extracts ``--source `` (or ``--source=``) from *forward*. + + Returns ``(source_name, remaining_tokens)``, where ``source_name`` is + ``None`` when no ``--source`` was supplied. ``--source`` is required for + the discover/convert phases (there is no default source); the caller + reports the error. + """ + source: str | None = None + remaining: list[str] = [] + tokens = list(forward or []) + index = 0 + while index < len(tokens): + token = tokens[index] + if token == "--source": + if index + 1 < len(tokens): + source = tokens[index + 1] + index += 2 + continue + index += 1 + continue + if token.startswith("--source="): + source = token.split("=", 1)[1] + index += 1 + continue + remaining.append(token) + index += 1 + return source, remaining + + def _run_phase(phase: str, forward: list[str]) -> int: - """Forward a phase runner subcommand to the underlying phase module, **in-process**. + """Forward a phase runner subcommand to the source's phase module, **in-process**. - ``python -m flowx.adapter discover --adf-source-path X --output-dir Y`` runs - ``flowx.parser.adf_loader.main(["--source-dir", "X", "--output-dir", "Y"])`` in this same - interpreter -- no second ``python -m`` spawn. The module's ``main(argv)`` reuses the existing, - tested phase CLI surface, so there is a single entry point with no argument-surface duplication. - Collapsing the former double-spawn (adapter process -> module process) shaves an interpreter - start + re-import off every ``discover``/``convert``/``package`` call. + ``python -m flowx.adapter discover --source airflow --source-path X --output-dir Y`` runs + ``flowx.sources.airflow.discover.main(["--source-dir", "X", "--output-dir", "Y"])`` in this same + interpreter -- no second ``python -m`` spawn. ``--source`` is required for discover/convert + (no default: the user must choose a source); ``package`` is source-independent and always routes + to the shared bundler. The generic ``--source-path`` and each source's own alias normalise to + the phase CLI's ``--source-dir``. Args: phase: One of ``"discover"`` / ``"convert"`` / ``"package"``. forward: Tokens after the phase name (flags for the phase CLI). Returns: - The phase's exit code (0 on success). + The phase's exit code (0 on success), or 2 when ``--source`` is missing + or names an unknown source. """ import importlib - module = importlib.import_module(_PHASE_MODULES[phase]) - mapped = [_PHASE_FLAG_ALIASES.get(token, token) for token in (forward or [])] + from flowx.sources import available_sources, get_source + + source_name, remaining = _split_source(forward) + + if phase == "package": + module_path = _PACKAGE_MODULE + aliases: dict[str, str] = {} + else: + if source_name is None: + print( + f"--source is required for the {phase} phase; choose one of: {', '.join(available_sources())}", + file=sys.stderr, + ) + return 2 + try: + source = get_source(source_name) + except KeyError as error: + print(str(error), file=sys.stderr) + return 2 + module_path = source.discover_module if phase == "discover" else source.convert_module + aliases = {_SOURCE_PATH_FLAG: "--source-dir", source.source_path_flag: "--source-dir"} + + module = importlib.import_module(module_path) + mapped = [aliases.get(token, token) for token in remaining] try: return module.main(mapped) or 0 except SystemExit as exit_signal: # e.g. argparse usage error -> parser.error() raises SystemExit @@ -562,9 +616,9 @@ def _run_modify(args: argparse.Namespace) -> int: provisioned_pipelines.append(provisioned) for message in messages: print(message, file=sys.stderr) - from flowx.translator.engine import _pipeline_to_dict # lazy: heavy import (sqlglot) + from flowx.ir_serde import pipeline_to_dict - modified = [_pipeline_to_dict(pipeline) for pipeline in provisioned_pipelines] + modified = [pipeline_to_dict(pipeline) for pipeline in provisioned_pipelines] _write_modified_report(args.report, modified, stamped_out) # Persist the collected answers as the kept configuration record. diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 362d48a..5e5588e 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -29,6 +29,7 @@ Activity, AppendVariableActivity, CopyActivity, + DbtFactoryActivity, DeleteActivity, Dependency, ExecutePipelineActivity, @@ -1588,6 +1589,18 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: notebook_path_unresolved=bool(task_ir.get("notebook_path_unresolved", False)), notebook_path_expression=task_ir.get("notebook_path_expression"), unresolved_libraries=list(task_ir.get("unresolved_libraries") or []), + generated_source=task_ir.get("generated_source"), + ) + if task_type == "DbtFactoryActivity": + return DbtFactoryActivity( + **base, + project_dir=task_ir.get("project_dir", "."), + profiles_dir=task_ir.get("profiles_dir", "dbt_profiles"), + target=task_ir.get("target", "dev"), + manifest_path=task_ir.get("manifest_path"), + render_mode=task_ir.get("render_mode", "static"), + selectors=list(task_ir.get("selectors") or []), + nodes=list(task_ir.get("nodes") or []), ) if task_type == "SparkJarActivity": return SparkJarActivity( diff --git a/src/flowx/dbt/__init__.py b/src/flowx/dbt/__init__.py new file mode 100644 index 0000000..f61e19d --- /dev/null +++ b/src/flowx/dbt/__init__.py @@ -0,0 +1 @@ +"""dbt-factory support: read a dbt manifest and explode it into task specs.""" diff --git a/src/flowx/dbt/manifest.py b/src/flowx/dbt/manifest.py new file mode 100644 index 0000000..35b43a1 --- /dev/null +++ b/src/flowx/dbt/manifest.py @@ -0,0 +1,155 @@ +"""Read a dbt ``manifest.json`` and explode it into per-node task specs. + +This is the deterministic core of dbt-factory mode: it turns the stable +dbt-core manifest artifact into an ordered list of :class:`DbtNode` objects, one +per dbt model / seed / snapshot / test, with the dependency edges between them +pruned to the exploded set. It performs no I/O beyond reading the manifest file +and needs no dbt install, so it is unit-testable against a synthetic manifest. + +Both renderers (static explosion and the PyDABs deploy-time hook) consume the +same :class:`DbtNode` list, so the "one IR node, two renderers" contract holds. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path + +# dbt resource_types that dbt-factory turns into their own orchestrator task. deps/docs and source +# definitions are not runnable nodes; snapshots/seeds/models/tests are. +_RUNNABLE_RESOURCE_TYPES: frozenset[str] = frozenset({"model", "seed", "snapshot", "test"}) + +# The dbt command each runnable resource_type maps to (dbt-factory's model->run, seed->seed, etc.). +_RESOURCE_TYPE_TO_COMMAND: dict[str, str] = { + "model": "run", + "seed": "seed", + "snapshot": "snapshot", + "test": "test", +} + +# FQN components go into a `--select fqn:a.b.c` selector; restrict to characters dbt's own selector +# grammar accepts so a crafted node name can't inject extra selector syntax. +_FQN_COMPONENT = re.compile(r"[A-Za-z0-9_.-]+") + + +@dataclass(slots=True, kw_only=True) +class DbtNode: + """One runnable dbt node exploded from the manifest. + + Attributes: + unique_id: dbt manifest unique_id (e.g. ``model.pkg.stg_orders``). + resource_type: ``model`` / ``seed`` / ``snapshot`` / ``test``. + name: dbt node name. + command: dbt subcommand for this node (``run`` / ``seed`` / ...). + selector: The ``fqn:`` selector that resolves to exactly this node. + task_key: Databricks task key (``_`` sanitized). + depends_on: Task keys of upstream exploded nodes (pruned to the set). + """ + + unique_id: str + resource_type: str + name: str + command: str + selector: str + task_key: str + depends_on: list[str] = field(default_factory=list) + + +def _sanitize_task_key(resource_type: str, name: str) -> str: + """Builds a Databricks task key from a dbt node's type and name.""" + raw = f"{resource_type}_{name}" + key = re.sub(r"[^a-zA-Z0-9_-]", "_", raw) + key = re.sub(r"_+", "_", key).strip("_") + return key or "dbt_node" + + +def _fqn_selector(fqn: list[str]) -> str: + """Builds a ``fqn:`` selector string from a node's fqn components. + + Raises: + ValueError: When a component contains characters outside dbt's + selector grammar, so a crafted node name cannot inject extra + selector syntax into the generated ``--select`` argument. + """ + for component in fqn: + if not _FQN_COMPONENT.fullmatch(component): + raise ValueError(f"Unsafe fqn component {component!r} in {'.'.join(fqn)!r}") + return "fqn:" + ".".join(fqn) + + +def load_dbt_nodes(manifest_path: Path) -> list[DbtNode]: + """Reads a dbt manifest and returns its runnable nodes as task specs. + + Args: + manifest_path: Path to a dbt ``manifest.json``. + + Returns: + Ordered list of :class:`DbtNode`, one per runnable node, with + ``depends_on`` pruned to the exploded set (edges to sources, + macros, or filtered-out nodes are dropped). + + Raises: + ValueError: When the test factory would be enabled but the + manifest carries unit tests (dbt-factory 0.2.1 silently drops + them), or when a node's fqn contains unsafe characters. + """ + manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + return explode_manifest(manifest) + + +def explode_manifest(manifest: dict) -> list[DbtNode]: + """Explodes an in-memory dbt manifest dict into runnable task specs. + + Split out from :func:`load_dbt_nodes` so tests can pass a synthetic + manifest dict without touching the filesystem. + """ + nodes: dict[str, dict] = manifest.get("nodes", {}) + + runnable: dict[str, DbtNode] = {} + for unique_id, node in nodes.items(): + resource_type = node.get("resource_type", "") + if resource_type not in _RUNNABLE_RESOURCE_TYPES: + continue + fqn = node.get("fqn") or [node.get("name", unique_id)] + runnable[unique_id] = DbtNode( + unique_id=unique_id, + resource_type=resource_type, + name=node.get("name", unique_id), + command=_RESOURCE_TYPE_TO_COMMAND[resource_type], + selector=_fqn_selector(fqn), + task_key=_sanitize_task_key(resource_type, node.get("name", unique_id)), + ) + + # Fail closed on unit tests: dbt-factory 0.2.1 does not emit unit-test tasks, so a manifest that + # carries them would silently lose coverage if any test task is exploded. + if manifest.get("unit_tests") and any(n.resource_type == "test" for n in runnable.values()): + raise ValueError( + "Manifest declares unit_tests, which dbt-factory 0.2.1 does not explode into tasks. " + "Refusing to emit an incomplete dbt job (fail-closed)." + ) + + # Prune dependency edges to the exploded set. dbt nodes depend on sources, macros, and each other; + # only edges between two runnable nodes become task dependencies. + task_key_by_uid = {uid: dbt_node.task_key for uid, dbt_node in runnable.items()} + for uid, dbt_node in runnable.items(): + upstream_uids = nodes[uid].get("depends_on", {}).get("nodes") or [] + dbt_node.depends_on = [task_key_by_uid[up] for up in upstream_uids if up in task_key_by_uid] + + _assert_unique_task_keys(list(runnable.values())) + # Deterministic order: manifest iteration order is stable, but sort by task_key so the emitted + # job is byte-identical across runs regardless of dict ordering. + return sorted(runnable.values(), key=lambda n: n.task_key) + + +def _assert_unique_task_keys(nodes: list[DbtNode]) -> None: + """Raises when two distinct dbt nodes sanitize to the same task key.""" + seen: dict[str, str] = {} + for node in nodes: + if node.task_key in seen: + raise ValueError( + f"dbt nodes {seen[node.task_key]!r} and {node.unique_id!r} collide on task key " + f"{node.task_key!r}; refusing to emit a job with a duplicate task." + ) + seen[node.task_key] = node.unique_id diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py new file mode 100644 index 0000000..408b99e --- /dev/null +++ b/src/flowx/ir_serde.py @@ -0,0 +1,482 @@ +"""Source-neutral serialization for the flowx Pipeline IR. + +Every source's convert phase serialises its :class:`~flowx.models.ir.Pipeline` +to the ``translation_report.json`` shape these functions produce, and the +package phase rehydrates from it, so the report format is the one contract both +halves share. This lives at the top level (not inside a source) because it +belongs to the IR, not to ADF: the Airflow convert phase and the bundler import +it just as the ADF engine does. + +Also hosts ``merge_agentic_results`` -- the agentic-gap merge operates purely on +serialised report dicts, so it is source-neutral too. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +from flowx.models.ir import ( + Activity, + AppendVariableActivity, + CopyActivity, + DbtFactoryActivity, + DeleteActivity, + ExecutePipelineActivity, + FilterActivity, + ForEachActivity, + IfConditionActivity, + LookupActivity, + MotifActivity, + NotebookActivity, + Pipeline, + PlaceholderActivity, + RunJobActivity, + SetVariableActivity, + SparkJarActivity, + SparkPythonActivity, + SwitchActivity, + UnsupportedActivity, + WaitActivity, + WebActivity, +) + +logger = logging.getLogger(__name__) + + +def pipeline_to_dict(pipeline: Pipeline) -> dict[str, Any]: + """Serialise a Pipeline IR to a JSON-friendly dictionary. + + Args: + pipeline: The translated pipeline IR. + + Returns: + Dictionary suitable for ``json.dumps``. + """ + result: dict[str, Any] = { + "name": pipeline.name, + "parameters": pipeline.parameters, + "schedule": pipeline.schedule, + "tags": pipeline.tags, + "tasks": [activity_to_dict(task) for task in pipeline.tasks], + } + if pipeline.translation_configuration is not None: + result["translation_configuration"] = configuration_to_dict(pipeline.translation_configuration) + return result + + +def configuration_to_dict(configuration: Any) -> dict[str, Any]: + """Serialise a TranslationConfiguration instance to a JSON-friendly dictionary. + + Args: + configuration: The :class:`TranslationConfiguration` snapshot to serialise. + + Returns: + Dictionary with each StrEnum field rendered as its string value + and per-task overrides preserved verbatim. + """ + return { + "copy_activity_paradigm": str(configuration.copy_activity_paradigm), + "non_databricks_task_compute": str(configuration.non_databricks_task_compute), + "use_lakeflow_connectors": str(configuration.use_lakeflow_connectors), + "lakeflow_connector_type": str(configuration.lakeflow_connector_type), + "motif_consolidations": { + motif_id: str(choice) for motif_id, choice in configuration.motif_consolidations.items() + }, + "per_task": dict(configuration.per_task), + } + + +def activity_to_dict(task: Activity) -> dict[str, Any]: + """Serialise a single Activity IR node to a JSON-friendly dictionary. + + Args: + task: Any Activity IR node. + + Returns: + Dictionary suitable for ``json.dumps``. + """ + task_dict: dict[str, Any] = { + "name": task.name, + "task_key": task.task_key, + "type": type(task).__name__, + } + if task.description: + task_dict["description"] = task.description + if task.timeout_seconds: + task_dict["timeout_seconds"] = task.timeout_seconds + if task.max_retries: + task_dict["max_retries"] = task.max_retries + if task.min_retry_interval_millis: + task_dict["min_retry_interval_millis"] = task.min_retry_interval_millis + if task.depends_on: + task_dict["depends_on"] = [ + {"task_key": dependency.task_key, "outcome": dependency.outcome} for dependency in task.depends_on + ] + if task.cluster: + task_dict["cluster"] = task.cluster + if task.existing_cluster_id: + task_dict["existing_cluster_id"] = task.existing_cluster_id + if task.compute_mode: + task_dict["compute_mode"] = task.compute_mode + if task.notifications: + task_dict["notifications"] = task.notifications + if task.libraries: + task_dict["libraries"] = task.libraries + if task.parameter_approximations: + task_dict["parameter_approximations"] = task.parameter_approximations + + extra = activity_extra_fields(task) + task_dict.update(extra) + return task_dict + + +def activity_extra_fields(activity: Activity) -> dict[str, Any]: + """Extracts type-specific fields from an Activity subclass. + + Args: + activity: Any Activity IR node. + + Returns: + Dictionary of extra fields beyond the base Activity. + """ + extra: dict[str, Any] = {} + + match activity: + case NotebookActivity(): + extra["notebook_path"] = activity.notebook_path + if activity.base_parameters: + extra["base_parameters"] = activity.base_parameters + if activity.notebook_path_unresolved: + extra["notebook_path_unresolved"] = True + if activity.notebook_path_expression is not None: + extra["notebook_path_expression"] = activity.notebook_path_expression + if activity.unresolved_libraries: + extra["unresolved_libraries"] = list(activity.unresolved_libraries) + if activity.generated_source is not None: + extra["generated_source"] = activity.generated_source + case DbtFactoryActivity(): + extra["project_dir"] = activity.project_dir + extra["profiles_dir"] = activity.profiles_dir + extra["target"] = activity.target + if activity.manifest_path is not None: + extra["manifest_path"] = activity.manifest_path + extra["render_mode"] = activity.render_mode + if activity.selectors: + extra["selectors"] = list(activity.selectors) + if activity.nodes: + extra["nodes"] = list(activity.nodes) + case CopyActivity(): + extra["source_type"] = activity.source_type + extra["sink_type"] = activity.sink_type + if activity.source_properties: + extra["source_properties"] = activity.source_properties + if activity.sink_properties: + extra["sink_properties"] = activity.sink_properties + if activity.sink_dataset_type: + extra["sink_dataset_type"] = activity.sink_dataset_type + if activity.sink_format: + extra["sink_format"] = activity.sink_format + if activity.sink_resolved_path: + extra["sink_resolved_path"] = activity.sink_resolved_path + if activity.column_mapping: + extra["column_mapping"] = activity.column_mapping + if activity.target_format: + extra["target_format"] = activity.target_format + if activity.use_lakeflow_connector: + extra["use_lakeflow_connector"] = activity.use_lakeflow_connector + if activity.lakeflow_connector_type: + extra["lakeflow_connector_type"] = activity.lakeflow_connector_type + case ForEachActivity(): + extra["items_expression"] = activity.items_expression + extra["concurrency"] = activity.concurrency + extra["inner_activities"] = [activity_to_dict(inner) for inner in activity.inner_activities] + if activity.inputs_bridge_notebook_code: + extra["inputs_bridge_notebook_code"] = activity.inputs_bridge_notebook_code + if activity.inputs_bridge_notebook_imports: + extra["inputs_bridge_notebook_imports"] = list(activity.inputs_bridge_notebook_imports) + if activity.inputs_bridge_required_parameters: + extra["inputs_bridge_required_parameters"] = dict(activity.inputs_bridge_required_parameters) + case IfConditionActivity(): + extra["op"] = activity.op + extra["left"] = activity.left + extra["right"] = activity.right + extra["if_true_activities"] = [activity_to_dict(inner) for inner in activity.if_true_activities] + extra["if_false_activities"] = [activity_to_dict(inner) for inner in activity.if_false_activities] + if activity.bridge_notebook_code: + extra["bridge_notebook_code"] = activity.bridge_notebook_code + if activity.bridge_notebook_imports: + extra["bridge_notebook_imports"] = list(activity.bridge_notebook_imports) + if activity.bridge_required_parameters: + extra["bridge_required_parameters"] = dict(activity.bridge_required_parameters) + case LookupActivity(): + extra["source_type"] = activity.source_type + if activity.source_properties: + extra["source_properties"] = activity.source_properties + extra["first_row_only"] = activity.first_row_only + if activity.source_query: + extra["source_query"] = activity.source_query + case SetVariableActivity(): + extra["variable_name"] = activity.variable_name + extra["variable_value"] = activity.variable_value + extra["value_kind"] = activity.value_kind + if activity.notebook_code: + extra["notebook_code"] = activity.notebook_code + if activity.notebook_imports: + extra["notebook_imports"] = activity.notebook_imports + if activity.required_parameters: + extra["required_parameters"] = dict(activity.required_parameters) + if activity.raw_expression: + extra["raw_expression"] = activity.raw_expression + case FilterActivity(): + extra["items_expression"] = activity.items_expression + extra["condition_expression"] = activity.condition_expression + if activity.condition_code is not None: + extra["condition_code"] = activity.condition_code + if activity.condition_imports: + extra["condition_imports"] = list(activity.condition_imports) + case AppendVariableActivity(): + extra["variable_name"] = activity.variable_name + extra["append_value"] = activity.append_value + extra["value_kind"] = activity.value_kind + if activity.notebook_code: + extra["notebook_code"] = activity.notebook_code + if activity.notebook_imports: + extra["notebook_imports"] = activity.notebook_imports + if activity.required_parameters: + extra["required_parameters"] = dict(activity.required_parameters) + case SwitchActivity(): + extra["on_expression"] = activity.on_expression + extra["cases"] = [ + {"value": case_item.value, "activities": [activity_to_dict(inner) for inner in case_item.activities]} + for case_item in activity.cases + ] + extra["default_activities"] = [activity_to_dict(inner) for inner in activity.default_activities] + if activity.bridge_notebook_code: + extra["bridge_notebook_code"] = activity.bridge_notebook_code + if activity.bridge_notebook_imports: + extra["bridge_notebook_imports"] = list(activity.bridge_notebook_imports) + if activity.bridge_required_parameters: + extra["bridge_required_parameters"] = dict(activity.bridge_required_parameters) + case WaitActivity(): + extra["wait_time_seconds"] = activity.wait_time_seconds + case SparkJarActivity(): + extra["main_class_name"] = activity.main_class_name + if activity.parameters: + extra["parameters"] = activity.parameters + case SparkPythonActivity(): + extra["python_file"] = activity.python_file + if activity.parameters: + extra["parameters"] = activity.parameters + case WebActivity(): + extra["url"] = activity.url + extra["method"] = activity.method + if activity.body is not None: + extra["body"] = activity.body + if activity.headers: + extra["headers"] = activity.headers + if activity.authentication: + extra["authentication"] = activity.authentication + if activity.body_code is not None: + extra["body_code"] = activity.body_code + if activity.body_imports: + extra["body_imports"] = activity.body_imports + if activity.body_required_parameters: + extra["body_required_parameters"] = activity.body_required_parameters + if activity.disable_cert_validation: + extra["disable_cert_validation"] = activity.disable_cert_validation + if activity.http_request_timeout_seconds: + extra["http_request_timeout_seconds"] = activity.http_request_timeout_seconds + case DeleteActivity(): + extra["dataset_name"] = activity.dataset_name + if activity.folder_path: + extra["folder_path"] = activity.folder_path + extra["recursive"] = activity.recursive + case ExecutePipelineActivity(): + extra["pipeline_name"] = activity.pipeline_name + extra["wait_on_completion"] = activity.wait_on_completion + if activity.parameters: + extra["parameters"] = activity.parameters + case RunJobActivity(): + extra["job_name"] = activity.job_name + if activity.existing_job_id: + extra["existing_job_id"] = activity.existing_job_id + if activity.job_parameters: + extra["job_parameters"] = activity.job_parameters + case MotifActivity(): + extra["motif_id"] = activity.motif_id + extra["display_name"] = activity.display_name + extra["databricks_replacement"] = activity.databricks_replacement + extra["matched_activity_names"] = activity.matched_activity_names + if activity.source_type_hint: + extra["source_type_hint"] = activity.source_type_hint + if activity.confidence_notes: + extra["confidence_notes"] = activity.confidence_notes + if activity.notebook_template: + extra["notebook_template"] = activity.notebook_template + if activity.motif_config: + extra["motif_config"] = activity.motif_config + if activity.consolidate_metadata_driven: + extra["consolidate_metadata_driven"] = activity.consolidate_metadata_driven + if activity.lookup_values: + extra["lookup_values"] = activity.lookup_values + case PlaceholderActivity(): + extra["original_type"] = activity.original_type + extra["comment"] = activity.comment + case UnsupportedActivity(): + extra["original_type"] = activity.original_type + extra["reason"] = activity.reason + + return extra + + +def activity_to_debug_dict(activity: Activity) -> dict[str, Any]: + """Serialise an Activity to a full debug dict showing all dataclass fields. + + Args: + activity: Any Activity IR node. + + Returns: + Dict with ``__class__`` plus every dataclass field. + """ + result: dict[str, Any] = {"__class__": type(activity).__name__} + + for field in activity.__dataclass_fields__: + value = getattr(activity, field) + + if isinstance(value, Activity): + result[field] = activity_to_debug_dict(value) + elif isinstance(value, list) and value and isinstance(value[0], Activity): + result[field] = [activity_to_debug_dict(inner) for inner in value] + elif isinstance(value, list) and value and hasattr(value[0], "__dataclass_fields__"): + result[field] = [dataclass_to_debug_dict(item) for item in value] + else: + result[field] = value + + return result + + +def dataclass_to_debug_dict(obj: Any) -> dict[str, Any]: + """Serialise a generic dataclass (SwitchCase, Dependency, etc.) to a debug dict. + + Args: + obj: A dataclass instance. + + Returns: + Dict with ``__class__`` plus every dataclass field. + """ + result: dict[str, Any] = {"__class__": type(obj).__name__} + + for field in obj.__dataclass_fields__: + value = getattr(obj, field) + + if isinstance(value, Activity): + result[field] = activity_to_debug_dict(value) + elif isinstance(value, list) and value and isinstance(value[0], Activity): + result[field] = [activity_to_debug_dict(inner) for inner in value] + else: + result[field] = value + + return result + + +def pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: + """Serialise a Pipeline IR to a full debug dict. + + Args: + pipeline: The translated pipeline IR. + + Returns: + Dict with every field fully expanded. + """ + return { + "__class__": "Pipeline", + "name": pipeline.name, + "parameters": pipeline.parameters, + "schedule": pipeline.schedule, + "tags": pipeline.tags, + "tasks": [activity_to_debug_dict(task) for task in pipeline.tasks], + } + + +def _find_and_replace_task(tasks: list[dict[str, Any]], activity_name: str, replacement: dict[str, Any]) -> bool: + """Replace the task named *activity_name* with *replacement*, recursing into containers. + + Searches top-level tasks and the nested activity lists of IfCondition / + ForEach / Switch containers. Preserves the placeholder's ``task_key`` and + ``depends_on`` when the replacement omits them so downstream dependency + edges stay intact. Returns True when a match was replaced. + """ + nested_keys = ("inner_activities", "if_true_activities", "if_false_activities", "default_activities") + for index, task in enumerate(tasks): + if task.get("name") == activity_name: + replacement.setdefault("task_key", task.get("task_key")) + replacement.setdefault("name", activity_name) + if "depends_on" not in replacement and task.get("depends_on"): + replacement["depends_on"] = task["depends_on"] + tasks[index] = replacement + return True + for key in nested_keys: + child = task.get(key) + if isinstance(child, list) and _find_and_replace_task(child, activity_name, replacement): + return True + for case in task.get("cases") or []: + if isinstance(case, dict) and isinstance(case.get("activities"), list): + if _find_and_replace_task(case["activities"], activity_name, replacement): + return True + return False + + +def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Path | None = None) -> tuple[int, int]: + """Merge agent-produced per-activity translations into a translation report. + + Each ``*.json`` file in *results_dir* describes one resolved agentic gap:: + + { + "activity_name": "", # required + "pipeline": "", # optional; for multi-pipeline reports + "task": { ...IR task dict... } # required; replacement task + } + + The matching placeholder task (located by ``name``, recursing into + IfCondition / ForEach / Switch containers) is replaced by ``task``. Use a + ``NotebookActivity`` whose ``notebook_path`` points at a notebook the agent + wrote to the workspace; the prepare phase then references it directly. + + Args: + report_path: ``translation_report.json`` produced by the translate phase. + results_dir: Directory of per-activity result JSON files. + output_path: Where to write the merged report; defaults to overwriting + *report_path*. + + Returns: + ``(merged, unmatched)`` counts. + """ + report = json.loads(report_path.read_text(encoding="utf-8")) + pipelines = report["pipelines"] if isinstance(report, dict) and "pipelines" in report else [report] + + merged = 0 + unmatched = 0 + for result_file in sorted(results_dir.glob("*.json")): + data = json.loads(result_file.read_text(encoding="utf-8")) + activity_name = data.get("activity_name") or data.get("activity") + task = data.get("task") or data.get("ir") + if not activity_name or not isinstance(task, dict): + logger.warning("Skipping %s: missing 'activity_name' or 'task'.", result_file.name) + unmatched += 1 + continue + wanted = data.get("pipeline") + candidates = [pipeline for pipeline in pipelines if not wanted or pipeline.get("name") == wanted] + if any(_find_and_replace_task(pipeline.get("tasks", []), activity_name, dict(task)) for pipeline in candidates): + merged += 1 + logger.info("Merged agentic result for '%s' from %s", activity_name, result_file.name) + else: + logger.warning("No placeholder named '%s' found for %s", activity_name, result_file.name) + unmatched += 1 + + destination = output_path or report_path + destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") + logger.info("Wrote merged report to %s (%d merged, %d unmatched)", destination, merged, unmatched) + return merged, unmatched diff --git a/src/flowx/models/ir.py b/src/flowx/models/ir.py index aa1afe3..5b4435c 100644 --- a/src/flowx/models/ir.py +++ b/src/flowx/models/ir.py @@ -117,6 +117,12 @@ class NotebookActivity(Activity): dab_ref. Each entry has ``type`` (library shape key), ``expression`` (raw ADF text), and ``missing`` (referenced identifier names not bound in the translation context). + generated_source: Full notebook source a source front-end has + already produced (e.g. an Airflow PythonOperator callable body + or a BashOperator command lowered to a notebook). When set, the + preparer writes it as the notebook content instead of emitting a + reference/placeholder for an existing workspace notebook. ADF, + which references existing notebooks, leaves this ``None``. """ notebook_path: str @@ -125,6 +131,7 @@ class NotebookActivity(Activity): notebook_path_unresolved: bool = False notebook_path_expression: str | None = None unresolved_libraries: list[dict[str, Any]] = field(default_factory=list) + generated_source: str | None = None @dataclass(slots=True, kw_only=True) @@ -456,6 +463,46 @@ class AppendVariableActivity(Activity): notebook_imports: list[str] = field(default_factory=list) +@dataclass(slots=True, kw_only=True) +class DbtFactoryActivity(Activity): + """A dbt project exploded into one Databricks task per dbt node. + + Produced by a source front-end for a dbt workload (e.g. an Airflow + astronomer-cosmos ``DbtTaskGroup`` or a chain of dbt CLI operators). The + preparer renders it two ways from the same node list: + + - ``static`` (default): explode the manifest into an inner job of one task + per dbt node at package time, wired via a ``run_job_task`` hop. Every + dbt task is visible to flowx's coverage / validate / REPORT.csv. + - ``pydabs`` (opt-in): emit a PyDABs hook module that calls + ``databricks-dbt-factory`` at ``bundle deploy`` time, so the dbt job + tracks the project automatically (at the cost of being invisible to + static coverage until deploy). + + Attributes: + project_dir: Path to the dbt project (relative to the bundle root). + profiles_dir: Path to the dbt profiles directory. + target: dbt target name (``dev`` / ``prod`` / ...). + manifest_path: Path to ``manifest.json`` (read at package time for + static mode; referenced by the hook for pydabs mode). + render_mode: ``"static"`` or ``"pydabs"``. + selectors: dbt ``--select`` selectors the source restricted the run + to, if any (empty means the whole project). + nodes: Pre-exploded node specs (list of dicts with ``task_key``, + ``command``, ``selector``, ``depends_on``) when the front-end + already read the manifest; empty when the preparer should read + ``manifest_path`` itself. + """ + + project_dir: str + profiles_dir: str = "dbt_profiles" + target: str = "dev" + manifest_path: str | None = None + render_mode: str = "static" + selectors: list[str] = field(default_factory=list) + nodes: list[dict[str, Any]] = field(default_factory=list) + + @dataclass(slots=True, kw_only=True) class UnsupportedActivity(Activity): """Sentinel for activities that could not be translated. diff --git a/src/flowx/preparer/activity_preparers/dbt_factory.py b/src/flowx/preparer/activity_preparers/dbt_factory.py new file mode 100644 index 0000000..6945ca1 --- /dev/null +++ b/src/flowx/preparer/activity_preparers/dbt_factory.py @@ -0,0 +1,188 @@ +"""Preparer for DbtFactoryActivity -> a dbt job wired via run_job_task. + +Renders a dbt workload two ways from the same exploded node list: + +- ``static`` (default): one notebook task per dbt node in an inner job, wired + from the parent via a ``run_job_task`` hop. Every dbt task is a real DAB + task, so flowx's coverage / validate / REPORT.csv see them. +- ``pydabs`` (opt-in): a PyDABs hook module the bundle loads at deploy time, + which calls ``databricks-dbt-factory`` to build the dbt job from the live + manifest. The parent still gets the ``run_job_task`` hop. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from flowx.dbt.manifest import explode_manifest +from flowx.models.dab import DabNotebook, SetupTask +from flowx.preparer.workflow_preparer import ( + PreparedActivity, + PreparedWorkflow, + build_common_task_fields, +) +from flowx.utils import normalize_task_key + +if TYPE_CHECKING: + from flowx.models.ir import DbtFactoryActivity + +_RUNNER_RELATIVE_PATH = "notebooks/run_dbt_command.py" + + +def _nodes_from_activity(activity: DbtFactoryActivity) -> list[dict[str, Any]]: + """Returns the exploded dbt node specs for *activity*. + + Uses the front-end-supplied ``nodes`` when present; otherwise reads and + explodes ``manifest_path``. Each spec has ``task_key``, ``command``, + ``selector``, and ``depends_on`` (task keys). + """ + if activity.nodes: + return activity.nodes + if activity.manifest_path: + manifest = json.loads(Path(activity.manifest_path).read_text(encoding="utf-8")) + return [ + { + "task_key": node.task_key, + "command": node.command, + "selector": node.selector, + "depends_on": node.depends_on, + } + for node in explode_manifest(manifest) + ] + return [] + + +def _runner_notebook_source() -> str: + """Returns the owned dbt-runner notebook body. + + The runner reads its command / selector / target / project-dir from + widgets and shells out to the dbt CLI, so one notebook serves every dbt + node task. + """ + return ( + "# Databricks notebook source\n" + "# Owned dbt-command runner for flowx dbt-factory (static) mode.\n" + "# One task per dbt node passes its command + fqn: selector as widgets.\n\n" + "import subprocess\n\n" + "dbutils.widgets.text('dbt_command', 'run')\n" + "dbutils.widgets.text('dbt_select', '')\n" + "dbutils.widgets.text('dbt_target', 'dev')\n" + "dbutils.widgets.text('dbt_project_dir', '.')\n" + "dbutils.widgets.text('dbt_profiles_dir', 'dbt_profiles')\n\n" + "command = dbutils.widgets.get('dbt_command')\n" + "select = dbutils.widgets.get('dbt_select')\n" + "target = dbutils.widgets.get('dbt_target')\n" + "project_dir = dbutils.widgets.get('dbt_project_dir')\n" + "profiles_dir = dbutils.widgets.get('dbt_profiles_dir')\n\n" + "argv = ['dbt', command, '--target', target, '--project-dir', project_dir,\n" + " '--profiles-dir', profiles_dir]\n" + "if select:\n" + " argv += ['--select', select]\n" + " if command == 'test':\n" + " # Pin test selection to the node itself; don't pull in indirectly-selected tests.\n" + " argv += ['--indirect-selection', 'empty']\n\n" + "print('running:', ' '.join(argv))\n" + "result = subprocess.run(argv, check=False)\n" + "if result.returncode != 0:\n" + " raise RuntimeError(f'dbt {command} failed with exit code {result.returncode}')\n" + ) + + +def _node_task(node: dict[str, Any], activity: DbtFactoryActivity) -> dict[str, Any]: + """Builds one inner-job notebook task for a dbt node.""" + task: dict[str, Any] = { + "task_key": node["task_key"], + "notebook_task": { + "notebook_path": f"../src/{_RUNNER_RELATIVE_PATH}", + "base_parameters": { + "dbt_command": node["command"], + "dbt_select": node["selector"], + "dbt_target": activity.target, + "dbt_project_dir": activity.project_dir, + "dbt_profiles_dir": activity.profiles_dir, + }, + }, + } + depends_on = [{"task_key": key} for key in node.get("depends_on") or []] + if depends_on: + task["depends_on"] = depends_on + return task + + +def _prepare_static(activity: DbtFactoryActivity, nodes: list[dict[str, Any]]) -> PreparedActivity: + """Static renderer: inner job of per-node tasks + a run_job_task hop.""" + parent_task = build_common_task_fields(activity) + + inner_job_name = f"{activity.task_key}_dbt" + inner_tasks = [_node_task(node, activity) for node in nodes] + runner_notebook = DabNotebook(relative_path=_RUNNER_RELATIVE_PATH, content=_runner_notebook_source()) + + inner_workflow = PreparedWorkflow( + name=inner_job_name, + tasks=inner_tasks, + notebooks=[runner_notebook], + secrets=[], + setup_tasks=[], + ) + + inner_job_key = normalize_task_key(inner_job_name) + parent_task["run_job_task"] = {"job_id": f"${{resources.jobs.{inner_job_key}.id}}"} + + return PreparedActivity(task=parent_task, inner_workflows=[inner_workflow]) + + +def _pydabs_hook_source(activity: DbtFactoryActivity) -> str: + """Returns the PyDABs hook module body for deploy-time dbt-factory generation.""" + return ( + '"""PyDABs hook: build the dbt job from the live manifest at deploy time."""\n\n' + "from databricks.bundles.core import Bundle, Resources\n" + "from databricks_dbt_factory.DbtFactory import DbtFactory\n" + "from databricks_dbt_factory.Utils import read_dbt_manifest\n\n" + f"MANIFEST_PATH = {activity.manifest_path or 'target/manifest.json'!r}\n" + f"PROJECT_DIR = {activity.project_dir!r}\n" + f"PROFILES_DIR = {activity.profiles_dir!r}\n\n" + "def load_resources(bundle: Bundle) -> Resources:\n" + " resources = Resources()\n" + " factory = DbtFactory()\n" + " tasks = factory.create_tasks(read_dbt_manifest(MANIFEST_PATH))\n" + f" resources.add_job({normalize_task_key(activity.task_key + '_dbt')!r}, {{'tasks': tasks}})\n" + " return resources\n" + ) + + +def _prepare_pydabs(activity: DbtFactoryActivity) -> PreparedActivity: + """PyDABs renderer: emit the hook module + a run_job_task hop. + + The dbt job is defined by the hook at deploy time, so no inner workflow is + emitted here. A SetupTask records that databricks.yml needs a + ``python.resources`` entry pointing at the hook. + """ + parent_task = build_common_task_fields(activity) + inner_job_key = normalize_task_key(f"{activity.task_key}_dbt") + parent_task["run_job_task"] = {"job_id": f"${{resources.jobs.{inner_job_key}.id}}"} + + hook_relative_path = f"resources/{activity.task_key}_dbt_job.py" + hook_notebook = DabNotebook(relative_path=hook_relative_path, content=_pydabs_hook_source(activity)) + setup_task = SetupTask( + type="pydabs_dbt_factory", + config={ + "hook_module": f"resources.{activity.task_key}_dbt_job", + "job_key": inner_job_key, + "manifest_path": activity.manifest_path or "target/manifest.json", + "note": ( + "dbt-factory PyDABs mode: add a `python.resources` entry to databricks.yml pointing at " + f"`resources.{activity.task_key}_dbt_job:load_resources`, and `pip install databricks-dbt-factory`." + ), + }, + ) + return PreparedActivity(task=parent_task, notebooks=[hook_notebook], setup_tasks=[setup_task]) + + +def prepare(activity: DbtFactoryActivity, *, scope: str = "") -> PreparedActivity: + """Converts a DbtFactoryActivity into DAB tasks per its render mode.""" + if activity.render_mode == "pydabs": + return _prepare_pydabs(activity) + nodes = _nodes_from_activity(activity) + return _prepare_static(activity, nodes) diff --git a/src/flowx/preparer/activity_preparers/notebook.py b/src/flowx/preparer/activity_preparers/notebook.py index 429a2c9..a9b6962 100644 --- a/src/flowx/preparer/activity_preparers/notebook.py +++ b/src/flowx/preparer/activity_preparers/notebook.py @@ -206,8 +206,12 @@ def prepare( placeholder_filename = notebook_filename(activity.task_key, activity.name) notebook_relative_path = f"notebooks/{placeholder_filename}" - content = download_notebook(resolved_path) or _notebook_placeholder( - resolved_path, activity.name, placeholder_filename + # A source front-end may have already produced the notebook body (e.g. an Airflow + # PythonOperator callable). Prefer it over a workspace download or a placeholder. + content = ( + activity.generated_source + or download_notebook(resolved_path) + or _notebook_placeholder(resolved_path, activity.name, placeholder_filename) ) task["notebook_task"] = {"notebook_path": f"../src/{notebook_relative_path}"} diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py index 6148cbf..5b0b976 100644 --- a/src/flowx/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -11,6 +11,7 @@ Activity, AppendVariableActivity, CopyActivity, + DbtFactoryActivity, DeleteActivity, ExecutePipelineActivity, FilterActivity, @@ -124,6 +125,7 @@ def prepare_activity( append_variable, copy, databricks_job, + dbt_factory, delete, execute_pipeline, filter, @@ -158,6 +160,7 @@ def prepare_activity( SwitchActivity: switch.prepare, WaitActivity: wait.prepare, MotifActivity: motif.prepare, + DbtFactoryActivity: dbt_factory.prepare, } preparer_fn = dispatch.get(type(activity)) diff --git a/src/flowx/sources/__init__.py b/src/flowx/sources/__init__.py new file mode 100644 index 0000000..f152134 --- /dev/null +++ b/src/flowx/sources/__init__.py @@ -0,0 +1,71 @@ +"""Source registry: isolate each migration source behind a uniform interface. + +flowx converts *from* a source orchestrator (ADF, Airflow, ...) *to* Databricks +Lakeflow Jobs. The ``discover`` and ``convert`` phases are irreducibly +source-specific (ADF ARM JSON vs. Airflow Python DAGs share no parser), so each +source lives in its own subpackage and registers the phase modules the adapter +should route to. The ``package`` phase is source-independent -- it consumes the +shared :class:`~flowx.models.ir.Pipeline` IR every source produces -- so it is +not part of a source's registration. + +Each source's parser/translator lives under ``flowx.sources.`` (ADF's +loader + translate, Airflow's loader + discover/convert). The IR, the preparer, +and the bundler stay source-neutral and shared. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class Source: + """A registered migration source and the phase modules it routes to. + + Attributes: + name: Source identifier used by ``--source`` (e.g. ``"adf"``). + discover_module: Import path of the discover-phase module (exposes + ``main(argv)``). + convert_module: Import path of the convert-phase module (exposes + ``main(argv)``). + source_path_flag: The source-specific alias for ``--source-path`` + accepted on the discover/convert runners (e.g. ``--adf-source-path``). + """ + + name: str + discover_module: str + convert_module: str + source_path_flag: str + + +_REGISTRY: dict[str, Source] = { + "adf": Source( + name="adf", + discover_module="flowx.sources.adf.loader", + convert_module="flowx.sources.adf.translate", + source_path_flag="--adf-source-path", + ), + "airflow": Source( + name="airflow", + discover_module="flowx.sources.airflow.discover", + convert_module="flowx.sources.airflow.convert", + source_path_flag="--airflow-source-path", + ), +} + + +def available_sources() -> tuple[str, ...]: + """Returns the registered source names in a stable order.""" + return tuple(sorted(_REGISTRY)) + + +def get_source(name: str) -> Source: + """Returns the :class:`Source` for *name*. + + Raises: + KeyError: When *name* is not a registered source. + """ + try: + return _REGISTRY[name] + except KeyError: + raise KeyError(f"Unknown source {name!r}; available: {', '.join(available_sources())}") from None diff --git a/src/flowx/translator/__init__.py b/src/flowx/sources/adf/__init__.py similarity index 100% rename from src/flowx/translator/__init__.py rename to src/flowx/sources/adf/__init__.py diff --git a/src/flowx/parser/ir_rewriter.py b/src/flowx/sources/adf/ir_rewriter.py similarity index 100% rename from src/flowx/parser/ir_rewriter.py rename to src/flowx/sources/adf/ir_rewriter.py diff --git a/src/flowx/parser/adf_loader.py b/src/flowx/sources/adf/loader.py similarity index 100% rename from src/flowx/parser/adf_loader.py rename to src/flowx/sources/adf/loader.py diff --git a/src/flowx/translator/query_analysis.py b/src/flowx/sources/adf/query_analysis.py similarity index 100% rename from src/flowx/translator/query_analysis.py rename to src/flowx/sources/adf/query_analysis.py diff --git a/src/flowx/translator/engine.py b/src/flowx/sources/adf/translate.py similarity index 73% rename from src/flowx/translator/engine.py rename to src/flowx/sources/adf/translate.py index b45e30e..7a9785d 100644 --- a/src/flowx/translator/engine.py +++ b/src/flowx/sources/adf/translate.py @@ -13,6 +13,7 @@ from types import MappingProxyType from typing import Any, Callable +from flowx import ir_serde from flowx.models.adf_ast import ( AdfActivity, AdfDefinitions, @@ -22,35 +23,18 @@ from flowx.models.ir import ( Activity, AgenticGap, - AppendVariableActivity, - CopyActivity, - DeleteActivity, Dependency, - ExecutePipelineActivity, - FilterActivity, - ForEachActivity, - IfConditionActivity, - LookupActivity, - MotifActivity, - NotebookActivity, Pipeline, PlaceholderActivity, - RunJobActivity, SetVariableActivity, - SparkJarActivity, - SparkPythonActivity, - SwitchActivity, TranslationContext, TranslationReport, - UnsupportedActivity, - WaitActivity, - WebActivity, ) from flowx.motifs.collapser import collapse_motifs from flowx.motifs.detector import detect_motifs -from flowx.parser.adf_loader import classify_activity, load_adf_definitions -from flowx.parser.ir_rewriter import rewrite_pipeline_expressions -from flowx.translator.activity_translators import ( +from flowx.sources.adf.ir_rewriter import rewrite_pipeline_expressions +from flowx.sources.adf.loader import classify_activity, load_adf_definitions +from flowx.sources.adf.translators import ( append_variable, copy, databricks_job, @@ -111,7 +95,7 @@ def translate_pipeline( Notes: After dispatching individual activities the translator runs - :func:`~flowx.parser.ir_rewriter.rewrite_pipeline_expressions` + :func:`~flowx.sources.adf.ir_rewriter.rewrite_pipeline_expressions` over the whole IR so that ``@{...}`` ADF expressions embedded in SQL bodies, REST payloads, dataset paths, and other string-typed fields are rewritten through the same parser the per-activity @@ -1259,429 +1243,6 @@ def _extract_cluster_config( return config if config else None -def _pipeline_to_dict(pipeline: Pipeline) -> dict[str, Any]: - """Serialise a Pipeline IR to a JSON-friendly dictionary. - - Args: - pipeline: The translated pipeline IR. - - Returns: - Dictionary suitable for ``json.dumps``. - """ - result: dict[str, Any] = { - "name": pipeline.name, - "parameters": pipeline.parameters, - "schedule": pipeline.schedule, - "tags": pipeline.tags, - "tasks": [_activity_to_dict(task) for task in pipeline.tasks], - } - if pipeline.translation_configuration is not None: - result["translation_configuration"] = _configuration_to_dict(pipeline.translation_configuration) - return result - - -def _configuration_to_dict(configuration: Any) -> dict[str, Any]: - """Serialise a TranslationConfiguration instance to a JSON-friendly dictionary. - - Args: - configuration: The :class:`TranslationConfiguration` snapshot to serialise. - - Returns: - Dictionary with each StrEnum field rendered as its string value - and per-task overrides preserved verbatim. - """ - return { - "copy_activity_paradigm": str(configuration.copy_activity_paradigm), - "non_databricks_task_compute": str(configuration.non_databricks_task_compute), - "use_lakeflow_connectors": str(configuration.use_lakeflow_connectors), - "lakeflow_connector_type": str(configuration.lakeflow_connector_type), - "motif_consolidations": { - motif_id: str(choice) for motif_id, choice in configuration.motif_consolidations.items() - }, - "per_task": dict(configuration.per_task), - } - - -def _activity_to_dict(task: Activity) -> dict[str, Any]: - """Serialise a single Activity IR node to a JSON-friendly dictionary. - - Args: - task: Any Activity IR node. - - Returns: - Dictionary suitable for ``json.dumps``. - """ - task_dict: dict[str, Any] = { - "name": task.name, - "task_key": task.task_key, - "type": type(task).__name__, - } - if task.description: - task_dict["description"] = task.description - if task.timeout_seconds: - task_dict["timeout_seconds"] = task.timeout_seconds - if task.max_retries: - task_dict["max_retries"] = task.max_retries - if task.min_retry_interval_millis: - task_dict["min_retry_interval_millis"] = task.min_retry_interval_millis - if task.depends_on: - task_dict["depends_on"] = [ - {"task_key": dependency.task_key, "outcome": dependency.outcome} for dependency in task.depends_on - ] - if task.cluster: - task_dict["cluster"] = task.cluster - if task.existing_cluster_id: - task_dict["existing_cluster_id"] = task.existing_cluster_id - if task.compute_mode: - task_dict["compute_mode"] = task.compute_mode - if task.notifications: - task_dict["notifications"] = task.notifications - if task.libraries: - task_dict["libraries"] = task.libraries - if task.parameter_approximations: - task_dict["parameter_approximations"] = task.parameter_approximations - - extra = _activity_extra_fields(task) - task_dict.update(extra) - return task_dict - - -def _activity_extra_fields(activity: Activity) -> dict[str, Any]: - """Extracts type-specific fields from an Activity subclass. - - Args: - activity: Any Activity IR node. - - Returns: - Dictionary of extra fields beyond the base Activity. - """ - extra: dict[str, Any] = {} - - match activity: - case NotebookActivity(): - extra["notebook_path"] = activity.notebook_path - if activity.base_parameters: - extra["base_parameters"] = activity.base_parameters - if activity.notebook_path_unresolved: - extra["notebook_path_unresolved"] = True - if activity.notebook_path_expression is not None: - extra["notebook_path_expression"] = activity.notebook_path_expression - if activity.unresolved_libraries: - extra["unresolved_libraries"] = list(activity.unresolved_libraries) - case CopyActivity(): - extra["source_type"] = activity.source_type - extra["sink_type"] = activity.sink_type - if activity.source_properties: - extra["source_properties"] = activity.source_properties - if activity.sink_properties: - extra["sink_properties"] = activity.sink_properties - if activity.sink_dataset_type: - extra["sink_dataset_type"] = activity.sink_dataset_type - if activity.sink_format: - extra["sink_format"] = activity.sink_format - if activity.sink_resolved_path: - extra["sink_resolved_path"] = activity.sink_resolved_path - if activity.column_mapping: - extra["column_mapping"] = activity.column_mapping - if activity.target_format: - extra["target_format"] = activity.target_format - if activity.use_lakeflow_connector: - extra["use_lakeflow_connector"] = activity.use_lakeflow_connector - if activity.lakeflow_connector_type: - extra["lakeflow_connector_type"] = activity.lakeflow_connector_type - case ForEachActivity(): - extra["items_expression"] = activity.items_expression - extra["concurrency"] = activity.concurrency - extra["inner_activities"] = [_activity_to_dict(inner) for inner in activity.inner_activities] - if activity.inputs_bridge_notebook_code: - extra["inputs_bridge_notebook_code"] = activity.inputs_bridge_notebook_code - if activity.inputs_bridge_notebook_imports: - extra["inputs_bridge_notebook_imports"] = list(activity.inputs_bridge_notebook_imports) - if activity.inputs_bridge_required_parameters: - extra["inputs_bridge_required_parameters"] = dict(activity.inputs_bridge_required_parameters) - case IfConditionActivity(): - extra["op"] = activity.op - extra["left"] = activity.left - extra["right"] = activity.right - extra["if_true_activities"] = [_activity_to_dict(inner) for inner in activity.if_true_activities] - extra["if_false_activities"] = [_activity_to_dict(inner) for inner in activity.if_false_activities] - if activity.bridge_notebook_code: - extra["bridge_notebook_code"] = activity.bridge_notebook_code - if activity.bridge_notebook_imports: - extra["bridge_notebook_imports"] = list(activity.bridge_notebook_imports) - if activity.bridge_required_parameters: - extra["bridge_required_parameters"] = dict(activity.bridge_required_parameters) - case LookupActivity(): - extra["source_type"] = activity.source_type - if activity.source_properties: - extra["source_properties"] = activity.source_properties - extra["first_row_only"] = activity.first_row_only - if activity.source_query: - extra["source_query"] = activity.source_query - case SetVariableActivity(): - extra["variable_name"] = activity.variable_name - extra["variable_value"] = activity.variable_value - extra["value_kind"] = activity.value_kind - if activity.notebook_code: - extra["notebook_code"] = activity.notebook_code - if activity.notebook_imports: - extra["notebook_imports"] = activity.notebook_imports - if activity.required_parameters: - extra["required_parameters"] = dict(activity.required_parameters) - if activity.raw_expression: - extra["raw_expression"] = activity.raw_expression - case FilterActivity(): - extra["items_expression"] = activity.items_expression - extra["condition_expression"] = activity.condition_expression - if activity.condition_code is not None: - extra["condition_code"] = activity.condition_code - if activity.condition_imports: - extra["condition_imports"] = list(activity.condition_imports) - case AppendVariableActivity(): - extra["variable_name"] = activity.variable_name - extra["append_value"] = activity.append_value - extra["value_kind"] = activity.value_kind - if activity.notebook_code: - extra["notebook_code"] = activity.notebook_code - if activity.notebook_imports: - extra["notebook_imports"] = activity.notebook_imports - if activity.required_parameters: - extra["required_parameters"] = dict(activity.required_parameters) - case SwitchActivity(): - extra["on_expression"] = activity.on_expression - extra["cases"] = [ - {"value": case_item.value, "activities": [_activity_to_dict(inner) for inner in case_item.activities]} - for case_item in activity.cases - ] - extra["default_activities"] = [_activity_to_dict(inner) for inner in activity.default_activities] - if activity.bridge_notebook_code: - extra["bridge_notebook_code"] = activity.bridge_notebook_code - if activity.bridge_notebook_imports: - extra["bridge_notebook_imports"] = list(activity.bridge_notebook_imports) - if activity.bridge_required_parameters: - extra["bridge_required_parameters"] = dict(activity.bridge_required_parameters) - case WaitActivity(): - extra["wait_time_seconds"] = activity.wait_time_seconds - case SparkJarActivity(): - extra["main_class_name"] = activity.main_class_name - if activity.parameters: - extra["parameters"] = activity.parameters - case SparkPythonActivity(): - extra["python_file"] = activity.python_file - if activity.parameters: - extra["parameters"] = activity.parameters - case WebActivity(): - extra["url"] = activity.url - extra["method"] = activity.method - if activity.body is not None: - extra["body"] = activity.body - if activity.headers: - extra["headers"] = activity.headers - if activity.authentication: - extra["authentication"] = activity.authentication - if activity.body_code is not None: - extra["body_code"] = activity.body_code - if activity.body_imports: - extra["body_imports"] = activity.body_imports - if activity.body_required_parameters: - extra["body_required_parameters"] = activity.body_required_parameters - if activity.disable_cert_validation: - extra["disable_cert_validation"] = activity.disable_cert_validation - if activity.http_request_timeout_seconds: - extra["http_request_timeout_seconds"] = activity.http_request_timeout_seconds - case DeleteActivity(): - extra["dataset_name"] = activity.dataset_name - if activity.folder_path: - extra["folder_path"] = activity.folder_path - extra["recursive"] = activity.recursive - case ExecutePipelineActivity(): - extra["pipeline_name"] = activity.pipeline_name - extra["wait_on_completion"] = activity.wait_on_completion - if activity.parameters: - extra["parameters"] = activity.parameters - case RunJobActivity(): - extra["job_name"] = activity.job_name - if activity.existing_job_id: - extra["existing_job_id"] = activity.existing_job_id - if activity.job_parameters: - extra["job_parameters"] = activity.job_parameters - case MotifActivity(): - extra["motif_id"] = activity.motif_id - extra["display_name"] = activity.display_name - extra["databricks_replacement"] = activity.databricks_replacement - extra["matched_activity_names"] = activity.matched_activity_names - if activity.source_type_hint: - extra["source_type_hint"] = activity.source_type_hint - if activity.confidence_notes: - extra["confidence_notes"] = activity.confidence_notes - if activity.notebook_template: - extra["notebook_template"] = activity.notebook_template - if activity.motif_config: - extra["motif_config"] = activity.motif_config - if activity.consolidate_metadata_driven: - extra["consolidate_metadata_driven"] = activity.consolidate_metadata_driven - if activity.lookup_values: - extra["lookup_values"] = activity.lookup_values - case PlaceholderActivity(): - extra["original_type"] = activity.original_type - extra["comment"] = activity.comment - case UnsupportedActivity(): - extra["original_type"] = activity.original_type - extra["reason"] = activity.reason - - return extra - - -def _activity_to_debug_dict(activity: Activity) -> dict[str, Any]: - """Serialise an Activity to a full debug dict showing all dataclass fields. - - Args: - activity: Any Activity IR node. - - Returns: - Dict with ``__class__`` plus every dataclass field. - """ - result: dict[str, Any] = {"__class__": type(activity).__name__} - - for field in activity.__dataclass_fields__: - value = getattr(activity, field) - - if isinstance(value, Activity): - result[field] = _activity_to_debug_dict(value) - elif isinstance(value, list) and value and isinstance(value[0], Activity): - result[field] = [_activity_to_debug_dict(inner) for inner in value] - elif isinstance(value, list) and value and hasattr(value[0], "__dataclass_fields__"): - result[field] = [_dataclass_to_debug_dict(item) for item in value] - else: - result[field] = value - - return result - - -def _dataclass_to_debug_dict(obj: Any) -> dict[str, Any]: - """Serialise a generic dataclass (SwitchCase, Dependency, etc.) to a debug dict. - - Args: - obj: A dataclass instance. - - Returns: - Dict with ``__class__`` plus every dataclass field. - """ - result: dict[str, Any] = {"__class__": type(obj).__name__} - - for field in obj.__dataclass_fields__: - value = getattr(obj, field) - - if isinstance(value, Activity): - result[field] = _activity_to_debug_dict(value) - elif isinstance(value, list) and value and isinstance(value[0], Activity): - result[field] = [_activity_to_debug_dict(inner) for inner in value] - else: - result[field] = value - - return result - - -def _pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]: - """Serialise a Pipeline IR to a full debug dict. - - Args: - pipeline: The translated pipeline IR. - - Returns: - Dict with every field fully expanded. - """ - return { - "__class__": "Pipeline", - "name": pipeline.name, - "parameters": pipeline.parameters, - "schedule": pipeline.schedule, - "tags": pipeline.tags, - "tasks": [_activity_to_debug_dict(task) for task in pipeline.tasks], - } - - -def _find_and_replace_task(tasks: list[dict[str, Any]], activity_name: str, replacement: dict[str, Any]) -> bool: - """Replace the task named *activity_name* with *replacement*, recursing into containers. - - Searches top-level tasks and the nested activity lists of IfCondition / - ForEach / Switch containers. Preserves the placeholder's ``task_key`` and - ``depends_on`` when the replacement omits them so downstream dependency - edges stay intact. Returns True when a match was replaced. - """ - nested_keys = ("inner_activities", "if_true_activities", "if_false_activities", "default_activities") - for index, task in enumerate(tasks): - if task.get("name") == activity_name: - replacement.setdefault("task_key", task.get("task_key")) - replacement.setdefault("name", activity_name) - if "depends_on" not in replacement and task.get("depends_on"): - replacement["depends_on"] = task["depends_on"] - tasks[index] = replacement - return True - for key in nested_keys: - child = task.get(key) - if isinstance(child, list) and _find_and_replace_task(child, activity_name, replacement): - return True - for case in task.get("cases") or []: - if isinstance(case, dict) and isinstance(case.get("activities"), list): - if _find_and_replace_task(case["activities"], activity_name, replacement): - return True - return False - - -def merge_agentic_results(report_path: Path, results_dir: Path, output_path: Path | None = None) -> tuple[int, int]: - """Merge agent-produced per-activity translations into a translation report. - - Each ``*.json`` file in *results_dir* describes one resolved agentic gap:: - - { - "activity_name": "", # required - "pipeline": "", # optional; for multi-pipeline reports - "task": { ...IR task dict... } # required; replacement task - } - - The matching placeholder task (located by ``name``, recursing into - IfCondition / ForEach / Switch containers) is replaced by ``task``. Use a - ``NotebookActivity`` whose ``notebook_path`` points at a notebook the agent - wrote to the workspace; the prepare phase then references it directly. - - Args: - report_path: ``translation_report.json`` produced by the translate phase. - results_dir: Directory of per-activity result JSON files. - output_path: Where to write the merged report; defaults to overwriting - *report_path*. - - Returns: - ``(merged, unmatched)`` counts. - """ - report = json.loads(report_path.read_text(encoding="utf-8")) - pipelines = report["pipelines"] if isinstance(report, dict) and "pipelines" in report else [report] - - merged = 0 - unmatched = 0 - for result_file in sorted(results_dir.glob("*.json")): - data = json.loads(result_file.read_text(encoding="utf-8")) - activity_name = data.get("activity_name") or data.get("activity") - task = data.get("task") or data.get("ir") - if not activity_name or not isinstance(task, dict): - logger.warning("Skipping %s: missing 'activity_name' or 'task'.", result_file.name) - unmatched += 1 - continue - wanted = data.get("pipeline") - candidates = [pipeline for pipeline in pipelines if not wanted or pipeline.get("name") == wanted] - if any(_find_and_replace_task(pipeline.get("tasks", []), activity_name, dict(task)) for pipeline in candidates): - merged += 1 - logger.info("Merged agentic result for '%s' from %s", activity_name, result_file.name) - else: - logger.warning("No placeholder named '%s' found for %s", activity_name, result_file.name) - unmatched += 1 - - destination = output_path or report_path - destination.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") - logger.info("Wrote merged report to %s (%d merged, %d unmatched)", destination, merged, unmatched) - return merged, unmatched - - def main(argv: list[str] | None = None) -> int: """Convert-phase entry point: translate ADF pipelines to IR (or merge agentic results). @@ -1741,7 +1302,9 @@ def main(argv: list[str] | None = None) -> int: if args.merge_agentic: if not args.report or not args.agentic_results: parser.error("--merge-agentic requires --report and --agentic-results") - merged_count, unmatched_count = merge_agentic_results(args.report, args.agentic_results, args.output) + merged_count, unmatched_count = ir_serde.merge_agentic_results( + args.report, args.agentic_results, args.output + ) print("\nAgentic Merge Summary") print("=====================") print(f"Merged: {merged_count}") @@ -1776,7 +1339,7 @@ def main(argv: list[str] | None = None) -> int: total_unsupported += report.unsupported_count pipeline_file = work_dir / f"{_sanitize_task_key(pipeline.name)}.json" - pipeline_dict = _pipeline_to_dict(report.pipeline) + pipeline_dict = ir_serde.pipeline_to_dict(report.pipeline) pipeline_file.write_text(json.dumps(pipeline_dict, indent=2, default=str), encoding="utf-8") logger.info("Wrote pipeline IR to %s", pipeline_file) all_pipeline_dicts.append(pipeline_dict) @@ -1784,7 +1347,7 @@ def main(argv: list[str] | None = None) -> int: # Write debug IR if requested if args.debug: debug_file = work_dir / f"{_sanitize_task_key(pipeline.name)}.debug.json" - debug_dict = _pipeline_to_debug_dict(report.pipeline) + debug_dict = ir_serde.pipeline_to_debug_dict(report.pipeline) debug_file.write_text(json.dumps(debug_dict, indent=2, default=str), encoding="utf-8") logger.info("Wrote debug IR to %s", debug_file) diff --git a/src/flowx/translator/activity_translators/__init__.py b/src/flowx/sources/adf/translators/__init__.py similarity index 100% rename from src/flowx/translator/activity_translators/__init__.py rename to src/flowx/sources/adf/translators/__init__.py diff --git a/src/flowx/translator/activity_translators/append_variable.py b/src/flowx/sources/adf/translators/append_variable.py similarity index 100% rename from src/flowx/translator/activity_translators/append_variable.py rename to src/flowx/sources/adf/translators/append_variable.py diff --git a/src/flowx/translator/activity_translators/copy.py b/src/flowx/sources/adf/translators/copy.py similarity index 99% rename from src/flowx/translator/activity_translators/copy.py rename to src/flowx/sources/adf/translators/copy.py index af595a4..fc65bc9 100644 --- a/src/flowx/translator/activity_translators/copy.py +++ b/src/flowx/sources/adf/translators/copy.py @@ -13,7 +13,7 @@ resolve_interpolated_string, resolve_interpolated_string_for_notebook, ) -from flowx.translator.query_analysis import analyze_copy_query, dialect_for_source_type +from flowx.sources.adf.query_analysis import analyze_copy_query, dialect_for_source_type _DATASET_TYPE_TO_SPARK_FORMAT: dict[str, str] = { "DelimitedText": "csv", diff --git a/src/flowx/translator/activity_translators/databricks_job.py b/src/flowx/sources/adf/translators/databricks_job.py similarity index 93% rename from src/flowx/translator/activity_translators/databricks_job.py rename to src/flowx/sources/adf/translators/databricks_job.py index f39f084..7775297 100644 --- a/src/flowx/translator/activity_translators/databricks_job.py +++ b/src/flowx/sources/adf/translators/databricks_job.py @@ -6,7 +6,7 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, RunJobActivity, TranslationContext -from flowx.translator.activity_translators.resolve import resolve_dict_values, resolve_field +from flowx.sources.adf.translators.resolve import resolve_dict_values, resolve_field def translate( diff --git a/src/flowx/translator/activity_translators/delete.py b/src/flowx/sources/adf/translators/delete.py similarity index 96% rename from src/flowx/translator/activity_translators/delete.py rename to src/flowx/sources/adf/translators/delete.py index e133339..dc940cc 100644 --- a/src/flowx/translator/activity_translators/delete.py +++ b/src/flowx/sources/adf/translators/delete.py @@ -6,7 +6,7 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, DeleteActivity, TranslationContext -from flowx.translator.activity_translators.resolve import resolve_field +from flowx.sources.adf.translators.resolve import resolve_field def translate( diff --git a/src/flowx/translator/activity_translators/execute_pipeline.py b/src/flowx/sources/adf/translators/execute_pipeline.py similarity index 98% rename from src/flowx/translator/activity_translators/execute_pipeline.py rename to src/flowx/sources/adf/translators/execute_pipeline.py index 89276c9..dfe169d 100644 --- a/src/flowx/translator/activity_translators/execute_pipeline.py +++ b/src/flowx/sources/adf/translators/execute_pipeline.py @@ -7,7 +7,7 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, ExecutePipelineActivity, TranslationContext from flowx.parser.expression_parser import resolve_expression -from flowx.translator.activity_translators.resolve import resolve_field +from flowx.sources.adf.translators.resolve import resolve_field def translate( diff --git a/src/flowx/translator/activity_translators/filter.py b/src/flowx/sources/adf/translators/filter.py similarity index 100% rename from src/flowx/translator/activity_translators/filter.py rename to src/flowx/sources/adf/translators/filter.py diff --git a/src/flowx/translator/activity_translators/for_each.py b/src/flowx/sources/adf/translators/for_each.py similarity index 98% rename from src/flowx/translator/activity_translators/for_each.py rename to src/flowx/sources/adf/translators/for_each.py index 12866d1..0a75aa2 100644 --- a/src/flowx/translator/activity_translators/for_each.py +++ b/src/flowx/sources/adf/translators/for_each.py @@ -7,7 +7,7 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, ForEachActivity, TranslationContext from flowx.parser.expression_parser import resolve_expression -from flowx.translator.activity_translators.resolve import resolve_field_int +from flowx.sources.adf.translators.resolve import resolve_field_int def translate( diff --git a/src/flowx/translator/activity_translators/if_condition.py b/src/flowx/sources/adf/translators/if_condition.py similarity index 99% rename from src/flowx/translator/activity_translators/if_condition.py rename to src/flowx/sources/adf/translators/if_condition.py index 9131222..96561d2 100644 --- a/src/flowx/translator/activity_translators/if_condition.py +++ b/src/flowx/sources/adf/translators/if_condition.py @@ -11,7 +11,7 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, IfConditionActivity, TranslationContext -from flowx.translator.activity_translators.resolve import ( +from flowx.sources.adf.translators.resolve import ( BridgeRequest, lower_to_bridge, merge_bridge_requests, diff --git a/src/flowx/translator/activity_translators/lookup.py b/src/flowx/sources/adf/translators/lookup.py similarity index 99% rename from src/flowx/translator/activity_translators/lookup.py rename to src/flowx/sources/adf/translators/lookup.py index 1700e4f..04161f2 100644 --- a/src/flowx/translator/activity_translators/lookup.py +++ b/src/flowx/sources/adf/translators/lookup.py @@ -7,7 +7,7 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, LookupActivity, TranslationContext -from flowx.translator.activity_translators.resolve import resolve_field +from flowx.sources.adf.translators.resolve import resolve_field def _dataset_parameter_scope(activity: AdfActivity, context: TranslationContext) -> dict[str, str]: diff --git a/src/flowx/translator/activity_translators/notebook.py b/src/flowx/sources/adf/translators/notebook.py similarity index 99% rename from src/flowx/translator/activity_translators/notebook.py rename to src/flowx/sources/adf/translators/notebook.py index 74dd5bc..513b8c2 100644 --- a/src/flowx/translator/activity_translators/notebook.py +++ b/src/flowx/sources/adf/translators/notebook.py @@ -8,7 +8,7 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, NotebookActivity, TranslationContext from flowx.parser.expression_parser import resolve_expression -from flowx.translator.activity_translators.resolve import resolve_field +from flowx.sources.adf.translators.resolve import resolve_field def translate( diff --git a/src/flowx/translator/activity_translators/resolve.py b/src/flowx/sources/adf/translators/resolve.py similarity index 100% rename from src/flowx/translator/activity_translators/resolve.py rename to src/flowx/sources/adf/translators/resolve.py diff --git a/src/flowx/translator/activity_translators/set_variable.py b/src/flowx/sources/adf/translators/set_variable.py similarity index 100% rename from src/flowx/translator/activity_translators/set_variable.py rename to src/flowx/sources/adf/translators/set_variable.py diff --git a/src/flowx/translator/activity_translators/spark_jar.py b/src/flowx/sources/adf/translators/spark_jar.py similarity index 96% rename from src/flowx/translator/activity_translators/spark_jar.py rename to src/flowx/sources/adf/translators/spark_jar.py index 4da5c6f..c1faa7d 100644 --- a/src/flowx/translator/activity_translators/spark_jar.py +++ b/src/flowx/sources/adf/translators/spark_jar.py @@ -7,7 +7,7 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, SparkJarActivity, TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string -from flowx.translator.activity_translators.resolve import resolve_field +from flowx.sources.adf.translators.resolve import resolve_field def translate( diff --git a/src/flowx/translator/activity_translators/spark_python.py b/src/flowx/sources/adf/translators/spark_python.py similarity index 96% rename from src/flowx/translator/activity_translators/spark_python.py rename to src/flowx/sources/adf/translators/spark_python.py index 87c53de..46e4a7e 100644 --- a/src/flowx/translator/activity_translators/spark_python.py +++ b/src/flowx/sources/adf/translators/spark_python.py @@ -7,7 +7,7 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, SparkPythonActivity, TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string -from flowx.translator.activity_translators.resolve import resolve_field +from flowx.sources.adf.translators.resolve import resolve_field def _resolve_parameter(param: str, context: TranslationContext) -> str: diff --git a/src/flowx/translator/activity_translators/switch.py b/src/flowx/sources/adf/translators/switch.py similarity index 97% rename from src/flowx/translator/activity_translators/switch.py rename to src/flowx/sources/adf/translators/switch.py index 470407c..ad5ea56 100644 --- a/src/flowx/translator/activity_translators/switch.py +++ b/src/flowx/sources/adf/translators/switch.py @@ -6,9 +6,9 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, SwitchActivity, SwitchCase, TranslationContext -from flowx.parser.adf_loader import parse_activity from flowx.parser.expression_parser import resolve_interpolated_string -from flowx.translator.activity_translators.resolve import ( +from flowx.sources.adf.loader import parse_activity +from flowx.sources.adf.translators.resolve import ( BridgeRequest, lower_to_bridge, resolve_field, diff --git a/src/flowx/translator/activity_translators/wait.py b/src/flowx/sources/adf/translators/wait.py similarity index 93% rename from src/flowx/translator/activity_translators/wait.py rename to src/flowx/sources/adf/translators/wait.py index 6577f06..3762c68 100644 --- a/src/flowx/translator/activity_translators/wait.py +++ b/src/flowx/sources/adf/translators/wait.py @@ -6,7 +6,7 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions from flowx.models.ir import Activity, TranslationContext, WaitActivity -from flowx.translator.activity_translators.resolve import resolve_field_int +from flowx.sources.adf.translators.resolve import resolve_field_int def translate( diff --git a/src/flowx/translator/activity_translators/web_activity.py b/src/flowx/sources/adf/translators/web_activity.py similarity index 98% rename from src/flowx/translator/activity_translators/web_activity.py rename to src/flowx/sources/adf/translators/web_activity.py index c1da35b..c82f426 100644 --- a/src/flowx/translator/activity_translators/web_activity.py +++ b/src/flowx/sources/adf/translators/web_activity.py @@ -12,7 +12,7 @@ resolve_expression, resolve_interpolated_string_for_notebook, ) -from flowx.translator.activity_translators.resolve import resolve_dict_values, resolve_field +from flowx.sources.adf.translators.resolve import resolve_dict_values, resolve_field def translate( diff --git a/src/flowx/sources/airflow/__init__.py b/src/flowx/sources/airflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/flowx/sources/airflow/convert.py b/src/flowx/sources/airflow/convert.py new file mode 100644 index 0000000..0c8638a --- /dev/null +++ b/src/flowx/sources/airflow/convert.py @@ -0,0 +1,57 @@ +"""Airflow convert phase: parse DAGs into the shared translation report. + +Writes ``.work/translation_report.json`` (single pipeline dict, or a +``{"pipelines": [...]}`` wrapper for many) in the exact shape the ADF convert +phase emits, so the shared package phase consumes it unchanged. Reuses the +source-neutral ``flowx.ir_serde.pipeline_to_dict`` so both sources converge on +one report format. Exposes ``main(argv)`` for the adapter to run in-process. +""" + +from __future__ import annotations + +import argparse +import json +import logging +from pathlib import Path + +from flowx.ir_serde import pipeline_to_dict +from flowx.sources.airflow.loader import load_pipelines + +logger = logging.getLogger(__name__) + + +def main(argv: list[str] | None = None) -> int: + """Convert-phase entry point for the Airflow source.""" + parser = argparse.ArgumentParser(description="Translate Airflow DAGs into flowx Pipeline IR.") + parser.add_argument("--source-dir", required=True, type=Path, help="A DAG .py file or directory of DAGs.") + parser.add_argument("--output-dir", type=Path, default=Path("./flowx_output"), help="Shared migration output dir.") + parser.add_argument("--pipeline", type=str, default=None, help="Translate only the named DAG (default: all).") + args = parser.parse_args(argv) + + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + pipelines = load_pipelines(args.source_dir, pipeline=args.pipeline) + if not pipelines: + logger.error("No Airflow DAGs found under %s (or none matched --pipeline).", args.source_dir) + return 1 + + output_dir: Path = args.output_dir.resolve() + work_dir = output_dir / ".work" + work_dir.mkdir(parents=True, exist_ok=True) + + pipeline_dicts = [pipeline_to_dict(pipeline) for pipeline in pipelines] + payload = pipeline_dicts[0] if len(pipeline_dicts) == 1 else {"pipelines": pipeline_dicts} + report_file = work_dir / "translation_report.json" + report_file.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") + + total_tasks = sum(len(p.tasks) for p in pipelines) + print("\nAirflow Translation Summary") + print("===========================") + print(f"DAGs translated: {len(pipelines)}") + print(f"Total tasks: {total_tasks}") + print(f"\nTranslation report (intermediate): {report_file}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/flowx/sources/airflow/discover.py b/src/flowx/sources/airflow/discover.py new file mode 100644 index 0000000..df3fba0 --- /dev/null +++ b/src/flowx/sources/airflow/discover.py @@ -0,0 +1,114 @@ +"""Airflow discover phase: parse DAGs into a classified inventory. + +Mirrors the ADF discover contract: writes ``metadata/inventory.json`` and +``metadata/profile_report.csv`` under the shared output dir, classifying each +task as deterministic (a mapped operator -> NotebookActivity) or agentic (an +unmapped operator -> PlaceholderActivity, needing LLM-assisted translation). +Exposes ``main(argv)`` so the adapter runs it in-process, like the ADF loader. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import logging +from pathlib import Path +from typing import Any + +from flowx.models.ir import NotebookActivity, Pipeline, PlaceholderActivity +from flowx.sources.adf.loader import clear_stale_outputs +from flowx.sources.airflow.loader import load_pipelines + +logger = logging.getLogger(__name__) + + +def _classify(pipeline: Pipeline) -> list[dict[str, str]]: + """Classifies each task in *pipeline* for the inventory.""" + items: list[dict[str, str]] = [] + for task in pipeline.tasks: + if isinstance(task, NotebookActivity): + strategy = "deterministic" + elif isinstance(task, PlaceholderActivity): + strategy = "agentic" + else: + strategy = "deterministic" + items.append({"name": task.name, "task_key": task.task_key, "strategy": strategy}) + return items + + +def build_inventory_dict(pipelines: list[Pipeline], source_dir: str) -> dict[str, Any]: + """Builds the inventory.json payload matching the ADF discover shape.""" + pipeline_entries: list[dict[str, Any]] = [] + deterministic = agentic = 0 + for pipeline in pipelines: + items = _classify(pipeline) + deterministic += sum(1 for i in items if i["strategy"] == "deterministic") + agentic += sum(1 for i in items if i["strategy"] == "agentic") + pipeline_entries.append({"name": pipeline.name, "activities": items}) + activity_count = deterministic + agentic + coverage = round(100.0 * deterministic / activity_count, 1) if activity_count else 0.0 + return { + "source": "airflow", + "source_dir": source_dir, + "pipelines": pipeline_entries, + "summary": { + "pipeline_count": len(pipelines), + "activity_count": activity_count, + "deterministic_count": deterministic, + "agentic_count": agentic, + "unsupported_count": 0, + "coverage_pct": coverage, + }, + } + + +def _write_profile_csv(pipelines: list[Pipeline], path: Path) -> None: + """Writes one per-pipeline complexity row, mirroring the ADF profile report.""" + with open(path, "w", newline="", encoding="utf-8") as handle: + writer = csv.writer(handle) + writer.writerow(["pipeline", "activities", "complexity_size"]) + for pipeline in pipelines: + count = len(pipeline.tasks) + size = "S" if count <= 5 else "M" if count <= 15 else "L" if count <= 30 else "XL" + writer.writerow([pipeline.name, count, size]) + + +def main(argv: list[str] | None = None) -> int: + """Discover-phase entry point for the Airflow source.""" + parser = argparse.ArgumentParser(description="Parse Airflow DAGs into a flowx inventory.") + parser.add_argument("--source-dir", required=True, type=Path, help="A DAG .py file or directory of DAGs.") + parser.add_argument("--output-dir", type=Path, default=Path("./flowx_output"), help="Shared migration output dir.") + parser.add_argument("--pipeline", type=str, default=None, help="Filter to a single DAG by dag_id.") + args = parser.parse_args(argv) + + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + pipelines = load_pipelines(args.source_dir, pipeline=args.pipeline) + if not pipelines: + logger.error("No Airflow DAGs found under %s (or none matched --pipeline).", args.source_dir) + return 1 + logger.info("Parsed %d DAG(s) from %s", len(pipelines), args.source_dir) + + output_dir: Path = args.output_dir.resolve() + clear_stale_outputs(output_dir) + metadata_dir = output_dir / "metadata" + metadata_dir.mkdir(parents=True, exist_ok=True) + + inventory = build_inventory_dict(pipelines, str(args.source_dir)) + (metadata_dir / "inventory.json").write_text(json.dumps(inventory, indent=2), encoding="utf-8") + _write_profile_csv(pipelines, metadata_dir / "profile_report.csv") + + summary = inventory["summary"] + print("\nAirflow Discover Summary") + print("========================") + print(f"DAGs parsed: {summary['pipeline_count']}") + print(f"Total tasks: {summary['activity_count']}") + print(f" Deterministic: {summary['deterministic_count']}") + print(f" Agentic: {summary['agentic_count']}") + print(f"Coverage: {summary['coverage_pct']}%") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py new file mode 100644 index 0000000..9980294 --- /dev/null +++ b/src/flowx/sources/airflow/loader.py @@ -0,0 +1,295 @@ +"""Airflow DAG parser: parse a DAG file into a flowx Pipeline IR. + +Core parser for the Airflow source (``flowx.sources.airflow``). It reads a DAG +module statically with :mod:`ast` (no Airflow install or DAG execution) and +produces the same :class:`~flowx.models.ir.Pipeline` IR the ADF path emits, so +the shared downstream half -- ``prepare_workflow`` -> ``write_bundle`` -> DABs -- +is reused unchanged. The ``discover`` and ``convert`` phase entry points in +this package wrap :func:`load_airflow_dag`. + +Coverage: PythonOperator (callable body -> generated notebook), BashOperator +(command -> generated notebook), ``>>`` / ``<<`` dependencies, and a cron +``schedule_interval`` -> Quartz. Operators without a mapping become +PlaceholderActivity so coverage still counts them. +""" + +from __future__ import annotations + +import ast +import textwrap +from pathlib import Path + +from flowx.models.ir import ( + Activity, + Dependency, + NotebookActivity, + Pipeline, + PlaceholderActivity, +) + + +def _sanitize_task_key(name: str) -> str: + """Converts an Airflow task_id into a valid Databricks task key.""" + import re + + key = re.sub(r"[^a-zA-Z0-9_-]", "_", name) + key = re.sub(r"_+", "_", key).strip("_") + return key or "unnamed" + + +def _cron_to_quartz(cron: str) -> str | None: + """Converts a 5-field Unix cron to a 6-field Quartz expression. + + Quartz is ``second minute hour day-of-month month day-of-week``; Unix cron + is ``minute hour day-of-month month day-of-week``. Prepend the seconds + field and reconcile the day-of-month / day-of-week wildcard (Quartz rejects + ``*`` in both simultaneously -- one must be ``?``). + """ + fields = cron.split() + if len(fields) != 5: + return None + minute, hour, dom, month, dow = fields + if dow == "*" and dom != "*": + dow = "?" + elif dom == "*": + dom = "?" + return f"0 {minute} {hour} {dom} {month} {dow}" + + +_CRON_PRESETS: dict[str, str] = { + "@hourly": "0 0 * * * ?", + "@daily": "0 0 0 * * ?", + "@midnight": "0 0 0 * * ?", + "@weekly": "0 0 0 ? * SUN", + "@monthly": "0 0 0 1 * ?", + "@yearly": "0 0 0 1 1 ?", + "@annually": "0 0 0 1 1 ?", +} + + +def _schedule_from_interval(interval: str | None) -> dict[str, object] | None: + """Builds a Pipeline.schedule spec from an Airflow schedule_interval.""" + if not interval: + return None + quartz: str | None = _CRON_PRESETS.get(interval) or _cron_to_quartz(interval) + if quartz is None: + return None + return { + "kind": "schedule", + "quartz_cron_expression": quartz, + "timezone_id": "UTC", + "pause_status": "UNPAUSED", + } + + +def _notebook_source_from_callable(func: ast.FunctionDef, source: str) -> str: + """Renders a PythonOperator callable's body as a Databricks notebook. + + Slices each body statement's source segment out of the module *source* and + dedents so the extracted block is valid top-level notebook code. + """ + segments = [ast.get_source_segment(source, stmt) for stmt in func.body] + body_src = "\n\n".join(seg for seg in segments if seg) + body_src = textwrap.dedent(body_src) + return f"# Databricks notebook source\n# Migrated from Airflow PythonOperator '{func.name}'.\n\n" + body_src + "\n" + + +def _notebook_source_from_bash(task_id: str, command: str) -> str: + """Renders a BashOperator command as a Databricks notebook shell cell.""" + return ( + "# Databricks notebook source\n" + f"# Migrated from Airflow BashOperator '{task_id}'.\n\n" + "# MAGIC %sh\n" + "".join(f"# MAGIC {line}\n" for line in command.splitlines()) + ) + + +class _DagVisitor(ast.NodeVisitor): + """Collects operator calls, dependency edges, and the DAG's schedule.""" + + def __init__(self, module: ast.Module) -> None: + self._functions: dict[str, ast.FunctionDef] = { + node.name: node for node in module.body if isinstance(node, ast.FunctionDef) + } + # task variable name -> (task_id, operator, kwargs) + self.operators: dict[str, tuple[str, str, dict[str, ast.expr]]] = {} + self.edges: list[tuple[str, str]] = [] # (upstream_var, downstream_var) + self.dag_id: str | None = None + self.schedule_interval: str | None = None + + def functions(self) -> dict[str, ast.FunctionDef]: + return self._functions + + def visit_Assign(self, node: ast.Assign) -> None: + if ( + isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id.endswith("Operator") + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + ): + var = node.targets[0].id + kwargs = {kw.arg: kw.value for kw in node.value.keywords if kw.arg} + task_id = _literal_str(kwargs.get("task_id")) or var + self.operators[var] = (task_id, node.value.func.id, kwargs) + self.generic_visit(node) + + def visit_With(self, node: ast.With) -> None: + for item in node.items: + call = item.context_expr + if isinstance(call, ast.Call) and isinstance(call.func, ast.Name) and call.func.id == "DAG": + self._read_dag_kwargs(call) + self.generic_visit(node) + + def _read_dag_kwargs(self, call: ast.Call) -> None: + kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg} + self.dag_id = _literal_str(kwargs.get("dag_id")) + self.schedule_interval = _literal_str(kwargs.get("schedule_interval")) or _literal_str(kwargs.get("schedule")) + + def visit_Expr(self, node: ast.Expr) -> None: + # Capture `a >> b >> c` and `a << b` dependency chains. + if isinstance(node.value, ast.BinOp) and isinstance(node.value.op, (ast.RShift, ast.LShift)): + self._collect_shift_chain(node.value) + self.generic_visit(node) + + def _collect_shift_chain(self, binop: ast.BinOp) -> None: + names = _flatten_shift(binop) + if not names: + return + pairs = zip(names, names[1:]) + for left, right in pairs: + if isinstance(binop.op, ast.RShift): + self.edges.append((left, right)) + else: + self.edges.append((right, left)) + + +def _flatten_shift(node: ast.expr) -> list[str]: + """Flattens a chain of ``>>`` / ``<<`` Name nodes into an ordered list.""" + if isinstance(node, ast.Name): + return [node.id] + if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.RShift, ast.LShift)): + return _flatten_shift(node.left) + _flatten_shift(node.right) + return [] + + +def _literal_str(node: ast.expr | None) -> str | None: + """Returns the string value of a constant AST node, else None.""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + return None + + +def load_airflow_dag(dag_path: Path) -> Pipeline: + """Parses an Airflow DAG file into a flowx Pipeline IR. + + Args: + dag_path: Path to a ``.py`` DAG module. + + Returns: + A :class:`~flowx.models.ir.Pipeline` whose tasks are NotebookActivity + (for mapped operators) or PlaceholderActivity (for unmapped ones). + """ + source = Path(dag_path).read_text(encoding="utf-8") + module = ast.parse(source) + visitor = _DagVisitor(module) + visitor.visit(module) + functions = visitor.functions() + + # var -> task_key, and per-var dependency edges resolved to task_keys. + var_to_task_key = {var: _sanitize_task_key(task_id) for var, (task_id, _, _) in visitor.operators.items()} + upstreams: dict[str, list[str]] = {var: [] for var in visitor.operators} + for upstream_var, downstream_var in visitor.edges: + if downstream_var in upstreams and upstream_var in var_to_task_key: + upstreams[downstream_var].append(upstream_var) + + tasks: list[Activity] = [] + for var, (task_id, operator, kwargs) in visitor.operators.items(): + task_key = var_to_task_key[var] + depends_on = [Dependency(task_key=var_to_task_key[u]) for u in upstreams[var]] or None + tasks.append(_build_activity(task_id, task_key, operator, kwargs, functions, depends_on, source)) + + return Pipeline( + name=visitor.dag_id or Path(dag_path).stem, + tasks=tasks, + schedule=_schedule_from_interval(visitor.schedule_interval), + tags={"source": "airflow", "dag_id": visitor.dag_id or ""}, + ) + + +def discover_dags(source_path: Path) -> list[Path]: + """Returns the DAG ``.py`` files under *source_path*. + + Accepts either a single ``.py`` file or a directory (scanned recursively). + Files whose source contains no ``DAG(`` construct are skipped so helper + modules in a DAGs folder are not mistaken for DAG definitions. + """ + source_path = Path(source_path) + candidates = [source_path] if source_path.is_file() else sorted(source_path.rglob("*.py")) + dags: list[Path] = [] + for candidate in candidates: + if candidate.suffix != ".py": + continue + try: + text = candidate.read_text(encoding="utf-8") + except OSError: + continue + if "DAG(" in text or "@dag" in text: + dags.append(candidate) + return dags + + +def load_pipelines(source_path: Path, pipeline: str | None = None) -> list[Pipeline]: + """Loads every DAG under *source_path* into Pipeline IR. + + Args: + source_path: A DAG ``.py`` file or a directory of them. + pipeline: When set, keep only the pipeline whose name (dag_id) matches. + + Returns: + One :class:`~flowx.models.ir.Pipeline` per discovered DAG, filtered to + *pipeline* when provided. + """ + pipelines = [load_airflow_dag(dag_path) for dag_path in discover_dags(source_path)] + if pipeline is not None: + pipelines = [p for p in pipelines if p.name == pipeline] + return pipelines + + +def _build_activity( + task_id: str, + task_key: str, + operator: str, + kwargs: dict[str, ast.expr], + functions: dict[str, ast.FunctionDef], + depends_on: list[Dependency] | None, + source: str, +) -> Activity: + """Maps one Airflow operator to an Activity IR node.""" + if operator == "PythonOperator": + callable_node = kwargs.get("python_callable") + func = functions.get(callable_node.id) if isinstance(callable_node, ast.Name) else None + if func is not None: + return NotebookActivity( + name=task_id, + task_key=task_key, + depends_on=depends_on, + notebook_path=f"notebooks/{task_key}.py", + generated_source=_notebook_source_from_callable(func, source), + ) + if operator == "BashOperator": + command = _literal_str(kwargs.get("bash_command")) + if command is not None: + return NotebookActivity( + name=task_id, + task_key=task_key, + depends_on=depends_on, + notebook_path=f"notebooks/{task_key}.py", + generated_source=_notebook_source_from_bash(task_id, command), + ) + return PlaceholderActivity( + name=task_id, + task_key=task_key, + depends_on=depends_on, + original_type=operator, + comment=f"Airflow operator '{operator}' has no deterministic flowx mapping yet.", + ) diff --git a/tests/conftest.py b/tests/conftest.py index 3be5be4..b658d51 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,6 @@ def fixtures_dir(): @pytest.fixture def adf_definitions(): """Load all ADF definitions from the test fixtures directory.""" - from flowx.parser.adf_loader import load_adf_definitions + from flowx.sources.adf.loader import load_adf_definitions return load_adf_definitions(FIXTURES_DIR) diff --git a/tests/integration/test_adf_live.py b/tests/integration/test_adf_live.py index 107d73b..31ebef8 100644 --- a/tests/integration/test_adf_live.py +++ b/tests/integration/test_adf_live.py @@ -21,7 +21,7 @@ from flowx.bundler.dab_writer import write_bundle from flowx.preparer.workflow_preparer import PreparedWorkflow, prepare_workflow -from flowx.translator.engine import translate_pipeline +from flowx.sources.adf.translate import translate_pipeline SUBSCRIPTION = "00000000-0000-0000-0000-000000000000" RESOURCE_GROUP = "flowx-rg" @@ -138,7 +138,7 @@ def adf_export_dir(tmp_path_factory): @pytest.fixture(scope="module") def live_definitions(adf_export_dir): """Load all exported ADF definitions.""" - from flowx.parser.adf_loader import load_adf_definitions + from flowx.sources.adf.loader import load_adf_definitions return load_adf_definitions(adf_export_dir) diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py index f729c3a..1945974 100644 --- a/tests/integration/test_end_to_end.py +++ b/tests/integration/test_end_to_end.py @@ -20,9 +20,9 @@ PlaceholderActivity, SwitchActivity, ) -from flowx.parser.adf_loader import build_inventory from flowx.preparer.workflow_preparer import PreparedWorkflow, prepare_workflow -from flowx.translator.engine import translate_pipeline +from flowx.sources.adf.loader import build_inventory +from flowx.sources.adf.translate import translate_pipeline # --------------------------------------------------------------------------- # TestTranslateAllPipelines — simulates "translate all pipelines" diff --git a/tests/integration/test_golden_output.py b/tests/integration/test_golden_output.py index 2c235e4..9c35cf3 100644 --- a/tests/integration/test_golden_output.py +++ b/tests/integration/test_golden_output.py @@ -19,9 +19,9 @@ CopyActivity, SetVariableActivity, ) -from flowx.parser.adf_loader import load_adf_definitions from flowx.preparer.workflow_preparer import PreparedWorkflow, prepare_workflow -from flowx.translator.engine import translate_pipeline +from flowx.sources.adf.loader import load_adf_definitions +from flowx.sources.adf.translate import translate_pipeline # --------------------------------------------------------------------------- # Fixtures diff --git a/tests/integration/test_path_equivalence.py b/tests/integration/test_path_equivalence.py index f4b4c89..1944a58 100644 --- a/tests/integration/test_path_equivalence.py +++ b/tests/integration/test_path_equivalence.py @@ -10,9 +10,10 @@ import yaml from flowx.bundler.dab_writer import _pipeline_dict_to_workflow, write_bundle -from flowx.parser.adf_loader import load_adf_definitions +from flowx.ir_serde import pipeline_to_dict from flowx.preparer.workflow_preparer import prepare_workflow -from flowx.translator.engine import _pipeline_to_dict, translate_pipeline +from flowx.sources.adf.loader import load_adf_definitions +from flowx.sources.adf.translate import translate_pipeline from flowx.validate.bundle_invariants import check_bundle_dir, format_result FIXTURES_DIR = Path(__file__).parent.parent / "resources" / "json" @@ -36,7 +37,7 @@ def test_inprocess_and_report_paths_agree(name: str, tmp_path: Path) -> None: # Serialize the report BEFORE the in-process write (write_bundle mutates the # workflow it is given, not the IR, but serialize first to be safe). - report_dict = _pipeline_to_dict(report.pipeline) + report_dict = pipeline_to_dict(report.pipeline) in_process = tmp_path / "in_process" write_bundle(prepare_workflow(report.pipeline), in_process, catalog="c", schema="s") @@ -54,6 +55,6 @@ def test_generated_bundle_satisfies_invariants(name: str, tmp_path: Path) -> Non pipeline = next(p for p in _DEFS.pipelines if p.name == name) report = translate_pipeline(pipeline, _DEFS) out = tmp_path / "bundle" - write_bundle(_pipeline_dict_to_workflow(_pipeline_to_dict(report.pipeline)), out, catalog="c", schema="s") + write_bundle(_pipeline_dict_to_workflow(pipeline_to_dict(report.pipeline)), out, catalog="c", schema="s") result = check_bundle_dir(out) assert result.ok, format_result(result) diff --git a/tests/resources/airflow/orders_analytics_dag.py b/tests/resources/airflow/orders_analytics_dag.py new file mode 100644 index 0000000..caff4a1 --- /dev/null +++ b/tests/resources/airflow/orders_analytics_dag.py @@ -0,0 +1,46 @@ +"""Sample Airflow DAG used by the airflow-source spike. + +Representative of a common field pattern: a Python ingest step, a bash step, and +a Python publish step wired with >> dependencies under a cron schedule. Parsed +statically by flowx.sources.airflow.loader (no Airflow install required). +""" + +from datetime import datetime + +from airflow import DAG +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator + + +def ingest_orders(): + df = spark.read.json("s3://acme-orders/raw/") + df.write.mode("append").saveAsTable("main.analytics.raw_orders") + + +def publish_metrics(): + daily = spark.table("main.analytics.raw_orders").groupBy("order_date").count() + daily.write.mode("overwrite").saveAsTable("main.analytics.daily_order_metrics") + + +with DAG( + dag_id="orders_analytics", + schedule_interval="0 6 * * *", + start_date=datetime(2024, 1, 1), + catchup=False, +) as dag: + ingest = PythonOperator( + task_id="ingest_orders", + python_callable=ingest_orders, + ) + + transform = BashOperator( + task_id="transform_orders", + bash_command="python /opt/etl/transform_orders.py --date {{ ds }}", + ) + + publish = PythonOperator( + task_id="publish_metrics", + python_callable=publish_metrics, + ) + + ingest >> transform >> publish diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index f283c1a..2b4f734 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -94,7 +94,7 @@ def _query_delta_copy(name: str = "copy_query") -> CopyActivity: The query analysis fields the translator normally stamps are included here so the IR is shaped exactly as it would be after - ``flowx.translator.engine`` runs against this Copy. + ``flowx.sources.adf.translate`` runs against this Copy. """ return CopyActivity( **_make_base(name), @@ -502,7 +502,7 @@ def test_find_option_returns_pending_option(self): class TestSerializationRoundtrip: def test_configuration_survive_json_roundtrip(self): from flowx.bundler.dab_writer import pipeline_dict_to_ir - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict pipeline = Pipeline( name="p", tasks=[_delta_copy(), NotebookActivity(**_make_base("nb"), notebook_path="/Shared/x")] @@ -513,7 +513,7 @@ def test_configuration_survive_json_roundtrip(self): use_lakeflow_connectors="lakeflow_connect", ) stamped = apply_configuration(pipeline, prefs) - roundtripped, _ = pipeline_dict_to_ir(json.loads(json.dumps(_pipeline_to_dict(stamped), default=str))) + roundtripped, _ = pipeline_dict_to_ir(json.loads(json.dumps(pipeline_to_dict(stamped), default=str))) assert roundtripped.translation_configuration.copy_activity_paradigm is CopyActivityParadigm.SDP assert roundtripped.tasks[0].target_format == "sdp" assert roundtripped.tasks[0].use_lakeflow_connector is True @@ -588,7 +588,7 @@ def test_collected_omits_required_when_missing(self): class TestWorkspacePathsCli: def test_workspace_paths_detects_notebook_paths(self, tmp_path: Path): - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict pipeline = Pipeline( name="p", @@ -598,7 +598,7 @@ def test_workspace_paths_detects_notebook_paths(self, tmp_path: Path): ], ) report_path = tmp_path / "report.json" - report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) out = tmp_path / "ws.json" exit_code = adapter_cli_main(["workspace-paths", str(report_path), "--out", str(out)]) assert exit_code == 0 @@ -608,11 +608,11 @@ def test_workspace_paths_detects_notebook_paths(self, tmp_path: Path): assert payload["suggested_hosts"] == [] def test_workspace_paths_reports_no_auth_when_paths_empty(self, tmp_path: Path): - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict pipeline = Pipeline(name="p", tasks=[_delta_copy()]) report_path = tmp_path / "report.json" - report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) out = tmp_path / "ws.json" adapter_cli_main(["workspace-paths", str(report_path), "--out", str(out)]) payload = json.loads(out.read_text()) @@ -620,14 +620,14 @@ def test_workspace_paths_reports_no_auth_when_paths_empty(self, tmp_path: Path): assert payload["needs_auth"] is False def test_workspace_paths_suggests_host_from_databricks_linked_service(self, tmp_path: Path): - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict pipeline = Pipeline( name="p", tasks=[NotebookActivity(**_make_base("nb"), notebook_path="/Shared/team/x")], ) report_path = tmp_path / "report.json" - report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) source_dir = tmp_path / "source" (source_dir / "linked_services").mkdir(parents=True) (source_dir / "linked_services" / "LS_AzureDatabricks.json").write_text( @@ -668,11 +668,11 @@ def test_inputs_writes_to_file(self, tmp_path: Path): class TestCli: def test_inspect_emits_pending_options(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict pipeline = Pipeline(name="p", tasks=[_delta_copy()]) report_path = tmp_path / "report.json" - report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) exit_code = adapter_cli_main(["inspect", str(report_path)]) assert exit_code == 0 payload = json.loads(capsys.readouterr().out) @@ -681,11 +681,11 @@ def test_inspect_emits_pending_options(self, tmp_path: Path, capsys: pytest.Capt assert OPTION_COPY_ACTIVITY_PARADIGM in option_ids def test_modify_stamps_configuration(self, tmp_path: Path): - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict pipeline = Pipeline(name="p", tasks=[_delta_copy()]) report_path = tmp_path / "report.json" - report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) out_path = tmp_path / "modified.json" exit_code = adapter_cli_main( [ @@ -735,12 +735,12 @@ def test_materialize_lookup_from_csv_file(self, tmp_path: Path): assert rows == [{"table_name": "orders"}, {"table_name": "customers"}] def test_modify_threads_lookup_values_into_metadata_driven_motif(self, tmp_path: Path): - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict motif = _metadata_driven_motif() pipeline = Pipeline(name="p", tasks=[motif]) report_path = tmp_path / "report.json" - report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) out_path = tmp_path / "modified.json" exit_code = adapter_cli_main( [ @@ -767,11 +767,11 @@ def test_modify_threads_lookup_values_into_metadata_driven_motif(self, tmp_path: assert motif_task["lookup_values"] == [{"source_table": "orders"}] def test_modify_rejects_invalid_answer(self, tmp_path: Path): - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict pipeline = Pipeline(name="p", tasks=[_delta_copy()]) report_path = tmp_path / "report.json" - report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) out_path = tmp_path / "modified.json" exit_code = adapter_cli_main( ["modify", str(report_path), "--answer", "copy_activity_paradigm=yaml", "--out", str(out_path)] @@ -779,12 +779,12 @@ def test_modify_rejects_invalid_answer(self, tmp_path: Path): assert exit_code == 2 def test_modify_output_dir_convention_writes_work_and_metadata(self, tmp_path: Path): - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict pipeline = Pipeline(name="p", tasks=[_delta_copy()]) report_path = tmp_path / ".work" / "translation_report.json" report_path.parent.mkdir(parents=True) - report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) exit_code = adapter_cli_main( ["modify", str(report_path), "--output-dir", str(tmp_path), "--answer", "copy_activity_paradigm=sdp"] ) @@ -795,19 +795,19 @@ def test_modify_output_dir_convention_writes_work_and_metadata(self, tmp_path: P assert config == {"copy_activity_paradigm": "sdp"} def test_modify_requires_output_dir_or_out(self, tmp_path: Path): - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict pipeline = Pipeline(name="p", tasks=[_delta_copy()]) report_path = tmp_path / "report.json" - report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) exit_code = adapter_cli_main(["modify", str(report_path), "--answer", "copy_activity_paradigm=sdp"]) assert exit_code == 2 def test_inspect_emits_full_schema_with_show_when(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): """inspect returns the whole option tree at once; follow-ups carry a show_when condition the agent evaluates locally (no per-follow-up round trip).""" + from flowx.ir_serde import pipeline_to_dict from flowx.models.ir import CopyActivity, Dependency, WebActivity - from flowx.translator.engine import _pipeline_to_dict copy = CopyActivity(name="Load", task_key="load") notify = WebActivity( @@ -819,7 +819,7 @@ def test_inspect_emits_full_schema_with_show_when(self, tmp_path: Path, capsys: ) pipeline = Pipeline(name="p", tasks=[copy, notify]) report_path = tmp_path / "report.json" - report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) assert adapter_cli_main(["inspect", str(report_path)]) == 0 options = {o["option_id"]: o for o in json.loads(capsys.readouterr().out)["pipelines"][0]["options"]} @@ -835,11 +835,11 @@ def test_inspect_emits_full_schema_with_show_when(self, tmp_path: Path, capsys: assert [c["value"] for c in options["notify_destination"]["choices"]][0] == "keep" def test_inspect_rejects_malformed_answer(self, tmp_path: Path): - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict pipeline = Pipeline(name="p", tasks=[_delta_copy()]) report_path = tmp_path / "report.json" - report_path.write_text(json.dumps(_pipeline_to_dict(pipeline))) + report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) # Missing '=' -> validation error -> exit 2. assert adapter_cli_main(["inspect", str(report_path), "--answer", "no_equals_sign"]) == 2 diff --git a/tests/unit/test_adf_loader.py b/tests/unit/test_adf_loader.py index 8213742..5b30a3d 100644 --- a/tests/unit/test_adf_loader.py +++ b/tests/unit/test_adf_loader.py @@ -6,7 +6,7 @@ AdfDefinitions, TranslationStrategy, ) -from flowx.parser.adf_loader import ( +from flowx.sources.adf.loader import ( AGENTIC_TYPES, DETERMINISTIC_TYPES, _normalize_arm, diff --git a/tests/unit/test_dbt_factory_preparer.py b/tests/unit/test_dbt_factory_preparer.py new file mode 100644 index 0000000..caaf882 --- /dev/null +++ b/tests/unit/test_dbt_factory_preparer.py @@ -0,0 +1,130 @@ +"""Unit tests for the DbtFactoryActivity preparer (static + pydabs modes).""" + +from __future__ import annotations + +from flowx.models.ir import DbtFactoryActivity, Dependency, NotebookActivity, Pipeline +from flowx.preparer.workflow_preparer import prepare_activity, prepare_workflow + +_NODES = [ + {"task_key": "seed_codes", "command": "seed", "selector": "fqn:p.codes", "depends_on": []}, + {"task_key": "model_stg", "command": "run", "selector": "fqn:p.staging.stg", "depends_on": []}, + { + "task_key": "model_fct", + "command": "run", + "selector": "fqn:p.marts.fct", + "depends_on": ["model_stg", "seed_codes"], + }, + {"task_key": "test_stg", "command": "test", "selector": "fqn:p.staging.t", "depends_on": ["model_stg"]}, +] + + +def _dbt_activity(**overrides): + kwargs = dict( + name="dbt_transform", + task_key="dbt_transform", + project_dir=".", + profiles_dir="dbt_profiles", + target="dev", + nodes=_NODES, + render_mode="static", + ) + kwargs.update(overrides) + return DbtFactoryActivity(**kwargs) + + +def test_static_parent_task_is_run_job_hop(): + prepared = prepare_activity(_dbt_activity()) + assert "run_job_task" in prepared.task + assert prepared.task["run_job_task"]["job_id"] == "${resources.jobs.dbt_transform_dbt.id}" + + +def test_static_emits_inner_job_with_one_task_per_node(): + prepared = prepare_activity(_dbt_activity()) + assert len(prepared.inner_workflows) == 1 + inner = prepared.inner_workflows[0] + task_keys = {t["task_key"] for t in inner.tasks} + assert task_keys == {"seed_codes", "model_stg", "model_fct", "test_stg"} + + +def test_static_preserves_node_dependencies(): + prepared = prepare_activity(_dbt_activity()) + inner = prepared.inner_workflows[0] + fct = next(t for t in inner.tasks if t["task_key"] == "model_fct") + deps = {d["task_key"] for d in fct["depends_on"]} + assert deps == {"model_stg", "seed_codes"} + + +def test_static_node_task_carries_command_and_selector(): + prepared = prepare_activity(_dbt_activity()) + inner = prepared.inner_workflows[0] + test_task = next(t for t in inner.tasks if t["task_key"] == "test_stg") + params = test_task["notebook_task"]["base_parameters"] + assert params["dbt_command"] == "test" + assert params["dbt_select"] == "fqn:p.staging.t" + assert params["dbt_target"] == "dev" + + +def test_static_emits_single_shared_runner_notebook(): + prepared = prepare_activity(_dbt_activity()) + inner = prepared.inner_workflows[0] + runner_paths = [nb.relative_path for nb in inner.notebooks] + assert runner_paths == ["notebooks/run_dbt_command.py"] + # Every node task points at the one runner. + for task in inner.tasks: + assert task["notebook_task"]["notebook_path"] == "../src/notebooks/run_dbt_command.py" + + +def test_static_parent_hop_keeps_upstream_dependency(): + activity = _dbt_activity(depends_on=[Dependency(task_key="ingest")]) + prepared = prepare_activity(activity) + assert prepared.task["depends_on"] == [{"task_key": "ingest"}] + + +def test_pydabs_emits_hook_module_and_no_inner_job(): + prepared = prepare_activity(_dbt_activity(render_mode="pydabs", manifest_path="target/manifest.json")) + assert prepared.inner_workflows == [] + hook_paths = [nb.relative_path for nb in prepared.notebooks] + assert hook_paths == ["resources/dbt_transform_dbt_job.py"] + assert "load_resources" in prepared.notebooks[0].content + assert "run_job_task" in prepared.task + + +def test_pydabs_records_setup_task(): + prepared = prepare_activity(_dbt_activity(render_mode="pydabs")) + setup_types = {t.type for t in prepared.setup_tasks} + assert "pydabs_dbt_factory" in setup_types + + +def test_full_pipeline_wires_two_jobs(): + pipeline = Pipeline( + name="orders", + tasks=[ + NotebookActivity( + name="ingest", + task_key="ingest", + notebook_path="notebooks/ingest.py", + generated_source="# Databricks notebook source\nprint('x')\n", + ), + _dbt_activity(depends_on=[Dependency(task_key="ingest")]), + ], + ) + wf = prepare_workflow(pipeline) + parent_keys = {t["task_key"] for t in wf.tasks} + assert parent_keys == {"ingest", "dbt_transform"} + assert len(wf.inner_workflows) == 1 + assert wf.inner_workflows[0].name == "dbt_transform_dbt" + + +def test_survives_json_report_round_trip(): + # The convert->package phase boundary serialises the IR to translation_report.json. + # DbtFactoryActivity must serialise and rehydrate without losing its node list. + from flowx.bundler.dab_writer import pipeline_dict_to_ir + from flowx.ir_serde import pipeline_to_dict + from flowx.models.ir import DbtFactoryActivity + + pipeline = Pipeline(name="orders", tasks=[_dbt_activity()]) + rehydrated, _ = pipeline_dict_to_ir(pipeline_to_dict(pipeline)) + dbt = rehydrated.tasks[0] + assert isinstance(dbt, DbtFactoryActivity) + assert dbt.render_mode == "static" + assert {n["task_key"] for n in dbt.nodes} == {"seed_codes", "model_stg", "model_fct", "test_stg"} diff --git a/tests/unit/test_dbt_manifest.py b/tests/unit/test_dbt_manifest.py new file mode 100644 index 0000000..25a2af4 --- /dev/null +++ b/tests/unit/test_dbt_manifest.py @@ -0,0 +1,124 @@ +"""Unit tests for the dbt manifest reader (flowx.dbt.manifest). + +Uses synthetic manifests so the suite runs on a fresh clone with no dbt install. +""" + +from __future__ import annotations + +import pytest + +from flowx.dbt.manifest import explode_manifest + + +def _model(name, fqn, deps=None): + return { + "resource_type": "model", + "name": name, + "fqn": fqn, + "depends_on": {"nodes": deps or []}, + } + + +def _seed(name, fqn): + return {"resource_type": "seed", "name": name, "fqn": fqn, "depends_on": {"nodes": []}} + + +def _test(name, fqn, deps=None): + return {"resource_type": "test", "name": name, "fqn": fqn, "depends_on": {"nodes": deps or []}} + + +def _manifest(nodes, unit_tests=None): + return {"nodes": nodes, "unit_tests": unit_tests or {}} + + +def test_explodes_each_runnable_resource_type(): + manifest = _manifest( + { + "model.p.stg": _model("stg", ["p", "staging", "stg"]), + "seed.p.codes": _seed("codes", ["p", "codes"]), + "test.p.t": _test("t", ["p", "staging", "t"], deps=["model.p.stg"]), + } + ) + nodes = explode_manifest(manifest) + by_key = {n.task_key: n for n in nodes} + assert by_key["model_stg"].command == "run" + assert by_key["seed_codes"].command == "seed" + assert by_key["test_t"].command == "test" + + +def test_fqn_selector_built_from_components(): + manifest = _manifest({"model.p.stg": _model("stg", ["p", "staging", "stg"])}) + (node,) = explode_manifest(manifest) + assert node.selector == "fqn:p.staging.stg" + + +def test_dependency_edges_pruned_to_exploded_set(): + # The model depends on a source (not runnable) and another model (runnable). + manifest = _manifest( + { + "model.p.stg": _model("stg", ["p", "stg"], deps=["source.p.raw.raw_orders"]), + "model.p.fct": _model("fct", ["p", "fct"], deps=["model.p.stg", "source.p.raw.x"]), + } + ) + by_key = {n.task_key: n for n in explode_manifest(manifest)} + assert by_key["model_stg"].depends_on == [] # source edge dropped + assert by_key["model_fct"].depends_on == ["model_stg"] # source edge dropped, model kept + + +def test_non_runnable_resource_types_skipped(): + manifest = _manifest( + { + "model.p.stg": _model("stg", ["p", "stg"]), + "source.p.raw": {"resource_type": "source", "name": "raw", "fqn": ["p", "raw"]}, + "operation.p.hook": {"resource_type": "operation", "name": "hook", "fqn": ["p", "hook"]}, + } + ) + keys = {n.task_key for n in explode_manifest(manifest)} + assert keys == {"model_stg"} + + +def test_output_is_sorted_by_task_key(): + manifest = _manifest( + { + "model.p.zeta": _model("zeta", ["p", "zeta"]), + "model.p.alpha": _model("alpha", ["p", "alpha"]), + } + ) + keys = [n.task_key for n in explode_manifest(manifest)] + assert keys == sorted(keys) + + +def test_rejects_unit_tests_when_test_node_present(): + manifest = _manifest( + {"test.p.t": _test("t", ["p", "t"])}, + unit_tests={"unit_test.p.a": {}}, + ) + with pytest.raises(ValueError, match="unit_test"): + explode_manifest(manifest) + + +def test_allows_unit_tests_when_no_test_node(): + # A manifest with unit_tests but no exploded test node is fine (nothing dropped). + manifest = _manifest( + {"model.p.stg": _model("stg", ["p", "stg"])}, + unit_tests={"unit_test.p.a": {}}, + ) + explode_manifest(manifest) # no raise + + +def test_rejects_unsafe_fqn_characters(): + manifest = _manifest({"model.p.bad": _model("bad", ["p", "foo,bar"])}) + with pytest.raises(ValueError, match="Unsafe fqn"): + explode_manifest(manifest) + + +def test_rejects_task_key_collision(): + # Distinct unique_ids whose (resource_type, name) sanitize to one key. + manifest = _manifest( + { + "model.p.a": _model("foo bar", ["p", "a"]), + "model.q.b": _model("foo_bar", ["q", "b"]), + } + ) + with pytest.raises(ValueError, match="collide"): + explode_manifest(manifest) diff --git a/tests/unit/test_ir_rewriter.py b/tests/unit/test_ir_rewriter.py index bdadef6..4cf253a 100644 --- a/tests/unit/test_ir_rewriter.py +++ b/tests/unit/test_ir_rewriter.py @@ -15,7 +15,7 @@ SwitchCase, WebActivity, ) -from flowx.parser.ir_rewriter import rewrite_pipeline_expressions +from flowx.sources.adf.ir_rewriter import rewrite_pipeline_expressions def _base(task_key: str, name: str | None = None) -> dict[str, object]: diff --git a/tests/unit/test_merge_agentic.py b/tests/unit/test_merge_agentic.py index 9723cbe..f3b8888 100644 --- a/tests/unit/test_merge_agentic.py +++ b/tests/unit/test_merge_agentic.py @@ -5,7 +5,7 @@ import json from pathlib import Path -from flowx.translator.engine import merge_agentic_results +from flowx.ir_serde import merge_agentic_results def _write(path: Path, obj: object) -> None: diff --git a/tests/unit/test_preparers.py b/tests/unit/test_preparers.py index f206c73..f912f2b 100644 --- a/tests/unit/test_preparers.py +++ b/tests/unit/test_preparers.py @@ -889,7 +889,7 @@ def test_run_job_round_trips_through_translation_report_json(self, tmp_path): import yaml from flowx.bundler.dab_writer import _load_report, write_bundle - from flowx.translator.engine import _activity_to_dict, _pipeline_to_dict + from flowx.ir_serde import activity_to_dict, pipeline_to_dict run_job = RunJobActivity( **_make_base("Nightly Aggregator", "nightly_aggregator"), @@ -898,11 +898,11 @@ def test_run_job_round_trips_through_translation_report_json(self, tmp_path): job_parameters={"window_start": "2024-01-01", "table": "orders"}, ) pipeline = Pipeline(name="rj_pipeline", tasks=[run_job]) - pipeline_dict = _pipeline_to_dict(pipeline) + pipeline_dict = pipeline_to_dict(pipeline) # Sanity: serialiser must include job_parameters. run_job_dict = next(t for t in pipeline_dict["tasks"] if t["task_key"] == "nightly_aggregator") assert run_job_dict["job_parameters"] == {"window_start": "2024-01-01", "table": "orders"} - assert _activity_to_dict(run_job)["job_parameters"] == run_job.job_parameters + assert activity_to_dict(run_job)["job_parameters"] == run_job.job_parameters report_path = tmp_path / "rj.json" report_path.write_text(json.dumps(pipeline_dict)) diff --git a/tests/unit/test_profile_report.py b/tests/unit/test_profile_report.py index e7cc7ef..4112046 100644 --- a/tests/unit/test_profile_report.py +++ b/tests/unit/test_profile_report.py @@ -15,7 +15,7 @@ AdfLinkedServiceReference, AdfPipeline, ) -from flowx.parser.adf_loader import ( +from flowx.sources.adf.loader import ( _activity_category, _complexity_score, _tshirt_size, diff --git a/tests/unit/test_query_analysis.py b/tests/unit/test_query_analysis.py index 90d0027..d22e5ac 100644 --- a/tests/unit/test_query_analysis.py +++ b/tests/unit/test_query_analysis.py @@ -4,7 +4,7 @@ import pytest -from flowx.translator.query_analysis import QueryAnalysis, analyze_copy_query, dialect_for_source_type +from flowx.sources.adf.query_analysis import QueryAnalysis, analyze_copy_query, dialect_for_source_type class TestParseabilityRejections: diff --git a/tests/unit/test_resolve_field.py b/tests/unit/test_resolve_field.py index c965066..9c19f2c 100644 --- a/tests/unit/test_resolve_field.py +++ b/tests/unit/test_resolve_field.py @@ -5,7 +5,7 @@ from types import MappingProxyType from flowx.models.ir import TranslationContext -from flowx.translator.activity_translators.resolve import ( +from flowx.sources.adf.translators.resolve import ( resolve_dict_values, resolve_field, resolve_field_int, diff --git a/tests/unit/test_source_router.py b/tests/unit/test_source_router.py new file mode 100644 index 0000000..a8921f3 --- /dev/null +++ b/tests/unit/test_source_router.py @@ -0,0 +1,98 @@ +"""Unit tests for the source registry and the adapter's --source routing.""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +import pytest + +from flowx.adapter.__main__ import _run_phase, _split_source +from flowx.sources import available_sources, get_source + +_DAG_FIXTURE = Path(__file__).resolve().parents[1] / "resources" / "airflow" / "orders_analytics_dag.py" + + +# -------------------------------------------------------------------------------------- +# Registry +# -------------------------------------------------------------------------------------- + + +def test_registry_lists_adf_and_airflow(): + assert set(available_sources()) >= {"adf", "airflow"} + + +def test_get_source_returns_phase_modules(): + airflow = get_source("airflow") + assert airflow.discover_module == "flowx.sources.airflow.discover" + assert airflow.convert_module == "flowx.sources.airflow.convert" + + +def test_get_unknown_source_raises(): + with pytest.raises(KeyError, match="Unknown source"): + get_source("oozie") + + +# -------------------------------------------------------------------------------------- +# --source extraction +# -------------------------------------------------------------------------------------- + + +def test_split_source_absent_is_none(): + assert _split_source([]) == (None, []) + assert _split_source(["--source-dir", "x"]) == (None, ["--source-dir", "x"]) + + +def test_split_source_space_form(): + assert _split_source(["--source", "airflow", "--source-dir", "x"]) == ("airflow", ["--source-dir", "x"]) + + +def test_split_source_equals_form(): + assert _split_source(["--source=airflow", "--output-dir", "o"]) == ("airflow", ["--output-dir", "o"]) + + +# -------------------------------------------------------------------------------------- +# Routing (end-to-end through the adapter phase runner, in-process) +# -------------------------------------------------------------------------------------- + + +def test_unknown_source_returns_exit_2(): + rc = _run_phase("discover", ["--source", "nope", "--source-path", str(_DAG_FIXTURE)]) + assert rc == 2 + + +def test_missing_source_is_required_for_discover(): + # No --source: discover/convert must error (there is no default source). + rc = _run_phase("discover", ["--source-path", str(_DAG_FIXTURE)]) + assert rc == 2 + + +def test_missing_source_is_required_for_convert(): + rc = _run_phase("convert", ["--source-path", str(_DAG_FIXTURE)]) + assert rc == 2 + + +def test_airflow_discover_then_convert_route(): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + rc = _run_phase( + "discover", ["--source", "airflow", "--source-path", str(_DAG_FIXTURE), "--output-dir", str(out)] + ) + assert rc == 0 + assert (out / "metadata" / "inventory.json").exists() + rc = _run_phase( + "convert", ["--source", "airflow", "--source-path", str(_DAG_FIXTURE), "--output-dir", str(out)] + ) + assert rc == 0 + assert (out / ".work" / "translation_report.json").exists() + + +def test_package_is_source_independent(): + # package ignores --source and routes to the shared bundler; drive the whole chain. + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + _run_phase("convert", ["--source", "airflow", "--source-path", str(_DAG_FIXTURE), "--output-dir", str(out)]) + rc = _run_phase("package", ["--output-dir", str(out)]) + assert rc == 0 + assert (out / "databricks.yml").exists() + assert list((out / "resources").glob("*.yml")) diff --git a/tests/unit/test_translators.py b/tests/unit/test_translators.py index 92d52d4..d253846 100644 --- a/tests/unit/test_translators.py +++ b/tests/unit/test_translators.py @@ -34,7 +34,7 @@ WaitActivity, WebActivity, ) -from flowx.translator.engine import translate_pipeline +from flowx.sources.adf.translate import translate_pipeline _EMPTY_DEFS = AdfDefinitions(pipelines=[], datasets={}, linked_services={}, triggers=[]) @@ -93,7 +93,7 @@ def _make_activity( class TestCopyTranslator: def test_translate_copy_basic(self): - from flowx.translator.activity_translators.copy import translate + from flowx.sources.adf.translators.copy import translate activity = _make_activity( "Copy Data", @@ -111,7 +111,7 @@ def test_translate_copy_basic(self): assert result.sink_properties["writeBatchSize"] == 10000 def test_translate_copy_with_column_mapping(self): - from flowx.translator.activity_translators.copy import translate + from flowx.sources.adf.translators.copy import translate activity = _make_activity( "Copy Mapped", @@ -142,7 +142,7 @@ def test_translate_copy_with_column_mapping(self): assert result.column_mapping[1]["sink_name"] == "full_name" def test_translate_copy_empty_type_properties(self): - from flowx.translator.activity_translators.copy import translate + from flowx.sources.adf.translators.copy import translate activity = _make_activity("Empty Copy", "Copy", {}) result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) @@ -153,7 +153,7 @@ def test_translate_copy_empty_type_properties(self): class TestNotebookTranslator: def test_translate_notebook_basic(self): - from flowx.translator.activity_translators.notebook import translate + from flowx.sources.adf.translators.notebook import translate activity = _make_activity( "Run Notebook", @@ -166,7 +166,7 @@ def test_translate_notebook_basic(self): assert result.base_parameters == {"env": "dev"} def test_translate_notebook_no_params(self): - from flowx.translator.activity_translators.notebook import translate + from flowx.sources.adf.translators.notebook import translate activity = _make_activity( "Run Notebook", @@ -180,7 +180,7 @@ def test_translate_notebook_no_params(self): def test_translate_notebook_resolves_library_with_globals(self): """C-01 (NB-ITER2-1, LSC2-004): @concat of literals collapses to a literal jar path so the library install succeeds.""" - from flowx.translator.activity_translators.notebook import translate + from flowx.sources.adf.translators.notebook import translate activity = _make_activity( "Run NB", @@ -203,7 +203,7 @@ def test_translate_notebook_resolves_library_with_globals(self): def test_translate_notebook_resolves_pipeline_param_in_library(self): """Library entry referencing a single pipeline parameter resolves to a literal.""" - from flowx.translator.activity_translators.notebook import translate + from flowx.sources.adf.translators.notebook import translate activity = _make_activity( "Run NB", @@ -223,7 +223,7 @@ def test_translate_notebook_resolves_pipeline_param_in_library(self): assert result.libraries == [{"jar": "/Volumes/my.jar"}] def test_translate_notebook_passes_libraries_through(self): - from flowx.translator.activity_translators.notebook import translate + from flowx.sources.adf.translators.notebook import translate libraries = [ {"jar": "dbfs:/libs/util.jar"}, @@ -244,7 +244,7 @@ def test_translate_notebook_passes_libraries_through(self): def test_translate_notebook_dynamic_path_marks_unresolved(self): """C-28 (NB-ITER4-001): an expression notebookPath is captured as ``notebook_path_unresolved`` so the preparer emits a dispatch stub.""" - from flowx.translator.activity_translators.notebook import translate + from flowx.sources.adf.translators.notebook import translate activity = _make_activity( "Dispatch", @@ -267,7 +267,7 @@ def test_translate_notebook_unresolved_library_captured(self): """C-30 (NB-ITER4-003): library jar/whl entries whose @concat references a missing globalParameter surface as ``unresolved_libraries`` so SETUP.md can flag them.""" - from flowx.translator.activity_translators.notebook import translate + from flowx.sources.adf.translators.notebook import translate activity = _make_activity( "Run NB", @@ -288,7 +288,7 @@ def test_translate_notebook_unresolved_library_captured(self): assert "proj4jLibFileName" in entry["missing"] def test_translate_notebook_captures_utcnow_approximation(self): - from flowx.translator.activity_translators.notebook import translate + from flowx.sources.adf.translators.notebook import translate activity = _make_activity( "Score", @@ -316,7 +316,7 @@ def test_translate_notebook_captures_utcnow_approximation(self): class TestCommonAttributes: def test_existing_cluster_id_extracted_from_linked_service(self): from flowx.models.adf_ast import AdfLinkedService - from flowx.translator.engine import _build_base_kwargs + from flowx.sources.adf.translate import _build_base_kwargs linked_service = AdfLinkedService( name="AzureDatabricks_LS", @@ -345,7 +345,7 @@ def test_existing_cluster_id_extracted_from_linked_service(self): def test_existing_cluster_id_none_when_linked_service_uses_new_cluster(self): from flowx.models.adf_ast import AdfLinkedService - from flowx.translator.engine import _build_base_kwargs + from flowx.sources.adf.translate import _build_base_kwargs linked_service = AdfLinkedService( name="AzureDatabricks_LS", @@ -375,7 +375,7 @@ def test_existing_cluster_id_none_when_linked_service_uses_new_cluster(self): def test_linked_service_parameter_overrides_cluster_version(self): """Change linked-service-parameter-resolution (P0): NB-4, LSC-001.""" from flowx.models.adf_ast import AdfLinkedService - from flowx.translator.engine import _build_base_kwargs + from flowx.sources.adf.translate import _build_base_kwargs linked_service = AdfLinkedService( name="APP0001_ls_databricks", @@ -419,26 +419,26 @@ def test_linked_service_parameter_overrides_cluster_version(self): def test_parameter_default_coerces_bool_string_to_real_bool(self): """Change expression-resolver-bool-and-numeric-coercion (P1): VAR-006.""" - from flowx.translator.engine import _coerce_parameter_default + from flowx.sources.adf.translate import _coerce_parameter_default assert _coerce_parameter_default("false", "Bool") is False assert _coerce_parameter_default("True", "Bool") is True assert _coerce_parameter_default("FALSE", "boolean") is False def test_parameter_default_coerces_int_string_to_int(self): - from flowx.translator.engine import _coerce_parameter_default + from flowx.sources.adf.translate import _coerce_parameter_default assert _coerce_parameter_default("42", "Int") == 42 assert _coerce_parameter_default(42, "Int") == 42 def test_parameter_default_string_left_alone(self): - from flowx.translator.engine import _coerce_parameter_default + from flowx.sources.adf.translate import _coerce_parameter_default assert _coerce_parameter_default("hello", "String") == "hello" def test_dependency_multi_condition_succeeded_and_failed_maps_to_completed(self): """Change dependency-multi-condition-mapping (P1): CF-004.""" - from flowx.translator.engine import _map_dependency_conditions + from flowx.sources.adf.translate import _map_dependency_conditions assert _map_dependency_conditions(["Succeeded"]) == "Succeeded" assert _map_dependency_conditions(["Failed"]) == "Failed" @@ -456,7 +456,7 @@ def test_ls_param_expression_wrapper_unwrapped_in_custom_tags(self): """C-02 (NB-ITER2-2 / LSC2-003): Expression-dict-wrapped LS params must collapse to plain scalars in cluster fields like custom_tags.""" from flowx.models.adf_ast import AdfLinkedService - from flowx.translator.engine import _build_base_kwargs + from flowx.sources.adf.translate import _build_base_kwargs linked_service = AdfLinkedService( name="LS", @@ -505,7 +505,7 @@ def test_ls_param_resolved_against_factory_global_parameters(self): that reference @pipeline().globalParameters.X must collapse to the factory-provided literal so cluster.spark_version is a real DBR.""" from flowx.models.adf_ast import AdfLinkedService - from flowx.translator.engine import _build_base_kwargs + from flowx.sources.adf.translate import _build_base_kwargs linked_service = AdfLinkedService( name="LS", @@ -555,7 +555,7 @@ def test_ls_param_resolved_against_pipeline_parameters_as_dab_ref(self): param values that reference @pipeline().parameters.X must collapse to {{job.parameters.X}} (a dab_ref), valid in custom_tags map values.""" from flowx.models.adf_ast import AdfLinkedService - from flowx.translator.engine import _build_base_kwargs + from flowx.sources.adf.translate import _build_base_kwargs linked_service = AdfLinkedService( name="LS", @@ -606,7 +606,7 @@ def test_ls_param_resolved_against_pipeline_parameters_as_dab_ref(self): def test_notebook_library_resolves_pipeline_param_dab_ref(self): """C-13 (NB-ITER3-004): a jar path referencing @pipeline().parameters.X collapses to {{job.parameters.X}} in the emitted library entry.""" - from flowx.translator.activity_translators.notebook import translate + from flowx.sources.adf.translators.notebook import translate activity = _make_activity( "Run NB", @@ -626,7 +626,7 @@ def test_notebook_library_resolves_pipeline_param_dab_ref(self): def test_extended_cluster_fields_propagated(self): """Change linked-service-cluster-field-coverage (P1): NB-3, LSC-003.""" from flowx.models.adf_ast import AdfLinkedService - from flowx.translator.engine import _build_base_kwargs + from flowx.sources.adf.translate import _build_base_kwargs linked_service = AdfLinkedService( name="LS", @@ -669,7 +669,7 @@ def test_extended_cluster_fields_propagated(self): class TestSparkJarTranslator: def test_translate_spark_jar(self): - from flowx.translator.activity_translators.spark_jar import translate + from flowx.sources.adf.translators.spark_jar import translate activity = _make_activity( "Run Jar", @@ -689,7 +689,7 @@ def test_translate_spark_jar(self): class TestSparkPythonTranslator: def test_translate_spark_python(self): - from flowx.translator.activity_translators.spark_python import translate + from flowx.sources.adf.translators.spark_python import translate activity = _make_activity( "Run Python", @@ -702,7 +702,7 @@ def test_translate_spark_python(self): assert result.parameters == ["--mode", "batch"] def test_translate_spark_python_passes_libraries_through(self): - from flowx.translator.activity_translators.spark_python import translate + from flowx.sources.adf.translators.spark_python import translate libraries = [ {"egg": "dbfs:/libs/util.egg"}, @@ -720,7 +720,7 @@ def test_translate_spark_python_passes_libraries_through(self): class TestLookupTranslator: def test_translate_lookup_first_row(self): - from flowx.translator.activity_translators.lookup import translate + from flowx.sources.adf.translators.lookup import translate activity = _make_activity( "Lookup Config", @@ -737,7 +737,7 @@ def test_translate_lookup_first_row(self): assert result.source_query == "SELECT TOP 1 * FROM config" def test_translate_lookup_all_rows(self): - from flowx.translator.activity_translators.lookup import translate + from flowx.sources.adf.translators.lookup import translate activity = _make_activity( "Lookup All", @@ -754,7 +754,7 @@ def test_translate_lookup_all_rows(self): def test_translate_lookup_resolves_json_file_dataset(self): """Change lookup-file-dataset-support (P0).""" from flowx.models.adf_ast import AdfDataset - from flowx.translator.activity_translators.lookup import translate + from flowx.sources.adf.translators.lookup import translate json_dataset = AdfDataset( name="ConfigDataset", @@ -806,7 +806,7 @@ def test_translate_lookup_substitutes_dataset_parameter_refs(self): ``dataset().X`` substitutes the dataset reference's parameter bindings so the baked path carries no literal ``dataset(`` expression.""" from flowx.models.adf_ast import AdfDataset - from flowx.translator.activity_translators.lookup import translate + from flowx.sources.adf.translators.lookup import translate ds = AdfDataset( name="arq_ds", @@ -866,7 +866,7 @@ def test_lookup_resolves_dataset_case_insensitively(self): """ADF identifiers are case-insensitive; a pipeline referencing 'app0001_a_ds_conf_json' must resolve dataset 'APP0001_a_ds_conf_json'.""" from flowx.models.adf_ast import AdfDataset, AdfLinkedService - from flowx.translator.activity_translators.lookup import translate + from flowx.sources.adf.translators.lookup import translate ds = AdfDataset( name="APP0001_a_ds_conf_json", @@ -947,7 +947,7 @@ def test_generated_file_lookup_notebook_uses_abfss_path(self): class TestWebActivityTranslator: def test_translate_web_activity_get(self): - from flowx.translator.activity_translators.web_activity import translate + from flowx.sources.adf.translators.web_activity import translate activity = _make_activity( "Call API", @@ -960,7 +960,7 @@ def test_translate_web_activity_get(self): assert result.method == "GET" def test_translate_web_activity_post(self): - from flowx.translator.activity_translators.web_activity import translate + from flowx.sources.adf.translators.web_activity import translate activity = _make_activity( "Post Data", @@ -983,7 +983,7 @@ def test_translate_web_activity_post(self): class TestDeleteTranslator: def test_translate_delete(self): - from flowx.translator.activity_translators.delete import translate + from flowx.sources.adf.translators.delete import translate activity = _make_activity( "Delete Files", @@ -999,7 +999,7 @@ def test_translate_delete(self): class TestExecutePipelineTranslator: def test_translate_execute_pipeline(self): - from flowx.translator.activity_translators.execute_pipeline import translate + from flowx.sources.adf.translators.execute_pipeline import translate activity = _make_activity( "Run Child", @@ -1021,7 +1021,7 @@ def test_translate_execute_pipeline_drops_notebook_code_parameters(self): to notebook_code (e.g. @concat('x', pipeline().parameters.Y)) must NOT ride through as a literal Python source string -- it's dropped from the parameters dict and surfaced via parameter_approximations.""" - from flowx.translator.activity_translators.execute_pipeline import translate + from flowx.sources.adf.translators.execute_pipeline import translate activity = _make_activity( "Run Child", @@ -1050,7 +1050,7 @@ def test_translate_execute_pipeline_drops_notebook_code_parameters(self): class TestDatabricksJobTranslator: def test_translate_databricks_job(self): - from flowx.translator.activity_translators.databricks_job import translate + from flowx.sources.adf.translators.databricks_job import translate activity = _make_activity( "Run Job", @@ -1065,7 +1065,7 @@ def test_translate_databricks_job(self): class TestWaitTranslator: def test_translate_wait(self): - from flowx.translator.activity_translators.wait import translate + from flowx.sources.adf.translators.wait import translate activity = _make_activity( "Pause", @@ -1077,7 +1077,7 @@ def test_translate_wait(self): assert result.wait_time_seconds == 60 def test_translate_wait_defaults_to_zero(self): - from flowx.translator.activity_translators.wait import translate + from flowx.sources.adf.translators.wait import translate activity = _make_activity("Pause", "Wait", {}) result = translate(activity, _base_kwargs(), _context(), _EMPTY_DEFS) @@ -1087,7 +1087,7 @@ def test_translate_wait_defaults_to_zero(self): class TestFilterTranslator: def test_translate_filter(self): - from flowx.translator.activity_translators.filter import translate + from flowx.sources.adf.translators.filter import translate activity = _make_activity( "Filter Items", @@ -1104,7 +1104,7 @@ def test_translate_filter(self): def test_translate_filter_lowers_simple_condition(self): """``@equals(item().X, 'Y')`` lowers to a Python expression with item.get(X).""" - from flowx.translator.activity_translators.filter import translate + from flowx.sources.adf.translators.filter import translate activity = _make_activity( "Filter Active", @@ -1121,7 +1121,7 @@ def test_translate_filter_lowers_simple_condition(self): def test_translate_filter_falls_back_to_placeholder_for_unresolvable(self): """A condition that doesn't lower cleanly leaves condition_code=None.""" - from flowx.translator.activity_translators.filter import translate + from flowx.sources.adf.translators.filter import translate activity = _make_activity( "Filter Mystery", @@ -1137,7 +1137,7 @@ def test_translate_filter_falls_back_to_placeholder_for_unresolvable(self): class TestForEachTranslator: def test_translate_foreach_basic(self): - from flowx.translator.activity_translators.for_each import translate + from flowx.sources.adf.translators.for_each import translate inner_activity = _make_activity( "InnerCopy", "Copy", {"source": {"type": "BlobSource"}, "sink": {"type": "DeltaSink"}} @@ -1159,7 +1159,7 @@ def test_translate_foreach_basic(self): assert result.concurrency == 5 def test_translate_foreach_sequential(self): - from flowx.translator.activity_translators.for_each import translate + from flowx.sources.adf.translators.for_each import translate inner_activity = _make_activity("InnerWait", "Wait", {"waitTimeInSeconds": 1}) activity = _make_activity( @@ -1176,7 +1176,7 @@ def test_translate_foreach_propagates_globals_to_child_context(self): """C-13 (NB-ITER3-001 / CF3-002 / LSC3-004): ForEach child context must carry global_parameters and linked_service_parameters so inner notebooks resolve @pipeline().globalParameters.X to literals.""" - from flowx.translator.activity_translators.for_each import translate + from flowx.sources.adf.translators.for_each import translate # Inner notebook whose library jar references a global parameter. inner_activity = _make_activity( @@ -1197,7 +1197,7 @@ def test_translate_foreach_propagates_globals_to_child_context(self): # The parent context carries the global parameter the inner notebook # needs. We use the real notebook translator inside our mock callback # so the inner activity is processed exactly as the engine would. - from flowx.translator.activity_translators.notebook import translate as translate_nb + from flowx.sources.adf.translators.notebook import translate as translate_nb def _mock_translate(activities, ctx, defs): results: list[Any] = [] @@ -1225,7 +1225,7 @@ def _mock_translate(activities, ctx, defs): class TestIfConditionTranslator: def test_translate_if_condition_equals(self): - from flowx.translator.activity_translators.if_condition import translate + from flowx.sources.adf.translators.if_condition import translate true_act = _make_activity("TrueAct", "Wait", {"waitTimeInSeconds": 1}) false_act = _make_activity("FalseAct", "Wait", {"waitTimeInSeconds": 2}) @@ -1257,7 +1257,7 @@ def _mock_translate(activities, context, definitions): assert len(result.if_false_activities) == 1 def test_translate_if_condition_greater(self): - from flowx.translator.activity_translators.if_condition import translate + from flowx.sources.adf.translators.if_condition import translate activity = _make_activity( "Check Count", @@ -1273,7 +1273,7 @@ def test_translate_if_condition_greater(self): def test_translate_if_condition_empty_bridges_via_notebook(self): """C-07 (CF-iter2-001 / VAREX-003): @empty(...) operand routes through a bridge SetVariable task rather than shipping as a raw ADF expression.""" - from flowx.translator.activity_translators.if_condition import translate + from flowx.sources.adf.translators.if_condition import translate activity = _make_activity( "Branch", @@ -1299,7 +1299,7 @@ def test_translate_if_condition_boolean_variable_uses_lowercase_false(self): """C-32 (CF4-002): the truthy fallback path emits ``right='false'`` (not ``'0'``) when the operand is a known-Boolean variable, since C-21 SetVariable now writes lowercase ``'true'/'false'`` strings.""" - from flowx.translator.activity_translators.if_condition import translate + from flowx.sources.adf.translators.if_condition import translate # Seed the context with a Boolean default-valued variable so the # truthy fallback knows the operand renders as 'true'/'false'. @@ -1321,7 +1321,7 @@ def test_translate_if_condition_boolean_variable_by_declared_type(self): init task never populates variable_value_cache as a dab_ref, so the IfCondition fallback must fall back to the declared type and still emit ``right='false'`` (not the always-true ``'0'``).""" - from flowx.translator.activity_translators.if_condition import translate + from flowx.sources.adf.translators.if_condition import translate # No dab_ref value cached — only the declared Boolean type is known. ctx = _context().with_variable_types({"continue": "Boolean"}) @@ -1342,7 +1342,7 @@ def test_translate_if_condition_boolean_variable_bridges_when_default_literal_kn rather than left as a parent-job task-value ref the bundler would blank. This keeps the operand local so an inner-ForEach condition survives the dangling-ref safety net.""" - from flowx.translator.activity_translators.if_condition import translate + from flowx.sources.adf.translators.if_condition import translate # Declared Boolean type AND a seeded literal default -> bridge. ctx = _context().with_variable_types({"continue": "Boolean"}, default_literals={"continue": "true"}) @@ -1362,7 +1362,7 @@ def test_translate_if_condition_not_of_function_uses_false_right(self): """C-15 (CF3-003 / VAREX3-004): @not() produces a bridge task value compared against 'False', not '' or '0', so the condition can actually evaluate to FALSE against the Python bool the bridge writes.""" - from flowx.translator.activity_translators.if_condition import translate + from flowx.sources.adf.translators.if_condition import translate activity = _make_activity( "Branch", @@ -1388,7 +1388,7 @@ def test_translate_if_condition_truthy_fallback_bridges_with_false_right(self): right='False' when the resolved operand is a bridge placeholder. Previously emitted right='0', which the bridge's Python bool output can never satisfy.""" - from flowx.translator.activity_translators.if_condition import translate + from flowx.sources.adf.translators.if_condition import translate # An expression with a function call that bridges (e.g. @toUpper). activity = _make_activity( @@ -1441,7 +1441,7 @@ def test_prepare_if_condition_emits_bridge_task(self): class TestSetVariableTranslator: def test_translate_set_variable_literal(self): - from flowx.translator.activity_translators.set_variable import translate + from flowx.sources.adf.translators.set_variable import translate activity = _make_activity( "Set Status", @@ -1461,7 +1461,7 @@ def test_translate_set_variable_return_value_pairs_resolves_inner(self): """C-42 (VAREX5-001): a Set Pipeline Return Value list-of-pairs value whose inner expression references a resolvable variable lowers to a dab_ref task-value reference instead of being stringified and blanked.""" - from flowx.translator.activity_translators.set_variable import translate + from flowx.sources.adf.translators.set_variable import translate # Seed the referenced variable so @variables('executionOutputs') # resolves to its setter task value. @@ -1488,7 +1488,7 @@ def test_translate_set_variable_return_value_pairs_resolves_inner(self): assert "{{tasks." in result.variable_value def test_translate_set_variable_utcnow(self): - from flowx.translator.activity_translators.set_variable import translate + from flowx.sources.adf.translators.set_variable import translate # ``utcNow('yyyy-MM-dd')`` now maps to a DAB dynamic value, so the # SetVariable result is dab_ref rather than notebook_code. @@ -1506,7 +1506,7 @@ def test_translate_set_variable_split_subscript_lowers_to_notebook_code(self): """C-33 (VAREX4-001): ``split(...)[N]`` previously left value_kind stamped as 'literal' with the raw @concat text; now it lowers to notebook_code so the SetVariable notebook computes the value.""" - from flowx.translator.activity_translators.set_variable import translate + from flowx.sources.adf.translators.set_variable import translate activity = _make_activity( "SetPart", @@ -1530,7 +1530,7 @@ def test_translate_set_variable_unresolved_expression_blanks_value(self): cannot lower no longer ships as value_kind='literal' with the raw @-expression. The value is blanked, value_kind='unresolved', and raw_expression captures the original text for SETUP.md.""" - from flowx.translator.activity_translators.set_variable import translate + from flowx.sources.adf.translators.set_variable import translate activity = _make_activity( "SetX", @@ -1551,7 +1551,7 @@ def test_translate_set_variable_unresolved_expression_blanks_value(self): assert result.raw_expression == "@foo(pipeline().parameters.bar)" def test_translate_set_variable_utcnow_unknown_format(self): - from flowx.translator.activity_translators.set_variable import translate + from flowx.sources.adf.translators.set_variable import translate # Unrecognised format falls back to the legacy notebook_code path. activity = _make_activity( @@ -1567,7 +1567,7 @@ def test_translate_set_variable_utcnow_unknown_format(self): assert "datetime" in result.notebook_imports[0] def test_translate_set_variable_pipeline_param(self): - from flowx.translator.activity_translators.set_variable import translate + from flowx.sources.adf.translators.set_variable import translate activity = _make_activity( "Set Env", @@ -1583,7 +1583,7 @@ def test_translate_set_variable_pipeline_param(self): class TestAppendVariableTranslator: def test_translate_append_variable(self): - from flowx.translator.activity_translators.append_variable import translate + from flowx.sources.adf.translators.append_variable import translate activity = _make_activity( "Append Log", @@ -1600,7 +1600,7 @@ def test_translate_append_variable(self): class TestSwitchTranslator: def test_translate_switch_with_cases(self): - from flowx.translator.activity_translators.switch import translate + from flowx.sources.adf.translators.switch import translate case_act = _make_activity("CaseWait", "Wait", {"waitTimeInSeconds": 1}) default_act = _make_activity("DefaultWait", "Wait", {"waitTimeInSeconds": 2}) @@ -1641,7 +1641,7 @@ def test_translate_switch_function_call_routes_through_bridge(self): """C-07 (CF-iter2-001 / CF-iter2-003): @toUpper(coalesce(...)) on the Switch on-expression lowers to a bridge SetVariable task rather than shipping as a raw ADF expression.""" - from flowx.translator.activity_translators.switch import translate + from flowx.sources.adf.translators.switch import translate activity = _make_activity( "Route", diff --git a/tests/unit/test_until_agentic_handler.py b/tests/unit/test_until_agentic_handler.py index 64d02e8..fea28a5 100644 --- a/tests/unit/test_until_agentic_handler.py +++ b/tests/unit/test_until_agentic_handler.py @@ -4,8 +4,8 @@ from flowx.models.adf_ast import AdfDefinitions from flowx.models.ir import PlaceholderActivity -from flowx.parser.adf_loader import _parse_pipeline_json -from flowx.translator.engine import translate_pipeline +from flowx.sources.adf.loader import _parse_pipeline_json +from flowx.sources.adf.translate import translate_pipeline _DEFS = AdfDefinitions(pipelines=[], datasets={}, linked_services={}, triggers=[]) diff --git a/tests/unit/test_web_body_and_param_defaults.py b/tests/unit/test_web_body_and_param_defaults.py index 48e1e48..90e0c60 100644 --- a/tests/unit/test_web_body_and_param_defaults.py +++ b/tests/unit/test_web_body_and_param_defaults.py @@ -6,8 +6,8 @@ from flowx.models.adf_ast import AdfActivity, AdfDefinitions, AdfParameter, AdfPipeline from flowx.models.ir import TranslationContext from flowx.preparer.code_generator import generate_web_activity_notebook -from flowx.translator.activity_translators import web_activity -from flowx.translator.engine import translate_pipeline +from flowx.sources.adf.translate import translate_pipeline +from flowx.sources.adf.translators import web_activity _DEFS = AdfDefinitions(pipelines=[], datasets={}, linked_services={}, triggers=[]) From 5a32894f6772d4840f83f4183d26c0f82ed44668 Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 20 Jul 2026 12:54:04 -0700 Subject: [PATCH 24/77] Add Airflow operator, sensor, and scheduling coverage --- skills/flowx-convert/sources/airflow.md | 35 +- src/flowx/adapter/__main__.py | 29 +- src/flowx/adapter/constants.py | 1 + src/flowx/adapter/session.py | 153 +++-- src/flowx/bundler/dab_writer.py | 32 + src/flowx/ir_serde.py | 8 + src/flowx/models/ir.py | 21 + src/flowx/preparer/activity_preparers/sql.py | 35 ++ src/flowx/preparer/workflow_preparer.py | 3 + src/flowx/sources/adf/translate.py | 4 +- src/flowx/sources/airflow/convert.py | 30 + src/flowx/sources/airflow/discover.py | 58 +- src/flowx/sources/airflow/loader.py | 580 +++++++++++++++--- src/flowx/sources/airflow/operators.py | 494 +++++++++++++++ src/flowx/sources/airflow/templating.py | 231 +++++++ tests/unit/test_airflow_adapter_reporting.py | 64 ++ tests/unit/test_airflow_operators.py | 548 +++++++++++++++++ tests/unit/test_sql_task_and_table_trigger.py | 79 +++ 18 files changed, 2216 insertions(+), 189 deletions(-) create mode 100644 src/flowx/preparer/activity_preparers/sql.py create mode 100644 src/flowx/sources/airflow/operators.py create mode 100644 src/flowx/sources/airflow/templating.py create mode 100644 tests/unit/test_airflow_adapter_reporting.py create mode 100644 tests/unit/test_airflow_operators.py create mode 100644 tests/unit/test_sql_task_and_table_trigger.py diff --git a/skills/flowx-convert/sources/airflow.md b/skills/flowx-convert/sources/airflow.md index 4dfa5e0..079999e 100644 --- a/skills/flowx-convert/sources/airflow.md +++ b/skills/flowx-convert/sources/airflow.md @@ -3,10 +3,11 @@ Source guide for `--source airflow`. Translate parsed Airflow DAGs into Databricks IR. See the parent `SKILL.md` for how to run the phase and the report contract. -Airflow translation is **deterministic** today: the same static parse the discover phase uses -produces the Pipeline IR directly. There is no separate agentic-gap round for Airflow yet — -operators without a mapping are emitted as placeholder tasks the user fills in manually (or via a -future agentic pass), not as pending gaps in the report. +Airflow translation is **deterministic-first with an agentic-gap round**, like the ADF source. The +static parse maps ~35 operator/sensor families directly to IR (Tier 1-3). Operators with no +deterministic mapping become `PlaceholderActivity` tasks *and* are recorded in `gaps.json`, each +carrying the operator's raw source so an agent can reason out the translation and replace the +placeholder — the same `gaps.json` + `merge_agentic` flow the ADF source uses. ## Step 1 — Run the translation @@ -28,13 +29,27 @@ PythonOperator callable or BashOperator command, carrying `generated_source`) or `PlaceholderActivity` (an unmapped operator). Dependencies come from `>>` / `<<`; the DAG's cron `schedule_interval` is carried as the pipeline `schedule`. -## Step 3 — Handle placeholders (optional) +## Step 3 — Handle agentic gaps -For any `PlaceholderActivity`, decide whether to hand-write the notebook body now or leave the -placeholder for the package phase to emit (it ships a notebook with a clear TODO). The shared -`inspect`/`modify`/`merge_agentic` commands from the parent `SKILL.md` are available if you want to -apply agent-translated results, but the ADF just-in-time option chain (notify motifs, -metadata-driven consolidation) does not apply to Airflow. +If convert wrote `/.work/gaps.json`, each entry is an unmapped operator awaiting +LLM-assisted translation. For each gap, read its `raw_definition` (the operator's source, embedded +in the placeholder notebook too) and translate it into a real Databricks task — most portably a +notebook you write to the workspace. Reason from the operator's arguments: e.g. a +`KubernetesPodOperator` running a Python image becomes a notebook (or `%pip install` + the image's +entrypoint logic); an `HttpSensor` becomes a polling notebook using `requests`; a `LivyOperator` +submits Spark directly. + +Write one result JSON per gap into `/agentic_results/` and merge them with the shared +`merge_agentic` command (see the parent `SKILL.md`): + +```bash +"$PY" -m flowx.adapter convert --merge-agentic \ + --report /.work/translation_report.json \ + --agentic-results /agentic_results +``` + +The ADF just-in-time option chain (notify motifs, metadata-driven consolidation) does not apply to +Airflow; only the agentic-gap round does. ## Step 4 — Proceed to package diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py index c45e1b0..6fcf69a 100644 --- a/src/flowx/adapter/__main__.py +++ b/src/flowx/adapter/__main__.py @@ -148,7 +148,14 @@ def _run_workspace_paths(args: argparse.Namespace) -> int: path / host lists so the skill can detect the no-op case. """ paths = collect_workspace_artifact_paths(args.report) - suggested_hosts = detect_databricks_hosts(args.source_dir) if args.source_dir else [] + suggested_hosts: list[str] = [] + if args.source_dir: + if getattr(args, "source", "adf") == "airflow": + from flowx.sources.airflow.loader import detect_hosts + + suggested_hosts = detect_hosts(args.source_dir) + else: + suggested_hosts = detect_databricks_hosts(args.source_dir) payload = { "paths": paths, "suggested_hosts": suggested_hosts, @@ -170,7 +177,7 @@ def _run_inputs(args: argparse.Namespace) -> int: """ from flowx.adapter.session import MigrationInputSession - session = MigrationInputSession(phase=args.phase) + session = MigrationInputSession(phase=args.phase, source=getattr(args, "source", "adf")) pending = session.pending() payload = { "phase": pending.phase, @@ -280,18 +287,23 @@ def _build_parser() -> argparse.ArgumentParser: "workspace-paths", help=( "Detect absolute workspace paths in a stamped report and suggest " - "Databricks workspace hosts from the ADF linked services." + "Databricks workspace hosts from the source (ADF linked services / Airflow DAGs)." ), ) workspace_paths.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") + workspace_paths.add_argument( + "--source", + default="adf", + help="Migration source (adf | airflow); selects how workspace hosts are detected.", + ) workspace_paths.add_argument( "--source-dir", type=Path, default=None, help=( - "Optional path to the ADF JSON export directory. When supplied, " - "the command reads ``linked_services/*.json`` to suggest the " - "workspace host that ``databricks auth login --host`` should use." + "Optional path to the source. For adf, the JSON export dir (reads " + "``linked_services/*.json``); for airflow, a DAG file/dir (scans DAG source for " + "workspace hosts). Used to suggest the host for ``databricks auth login --host``." ), ) workspace_paths.add_argument( @@ -310,6 +322,11 @@ def _build_parser() -> argparse.ArgumentParser: choices=("discover", "convert", "package"), help="Migration phase whose input prompts the agent should surface.", ) + inputs.add_argument( + "--source", + default="adf", + help="Migration source (adf | airflow); words the source-path prompt for discover/convert.", + ) inputs.add_argument( "--out", type=Path, diff --git a/src/flowx/adapter/constants.py b/src/flowx/adapter/constants.py index 02e30aa..4fc9d6f 100644 --- a/src/flowx/adapter/constants.py +++ b/src/flowx/adapter/constants.py @@ -29,6 +29,7 @@ INPUT_ADF_SOURCE_PATH: Final[str] = "adf_source_path" INPUT_ADF_RESOURCE_URL: Final[str] = "adf_resource_url" +INPUT_AIRFLOW_SOURCE_PATH: Final[str] = "airflow_source_path" INPUT_OUTPUT_DIR: Final[str] = "output_dir" INPUT_INVENTORY_PATH: Final[str] = "inventory_path" INPUT_TRANSLATION_REPORT_PATH: Final[str] = "translation_report_path" diff --git a/src/flowx/adapter/session.py b/src/flowx/adapter/session.py index 2b7509c..3a60374 100644 --- a/src/flowx/adapter/session.py +++ b/src/flowx/adapter/session.py @@ -16,6 +16,7 @@ from flowx.adapter.constants import ( INPUT_ADF_RESOURCE_URL, INPUT_ADF_SOURCE_PATH, + INPUT_AIRFLOW_SOURCE_PATH, INPUT_BUNDLE_NAME, INPUT_CATALOG, INPUT_DATABRICKS_PROFILE, @@ -239,65 +240,80 @@ def _collect_motif_consolidations(self) -> dict[str, MotifConsolidate]: return consolidations -_DISCOVER_OPTIONS: tuple[MigrationInputOption, ...] = ( - MigrationInputOption( - option_id=INPUT_ADF_SOURCE_PATH, - prompt="Where are the ADF JSON exports?", - description=( +# Per-source description of the source path the discover/convert phases read. +_SOURCE_PATH_OPTION: dict[str, dict[str, str]] = { + "adf": { + "option_id": INPUT_ADF_SOURCE_PATH, + "prompt": "Where are the ADF JSON exports?", + "description": ( "Unity Catalog volume path (``/Volumes///``) " "or a local directory containing the ADF ARM/JSON export." ), - required=True, - ), - MigrationInputOption( - option_id=INPUT_ADF_RESOURCE_URL, - prompt="ADF resource URL?", - description=( - "Azure portal URL of the source Data Factory. Captured for " - "traceability and surfaced in the generated bundle README; " - "leave blank when the source is exported from a local copy." - ), - default="", - required=False, - ), - MigrationInputOption( - option_id=INPUT_OUTPUT_DIR, - prompt="Which migration output directory should flowx use?", - description=( - "Single shared migration directory used by every phase (default ``./flowx_output``). " - "Discover writes ``metadata/inventory.json``, ``metadata/profile_report.csv``, and the " - "verbatim ``metadata/.arm.json`` into it." - ), - default="./flowx_output", - required=False, + }, + "airflow": { + "option_id": INPUT_AIRFLOW_SOURCE_PATH, + "prompt": "Where are the Airflow DAG files?", + "description": "A DAG ``.py`` file or a local directory of DAG modules to migrate.", + }, +} + +_OUTPUT_DIR_OPTION = MigrationInputOption( + option_id=INPUT_OUTPUT_DIR, + prompt="Which migration output directory should flowx use?", + description=( + "Single shared migration directory used by every phase (default ``./flowx_output``). " + "Discover writes ``metadata/inventory.json`` and ``metadata/profile_report.csv`` into it." ), + default="./flowx_output", + required=False, ) -_CONVERT_OPTIONS: tuple[MigrationInputOption, ...] = ( - MigrationInputOption( - option_id=INPUT_INVENTORY_PATH, - prompt="Path to the inventory.json from the discover phase?", - description="Inventory produced by the discover phase (under the shared migration dir's metadata/).", - default="./flowx_output/metadata/inventory.json", - required=False, - ), - MigrationInputOption( - option_id=INPUT_ADF_SOURCE_PATH, - prompt="Path to the ADF JSON exports?", - description="Same source directory the discover phase consumed; needed for cross-references.", - required=True, - ), - MigrationInputOption( - option_id=INPUT_OUTPUT_DIR, - prompt="Which migration output directory should flowx use?", - description=( - "The same shared migration directory the discover phase used (default ``./flowx_output``). " - "Convert writes its transient report and IR to the directory's ``.work/`` subfolder." + +def _discover_options(source: str) -> tuple[MigrationInputOption, ...]: + """Discover-phase input prompts for *source* (source-path prompt varies by source).""" + spec = _SOURCE_PATH_OPTION.get(source, _SOURCE_PATH_OPTION["adf"]) + options = [ + MigrationInputOption( + option_id=spec["option_id"], prompt=spec["prompt"], description=spec["description"], required=True + ) + ] + if source == "adf": + options.append( + MigrationInputOption( + option_id=INPUT_ADF_RESOURCE_URL, + prompt="ADF resource URL?", + description=( + "Azure portal URL of the source Data Factory. Captured for traceability and " + "surfaced in the generated bundle README; leave blank when exported from a local copy." + ), + default="", + required=False, + ) + ) + options.append(_OUTPUT_DIR_OPTION) + return tuple(options) + + +def _convert_options(source: str) -> tuple[MigrationInputOption, ...]: + """Convert-phase input prompts for *source*.""" + spec = _SOURCE_PATH_OPTION.get(source, _SOURCE_PATH_OPTION["adf"]) + return ( + MigrationInputOption( + option_id=INPUT_INVENTORY_PATH, + prompt="Path to the inventory.json from the discover phase?", + description="Inventory produced by the discover phase (under the shared migration dir's metadata/).", + default="./flowx_output/metadata/inventory.json", + required=False, ), - default="./flowx_output", - required=False, - ), -) + MigrationInputOption( + option_id=spec["option_id"], + prompt=spec["prompt"], + description="Same source the discover phase consumed; needed for cross-references.", + required=True, + ), + _OUTPUT_DIR_OPTION, + ) + _PACKAGE_OPTIONS: tuple[MigrationInputOption, ...] = ( MigrationInputOption( @@ -385,11 +401,16 @@ def _collect_motif_consolidations(self) -> dict[str, MotifConsolidate]: ), ) -_OPTIONS_BY_PHASE: dict[str, tuple[MigrationInputOption, ...]] = { - PHASE_DISCOVER: _DISCOVER_OPTIONS, - PHASE_CONVERT: _CONVERT_OPTIONS, - PHASE_PACKAGE: _PACKAGE_OPTIONS, -} +_SUPPORTED_PHASES: frozenset[str] = frozenset({PHASE_DISCOVER, PHASE_CONVERT, PHASE_PACKAGE}) + + +def _options_for(phase: str, source: str) -> tuple[MigrationInputOption, ...]: + """Returns the input options for *phase*, source-worded for discover/convert.""" + if phase == PHASE_DISCOVER: + return _discover_options(source) + if phase == PHASE_CONVERT: + return _convert_options(source) + return _PACKAGE_OPTIONS class UnknownMigrationPhaseError(ValueError): @@ -409,21 +430,23 @@ class MigrationInputSession: Attributes: phase: One of ``"discover"``, ``"convert"``, ``"package"``. + source: Migration source (``"adf"`` / ``"airflow"``); words the + source-path prompt for the discover/convert phases. """ phase: str + source: str = "adf" _answers: dict[str, str] = field(default_factory=dict) def __post_init__(self) -> None: """Validates that *phase* is one of the supported migration phases. Raises: - UnknownMigrationPhaseError: When *phase* is not registered in - :data:`_OPTIONS_BY_PHASE`. + UnknownMigrationPhaseError: When *phase* is not a supported phase. """ - if self.phase not in _OPTIONS_BY_PHASE: + if self.phase not in _SUPPORTED_PHASES: raise UnknownMigrationPhaseError( - f"Unknown migration phase {self.phase!r}; expected one of {sorted(_OPTIONS_BY_PHASE)}" + f"Unknown migration phase {self.phase!r}; expected one of {sorted(_SUPPORTED_PHASES)}" ) def pending(self) -> PendingMigrationInputs: @@ -433,7 +456,7 @@ def pending(self) -> PendingMigrationInputs: A :class:`PendingMigrationInputs` with the unanswered options for ``self.phase`` in registration order. """ - options = [option for option in _OPTIONS_BY_PHASE[self.phase] if option.option_id not in self._answers] + options = [option for option in _options_for(self.phase, self.source) if option.option_id not in self._answers] return PendingMigrationInputs(phase=self.phase, options=options) def answer(self, option_id: str, value: str) -> None: @@ -447,7 +470,7 @@ def answer(self, option_id: str, value: str) -> None: ValueError: When *option_id* is not a known input for the session's phase. """ - if not any(option.option_id == option_id for option in _OPTIONS_BY_PHASE[self.phase]): + if not any(option.option_id == option_id for option in _options_for(self.phase, self.source)): raise ValueError(f"Unknown input option {option_id!r} for phase {self.phase!r}") self._answers[option_id] = value @@ -461,7 +484,7 @@ def answer_many(self, answers: dict[str, str]) -> None: ValueError: When any pair references an unknown option. No answers are recorded when the call raises. """ - known_ids = {option.option_id for option in _OPTIONS_BY_PHASE[self.phase]} + known_ids = {option.option_id for option in _options_for(self.phase, self.source)} unknown = set(answers) - known_ids if unknown: raise ValueError(f"Unknown input options for phase {self.phase!r}: {sorted(unknown)}") @@ -478,7 +501,7 @@ def collected(self) -> dict[str, str]: missing are omitted so the caller can detect them. """ collected: dict[str, str] = {} - for option in _OPTIONS_BY_PHASE[self.phase]: + for option in _options_for(self.phase, self.source): if option.option_id in self._answers: collected[option.option_id] = self._answers[option.option_id] elif option.default is not None: diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 5e5588e..bef8a33 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -45,6 +45,7 @@ SetVariableActivity, SparkJarActivity, SparkPythonActivity, + SqlActivity, SwitchActivity, SwitchCase, UnsupportedActivity, @@ -122,6 +123,11 @@ def write_bundle( pipeline_resources = _collect_pipeline_resources(workflow) pipeline_variable_declarations = _build_pipeline_variable_declarations(pipeline_resources, catalog, schema) + # sql_task references ${var.warehouse_id}; declare it (no default -> user supplies at deploy). + if _bundle_uses_sql_task(workflow): + pipeline_variable_declarations.setdefault( + "warehouse_id", {"description": "SQL warehouse id for sql_task queries"} + ) # 1. Write databricks.yml. When any task runs on classic compute, spark_version / node_type_id # defaults come from the ADF linked-service configs; when every task is serverless, they're omitted. @@ -941,6 +947,12 @@ def _any_task_uses_classic_cluster(tasks: list[dict[str, Any]]) -> bool: return any(task.get("job_cluster_key") for task in _iter_tasks_recursively(tasks)) +def _bundle_uses_sql_task(workflow: PreparedWorkflow) -> bool: + """Return True if any task (parent or inner workflow) is a sql_task.""" + task_lists = [workflow.tasks, *(inner.tasks for inner in workflow.inner_workflows)] + return any("sql_task" in task for tasks in task_lists for task in _iter_tasks_recursively(tasks)) + + def _bind_cluster_to_notebook_tasks(tasks: list[dict[str, Any]]) -> None: """Binds notebook tasks to the cluster their compute_mode marker dictates. @@ -1106,6 +1118,18 @@ def _apply_schedule_to_job(job_def: dict[str, Any], spec: dict[str, Any]) -> Non trigger_block["pause_status"] = spec["pause_status"] job_def["trigger"] = trigger_block return + if kind == "table_update": + table_update: dict[str, Any] = { + "table_names": list(spec.get("table_names") or []), + "condition": spec.get("condition", "ANY_UPDATED"), + } + if spec.get("min_time_between_triggers_seconds"): + table_update["min_time_between_triggers_seconds"] = spec["min_time_between_triggers_seconds"] + trigger_block = {"table_update": table_update} + if spec.get("pause_status"): + trigger_block["pause_status"] = spec["pause_status"] + job_def["trigger"] = trigger_block + return if kind == "manual_setup": # No DAB primitive -- surface the raw spec so SETUP.md can flag it. job_def["schedule_setup_note"] = spec @@ -1602,6 +1626,13 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: selectors=list(task_ir.get("selectors") or []), nodes=list(task_ir.get("nodes") or []), ) + if task_type == "SqlActivity": + return SqlActivity( + **base, + sql=task_ir.get("sql", ""), + parameters=task_ir.get("parameters"), + warehouse_ref=task_ir.get("warehouse_ref", "${var.warehouse_id}"), + ) if task_type == "SparkJarActivity": return SparkJarActivity( **base, @@ -1699,6 +1730,7 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: original_type=task_ir.get("original_type", task_type), notebook_path=task_ir.get("notebook_path", "/UNSUPPORTED_ADF_ACTIVITY"), comment=task_ir.get("comment"), + raw_definition=task_ir.get("raw_definition"), ) return PlaceholderActivity( **base, diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py index 408b99e..2d86861 100644 --- a/src/flowx/ir_serde.py +++ b/src/flowx/ir_serde.py @@ -37,6 +37,7 @@ SetVariableActivity, SparkJarActivity, SparkPythonActivity, + SqlActivity, SwitchActivity, UnsupportedActivity, WaitActivity, @@ -262,6 +263,11 @@ def activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["bridge_required_parameters"] = dict(activity.bridge_required_parameters) case WaitActivity(): extra["wait_time_seconds"] = activity.wait_time_seconds + case SqlActivity(): + extra["sql"] = activity.sql + if activity.parameters: + extra["parameters"] = dict(activity.parameters) + extra["warehouse_ref"] = activity.warehouse_ref case SparkJarActivity(): extra["main_class_name"] = activity.main_class_name if activity.parameters: @@ -325,6 +331,8 @@ def activity_extra_fields(activity: Activity) -> dict[str, Any]: case PlaceholderActivity(): extra["original_type"] = activity.original_type extra["comment"] = activity.comment + if activity.raw_definition is not None: + extra["raw_definition"] = activity.raw_definition case UnsupportedActivity(): extra["original_type"] = activity.original_type extra["reason"] = activity.reason diff --git a/src/flowx/models/ir.py b/src/flowx/models/ir.py index 5b4435c..242bd91 100644 --- a/src/flowx/models/ir.py +++ b/src/flowx/models/ir.py @@ -347,6 +347,27 @@ class RunJobActivity(Activity): job_parameters: dict[str, Any] | None = None +@dataclass(slots=True, kw_only=True) +class SqlActivity(Activity): + """Warehouse-backed SQL activity -> Databricks ``sql_task``. + + A source SQL step (an Airflow SQLExecuteQuery/DatabricksSql/Hive operator, or + an ADF activity whose SQL runs on a warehouse) whose SQL text is extracted to + a ``.sql`` file and run on a SQL warehouse. + + Attributes: + sql: The SQL statement text (extracted to a ``.sql`` file by the preparer). + parameters: Named ``sql_task.parameters`` (e.g. ``run_date``) passed to + the query, referenced in the SQL as ``:name``. + warehouse_ref: DAB reference for the warehouse id (defaults to the + ``warehouse_id`` bundle variable). + """ + + sql: str + parameters: dict[str, str] | None = None + warehouse_ref: str = "${var.warehouse_id}" + + @dataclass(slots=True, kw_only=True) class SparkJarActivity(Activity): """Spark JAR activity. diff --git a/src/flowx/preparer/activity_preparers/sql.py b/src/flowx/preparer/activity_preparers/sql.py new file mode 100644 index 0000000..b775552 --- /dev/null +++ b/src/flowx/preparer/activity_preparers/sql.py @@ -0,0 +1,35 @@ +"""Preparer for SqlActivity -> sql_task dict.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from flowx.models.dab import DabNotebook +from flowx.preparer.workflow_preparer import PreparedActivity, build_common_task_fields + +if TYPE_CHECKING: + from flowx.models.ir import SqlActivity + + +def prepare(activity: SqlActivity, *, scope: str = "") -> PreparedActivity: + """Converts a SqlActivity into a DAB sql_task with an extracted .sql file. + + The SQL text is written to ``src/sql/.sql`` and the task references it + via ``sql_task.file.path`` on the warehouse given by ``warehouse_ref``. Named + ``parameters`` become ``sql_task.parameters`` (referenced as ``:name`` in the SQL). + """ + task = build_common_task_fields(activity) + + sql_rel_path = f"sql/{activity.task_key}.sql" + content = activity.sql if activity.sql.endswith("\n") else activity.sql + "\n" + notebooks = [DabNotebook(relative_path=sql_rel_path, content=content, language="sql")] + + sql_task: dict[str, object] = { + "warehouse_id": activity.warehouse_ref, + "file": {"path": f"../src/{sql_rel_path}", "source": "WORKSPACE"}, + } + if activity.parameters: + sql_task["parameters"] = dict(activity.parameters) + task["sql_task"] = sql_task + + return PreparedActivity(task=task, notebooks=notebooks) diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py index 5b0b976..49095ff 100644 --- a/src/flowx/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -26,6 +26,7 @@ SetVariableActivity, SparkJarActivity, SparkPythonActivity, + SqlActivity, SwitchActivity, UnsupportedActivity, WaitActivity, @@ -137,6 +138,7 @@ def prepare_activity( set_variable, spark_jar, spark_python, + sql, switch, wait, web_activity, @@ -146,6 +148,7 @@ def prepare_activity( NotebookActivity: notebook.prepare, SparkJarActivity: spark_jar.prepare, SparkPythonActivity: spark_python.prepare, + SqlActivity: sql.prepare, CopyActivity: copy.prepare, LookupActivity: lookup.prepare, WebActivity: web_activity.prepare, diff --git a/src/flowx/sources/adf/translate.py b/src/flowx/sources/adf/translate.py index 7a9785d..4ceaa52 100644 --- a/src/flowx/sources/adf/translate.py +++ b/src/flowx/sources/adf/translate.py @@ -1302,9 +1302,7 @@ def main(argv: list[str] | None = None) -> int: if args.merge_agentic: if not args.report or not args.agentic_results: parser.error("--merge-agentic requires --report and --agentic-results") - merged_count, unmatched_count = ir_serde.merge_agentic_results( - args.report, args.agentic_results, args.output - ) + merged_count, unmatched_count = ir_serde.merge_agentic_results(args.report, args.agentic_results, args.output) print("\nAgentic Merge Summary") print("=====================") print(f"Merged: {merged_count}") diff --git a/src/flowx/sources/airflow/convert.py b/src/flowx/sources/airflow/convert.py index 0c8638a..c2fd769 100644 --- a/src/flowx/sources/airflow/convert.py +++ b/src/flowx/sources/airflow/convert.py @@ -15,6 +15,7 @@ from pathlib import Path from flowx.ir_serde import pipeline_to_dict +from flowx.models.ir import PlaceholderActivity from flowx.sources.airflow.loader import load_pipelines logger = logging.getLogger(__name__) @@ -44,14 +45,43 @@ def main(argv: list[str] | None = None) -> int: report_file = work_dir / "translation_report.json" report_file.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") + # Emit gaps.json for every unmapped operator so the agentic-gap round (driven by the + # convert SKILL guide + merge_agentic) can reason from the operator source and replace + # the placeholder with a real notebook -- the same flow the ADF source uses. + gaps = _collect_gaps(pipelines) + if gaps: + (work_dir / "gaps.json").write_text(json.dumps(gaps, indent=2, default=str), encoding="utf-8") + total_tasks = sum(len(p.tasks) for p in pipelines) print("\nAirflow Translation Summary") print("===========================") print(f"DAGs translated: {len(pipelines)}") print(f"Total tasks: {total_tasks}") + print(f"Agentic gaps: {len(gaps)}") print(f"\nTranslation report (intermediate): {report_file}") return 0 +def _collect_gaps(pipelines: list) -> list[dict]: + """Returns one AgenticGap-shaped dict per PlaceholderActivity across all pipelines. + + Each carries the placeholder's ``activity_name``, ``activity_type`` (the Airflow + operator), and ``raw_definition`` (the operator's source) so the agentic round can + translate it -- the Airflow analog of ADF's gaps.json. + """ + gaps: list[dict] = [] + for pipeline in pipelines: + for task in pipeline.tasks: + if isinstance(task, PlaceholderActivity): + gaps.append( + { + "activity_name": task.name, + "activity_type": task.original_type, + "raw_definition": task.raw_definition, + } + ) + return gaps + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/src/flowx/sources/airflow/discover.py b/src/flowx/sources/airflow/discover.py index df3fba0..fce6d93 100644 --- a/src/flowx/sources/airflow/discover.py +++ b/src/flowx/sources/airflow/discover.py @@ -47,7 +47,9 @@ def build_inventory_dict(pipelines: list[Pipeline], source_dir: str) -> dict[str agentic += sum(1 for i in items if i["strategy"] == "agentic") pipeline_entries.append({"name": pipeline.name, "activities": items}) activity_count = deterministic + agentic - coverage = round(100.0 * deterministic / activity_count, 1) if activity_count else 0.0 + # Coverage counts both deterministic and agentic as "has a translation path", matching the + # shared reporting.coverage formula (agentic gaps are translated in the convert phase). + coverage = round(100.0 * (deterministic + agentic) / activity_count, 1) if activity_count else 0.0 return { "source": "airflow", "source_dir": source_dir, @@ -63,15 +65,57 @@ def build_inventory_dict(pipelines: list[Pipeline], source_dir: str) -> dict[str } +# Full profile column set the shared reporting.coverage / dashboard consume. Airflow has no +# dataset/linked-service/motif concept, so those are 0; the rest are computed from task types. +_PROFILE_COLUMNS: tuple[str, ...] = ( + "pipeline", + "activities", + "datasets", + "linked_services", + "collapsible_patterns", + "databricks_native_activities", + "control_flow_activities", + "other_activities", + "complexity_score", + "complexity_size", +) + +_NATIVE_TYPES = frozenset( + {"NotebookActivity", "SparkPythonActivity", "SparkJarActivity", "SqlActivity", "RunJobActivity"} +) +_CONTROL_FLOW_TYPES = frozenset({"ForEachActivity"}) + + +def _profile_row(pipeline: Pipeline) -> dict[str, Any]: + """Computes one profile row for *pipeline* over the full column set.""" + type_names = [type(task).__name__ for task in pipeline.tasks] + total = len(type_names) + native = sum(1 for name in type_names if name in _NATIVE_TYPES) + control = sum(1 for name in type_names if name in _CONTROL_FLOW_TYPES) + other = total - native - control + score = native * 1 + control * 2 + other * 3 + size = "S" if score <= 5 else "M" if score <= 15 else "L" if score <= 30 else "XL" + return { + "pipeline": pipeline.name, + "activities": total, + "datasets": 0, + "linked_services": 0, + "collapsible_patterns": 0, + "databricks_native_activities": native, + "control_flow_activities": control, + "other_activities": other, + "complexity_score": score, + "complexity_size": size, + } + + def _write_profile_csv(pipelines: list[Pipeline], path: Path) -> None: - """Writes one per-pipeline complexity row, mirroring the ADF profile report.""" + """Writes the per-pipeline complexity report with the full shared column set.""" with open(path, "w", newline="", encoding="utf-8") as handle: - writer = csv.writer(handle) - writer.writerow(["pipeline", "activities", "complexity_size"]) + writer = csv.DictWriter(handle, fieldnames=list(_PROFILE_COLUMNS)) + writer.writeheader() for pipeline in pipelines: - count = len(pipeline.tasks) - size = "S" if count <= 5 else "M" if count <= 15 else "L" if count <= 30 else "XL" - writer.writerow([pipeline.name, count, size]) + writer.writerow(_profile_row(pipeline)) def main(argv: list[str] | None = None) -> int: diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py index 9980294..fce3bf4 100644 --- a/src/flowx/sources/airflow/loader.py +++ b/src/flowx/sources/airflow/loader.py @@ -7,25 +7,33 @@ is reused unchanged. The ``discover`` and ``convert`` phase entry points in this package wrap :func:`load_airflow_dag`. -Coverage: PythonOperator (callable body -> generated notebook), BashOperator -(command -> generated notebook), ``>>`` / ``<<`` dependencies, and a cron -``schedule_interval`` -> Quartz. Operators without a mapping become -PlaceholderActivity so coverage still counts them. +Coverage spans the four tiers (per-operator builders live in +:mod:`flowx.sources.airflow.operators`): Tier 1 direct mappings (Python/Bash, +Spark-submit, Databricks provider, SQL, dbt CLI), Tier 2 semantic +(branch/virtualenv, cosmos ``DbtTaskGroup`` -> DbtFactoryActivity, Dummy/Empty +dropped + rewired), Tier 3 sensors (file sensors -> ``file_arrival`` trigger, +time sensors -> schedule), and Tier 4 (unmapped -> PlaceholderActivity). +``>>`` / ``<<`` dependencies and cron ``schedule_interval`` -> Quartz are +handled here. """ from __future__ import annotations import ast -import textwrap +import json +import re from pathlib import Path from flowx.models.ir import ( Activity, + DbtFactoryActivity, Dependency, - NotebookActivity, + ForEachActivity, Pipeline, - PlaceholderActivity, + SqlActivity, ) +from flowx.sources.airflow import operators as ops +from flowx.sources.airflow import templating def _sanitize_task_key(name: str) -> str: @@ -67,40 +75,85 @@ def _cron_to_quartz(cron: str) -> str | None: } -def _schedule_from_interval(interval: str | None) -> dict[str, object] | None: - """Builds a Pipeline.schedule spec from an Airflow schedule_interval.""" - if not interval: +_TIMEDELTA_UNIT_SECONDS: dict[str, int] = { + "weeks": 604800, + "days": 86400, + "hours": 3600, + "minutes": 60, + "seconds": 1, +} + + +def _extract_timezone(node: ast.expr | None) -> str | None: + """Extracts an IANA timezone from a ``pendulum.timezone("…")`` call or a tz string kwarg. + + Handles ``start_date=datetime(..., tzinfo=pendulum.timezone("Europe/Madrid"))``, + ``timezone="Europe/Madrid"``, and ``pendulum.timezone("…")`` directly. Returns None + when no literal timezone is present (caller falls back to UTC). + """ + if node is None: + return None + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.Call): + func = node.func + name = func.attr if isinstance(func, ast.Attribute) else (func.id if isinstance(func, ast.Name) else "") + if name in ("timezone", "timezone_") and node.args: + return ops.literal_str(node.args[0]) + # datetime(..., tzinfo=pendulum.timezone("…")) / tz=... + for kw in node.keywords: + if kw.arg in ("tzinfo", "tz"): + return _extract_timezone(kw.value) + return None + + +def _timedelta_to_periodic(node: ast.expr | None) -> dict[str, object] | None: + """Maps a ``timedelta(...)`` schedule to a ``trigger.periodic`` spec. + + Databricks periodic units are DAYS/HOURS/WEEKS. A timedelta of whole weeks/days/hours + maps to the largest exact unit; anything finer (minutes/seconds) is expressed as a + cron in the caller, so this returns None for those. + """ + if not isinstance(node, ast.Call): + return None + func = node.func + name = func.attr if isinstance(func, ast.Attribute) else (func.id if isinstance(func, ast.Name) else "") + if name != "timedelta": return None - quartz: str | None = _CRON_PRESETS.get(interval) or _cron_to_quartz(interval) - if quartz is None: + total = 0 + for kw in node.keywords: + if kw.arg in _TIMEDELTA_UNIT_SECONDS and isinstance(kw.value, ast.Constant): + if isinstance(kw.value.value, int): + total += kw.value.value * _TIMEDELTA_UNIT_SECONDS[kw.arg] + if total <= 0: return None - return { - "kind": "schedule", - "quartz_cron_expression": quartz, - "timezone_id": "UTC", - "pause_status": "UNPAUSED", - } + for unit, unit_seconds in (("WEEKS", 604800), ("DAYS", 86400), ("HOURS", 3600)): + if total % unit_seconds == 0: + return {"kind": "periodic", "interval": total // unit_seconds, "unit": unit, "pause_status": "UNPAUSED"} + return None -def _notebook_source_from_callable(func: ast.FunctionDef, source: str) -> str: - """Renders a PythonOperator callable's body as a Databricks notebook. +def _schedule_from_interval( + interval: str | None, + *, + node: ast.expr | None = None, + timezone: str | None = None, +) -> dict[str, object] | None: + """Builds a Pipeline.schedule spec from an Airflow schedule. - Slices each body statement's source segment out of the module *source* and - dedents so the extracted block is valid top-level notebook code. + A string cron / preset -> ``kind: schedule`` (Quartz) with the DAG timezone; + a ``timedelta(...)`` -> ``kind: periodic``. Returns None when neither applies. """ - segments = [ast.get_source_segment(source, stmt) for stmt in func.body] - body_src = "\n\n".join(seg for seg in segments if seg) - body_src = textwrap.dedent(body_src) - return f"# Databricks notebook source\n# Migrated from Airflow PythonOperator '{func.name}'.\n\n" + body_src + "\n" - - -def _notebook_source_from_bash(task_id: str, command: str) -> str: - """Renders a BashOperator command as a Databricks notebook shell cell.""" - return ( - "# Databricks notebook source\n" - f"# Migrated from Airflow BashOperator '{task_id}'.\n\n" - "# MAGIC %sh\n" + "".join(f"# MAGIC {line}\n" for line in command.splitlines()) - ) + if interval: + quartz: str | None = _CRON_PRESETS.get(interval) or _cron_to_quartz(interval) + if quartz is not None: + return { + "kind": "schedule", + "quartz_cron_expression": quartz, + "timezone_id": timezone or "UTC", + "pause_status": "UNPAUSED", + } + return _timedelta_to_periodic(node) class _DagVisitor(ast.NodeVisitor): @@ -112,38 +165,86 @@ def __init__(self, module: ast.Module) -> None: } # task variable name -> (task_id, operator, kwargs) self.operators: dict[str, tuple[str, str, dict[str, ast.expr]]] = {} + # task variable name -> the operator's ast.Call node (for source-slicing placeholders) + self.calls: dict[str, ast.Call] = {} self.edges: list[tuple[str, str]] = [] # (upstream_var, downstream_var) self.dag_id: str | None = None self.schedule_interval: str | None = None + self.schedule_node: ast.expr | None = None + self.timezone: str | None = None + self.default_args: dict[str, ast.expr] = {} + # task variable name -> TaskGroup id prefix (for task-key namespacing) + self.groups: dict[str, str] = {} + self._group_stack: list[str] = [] + # task variable names defined via dynamic mapping (.expand()) -> wrapped in a for_each + self.mapped: set[str] = set() def functions(self) -> dict[str, ast.FunctionDef]: return self._functions def visit_Assign(self, node: ast.Assign) -> None: - if ( - isinstance(node.value, ast.Call) - and isinstance(node.value.func, ast.Name) - and node.value.func.id.endswith("Operator") - and len(node.targets) == 1 - and isinstance(node.targets[0], ast.Name) - ): + if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name) and isinstance(node.value, ast.Call): var = node.targets[0].id - kwargs = {kw.arg: kw.value for kw in node.value.keywords if kw.arg} - task_id = _literal_str(kwargs.get("task_id")) or var - self.operators[var] = (task_id, node.value.func.id, kwargs) + direct = _direct_operator_call(node.value) + mapped = None if direct is not None else _mapped_operator_call(node.value) + call = direct or mapped + if call is not None and isinstance(call.func, ast.Name): + construct = call.func.id + kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg} + task_id = ops.literal_str(kwargs.get("task_id")) or ops.literal_str(kwargs.get("group_id")) or var + self.operators[var] = (task_id, construct, kwargs) + self.calls[var] = call + if mapped is not None: + self.mapped.add(var) + if self._group_stack: + self.groups[var] = "__".join(self._group_stack) self.generic_visit(node) def visit_With(self, node: ast.With) -> None: + pushed_group = False for item in node.items: call = item.context_expr - if isinstance(call, ast.Call) and isinstance(call.func, ast.Name) and call.func.id == "DAG": - self._read_dag_kwargs(call) + if isinstance(call, ast.Call) and isinstance(call.func, ast.Name): + if call.func.id == "DAG": + self._read_dag_kwargs(call) + elif call.func.id == "TaskGroup": + # `with TaskGroup("etl") as tg:` — namespace the member tasks by group id. + kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg} + group_id = ( + ops.literal_str(kwargs.get("group_id")) + or (ops.literal_str(call.args[0]) if call.args else None) + or "group" + ) + self._group_stack.append(_sanitize_task_key(group_id)) + pushed_group = True + elif _is_task_construct(call.func.id) and item.optional_vars is not None: + # `with DbtTaskGroup(...) as g:` — a cosmos group bound to a name. + if isinstance(item.optional_vars, ast.Name): + var = item.optional_vars.id + kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg} + task_id = ops.literal_str(kwargs.get("group_id")) or var + self.operators[var] = (task_id, call.func.id, kwargs) + self.calls[var] = call self.generic_visit(node) + if pushed_group: + self._group_stack.pop() def _read_dag_kwargs(self, call: ast.Call) -> None: kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg} - self.dag_id = _literal_str(kwargs.get("dag_id")) - self.schedule_interval = _literal_str(kwargs.get("schedule_interval")) or _literal_str(kwargs.get("schedule")) + self.dag_id = ops.literal_str(kwargs.get("dag_id")) + self.schedule_node = kwargs.get("schedule_interval") or kwargs.get("schedule") + self.schedule_interval = ops.literal_str(kwargs.get("schedule_interval")) or ops.literal_str( + kwargs.get("schedule") + ) + self.timezone = _extract_timezone(kwargs.get("start_date")) or _extract_timezone(kwargs.get("timezone")) + # default_args is a dict literal of DAG-wide task settings (retries, timeouts, email). + default_args = kwargs.get("default_args") + if isinstance(default_args, ast.Dict): + self.default_args = { + key.value: val + for key, val in zip(default_args.keys, default_args.values) + if isinstance(key, ast.Constant) and isinstance(key.value, str) + } def visit_Expr(self, node: ast.Expr) -> None: # Capture `a >> b >> c` and `a << b` dependency chains. @@ -172,13 +273,54 @@ def _flatten_shift(node: ast.expr) -> list[str]: return [] -def _literal_str(node: ast.expr | None) -> str | None: - """Returns the string value of a constant AST node, else None.""" - if isinstance(node, ast.Constant) and isinstance(node.value, str): - return node.value +def _direct_operator_call(node: ast.Call) -> ast.Call | None: + """Returns *node* if it is a direct ``SomeOperator(...)`` / ``SomeSensor(...)`` call.""" + if isinstance(node.func, ast.Name) and _is_task_construct(node.func.id): + return node return None +def _mapped_operator_call(node: ast.Call) -> ast.Call | None: + """Returns the underlying operator call for a dynamic-mapping ``.expand(...)`` chain. + + Handles ``Op(...).expand(...)`` and ``Op.partial(...).expand(...)``. The returned + Call's keywords are the merged operator kwargs (partial args + expand args), and its + ``.func`` is the operator Name, so the caller treats it like a direct operator call. + The mapped kwargs let the loader wrap the operator in a for_each_task. + """ + if not (isinstance(node.func, ast.Attribute) and node.func.attr == "expand"): + return None + inner = node.func.value # the Op(...) or Op.partial(...) call + if not isinstance(inner, ast.Call): + return None + if isinstance(inner.func, ast.Name) and _is_task_construct(inner.func.id): + operator_name = inner.func.id # Op(...).expand(...) + elif ( + isinstance(inner.func, ast.Attribute) + and inner.func.attr == "partial" + and isinstance(inner.func.value, ast.Name) + and _is_task_construct(inner.func.value.id) + ): + operator_name = inner.func.value.id # Op.partial(...).expand(...) + else: + return None + merged = ast.Call( + func=ast.Name(id=operator_name, ctx=ast.Load()), + args=[], + keywords=list(inner.keywords) + list(node.keywords), + ) + return merged + + +def _is_task_construct(name: str) -> bool: + """True when a call name is an Airflow task-defining construct we should capture. + + Covers operators (``*Operator``), sensors (``*Sensor``), and the cosmos + constructs (``DbtDag`` / ``DbtTaskGroup``) that don't follow either suffix. + """ + return name.endswith("Operator") or name.endswith("Sensor") or name in ops.COSMOS_CONSTRUCTS + + def load_airflow_dag(dag_path: Path) -> Pipeline: """Parses an Airflow DAG file into a flowx Pipeline IR. @@ -186,8 +328,12 @@ def load_airflow_dag(dag_path: Path) -> Pipeline: dag_path: Path to a ``.py`` DAG module. Returns: - A :class:`~flowx.models.ir.Pipeline` whose tasks are NotebookActivity - (for mapped operators) or PlaceholderActivity (for unmapped ones). + A :class:`~flowx.models.ir.Pipeline`. Mapped operators become their IR + node (NotebookActivity, SparkPython/JarActivity, RunJobActivity, + DbtFactoryActivity, ...); Dummy/Empty are dropped with dependency + rewiring; file sensors lift to a job-level file_arrival trigger; time + sensors are absorbed into the schedule; unmapped operators become a + PlaceholderActivity. """ source = Path(dag_path).read_text(encoding="utf-8") module = ast.parse(source) @@ -195,27 +341,285 @@ def load_airflow_dag(dag_path: Path) -> Pipeline: visitor.visit(module) functions = visitor.functions() - # var -> task_key, and per-var dependency edges resolved to task_keys. - var_to_task_key = {var: _sanitize_task_key(task_id) for var, (task_id, _, _) in visitor.operators.items()} + # Prefix TaskGroup member keys with the group id (e.g. extract__run) so two tasks named + # `run` in different groups don't collide. + def _task_key(var: str, task_id: str) -> str: + key = _sanitize_task_key(task_id) + return f"{visitor.groups[var]}__{key}" if var in visitor.groups else key + + var_to_task_key = {var: _task_key(var, task_id) for var, (task_id, _, _) in visitor.operators.items()} + + # Build the upstream adjacency in dependency terms, then drop structural nodes + # (Dummy/Empty, file/time sensors) by rewiring their downstreams to their upstreams. upstreams: dict[str, list[str]] = {var: [] for var in visitor.operators} for upstream_var, downstream_var in visitor.edges: if downstream_var in upstreams and upstream_var in var_to_task_key: upstreams[downstream_var].append(upstream_var) + dropped = {var for var, (_, op, kw) in visitor.operators.items() if _is_dropped_construct(op, kw)} + upstreams = _rewire_dropped(upstreams, dropped) + + # Lift file/time sensors to a job-level trigger / schedule note (they don't become tasks). + schedule = _schedule_from_interval(visitor.schedule_interval, node=visitor.schedule_node, timezone=visitor.timezone) + trigger = _trigger_from_sensors(visitor.operators) + if trigger is not None and schedule is None: + schedule = trigger + + # Collapse all dbt CLI operators over the one project into a single DbtFactoryActivity. + dbt_vars = [var for var, (_, op, _) in visitor.operators.items() if op in ops.DBT_CLI_OPERATORS] + tasks: list[Activity] = [] + referenced_params: set[str] = set() + emitted_dbt = False for var, (task_id, operator, kwargs) in visitor.operators.items(): + if var in dropped: + continue task_key = var_to_task_key[var] - depends_on = [Dependency(task_key=var_to_task_key[u]) for u in upstreams[var]] or None - tasks.append(_build_activity(task_id, task_key, operator, kwargs, functions, depends_on, source)) + outcome = templating.trigger_rule_outcome(kwargs) + depends_on = [Dependency(task_key=var_to_task_key[u], outcome=outcome) for u in upstreams[var]] or None + + if operator in ops.COSMOS_CONSTRUCTS: + tasks.append(_build_dbt_factory(task_id, task_key, [kwargs], depends_on)) + continue + if operator in ops.DBT_CLI_OPERATORS: + # Emit one factory job for the whole dbt chain, at the first dbt task's position. + if emitted_dbt: + continue + emitted_dbt = True + dbt_kwargs = [visitor.operators[v][2] for v in dbt_vars] + tasks.append(_build_dbt_factory(task_id, task_key, dbt_kwargs, depends_on)) + continue + call_node = visitor.calls.get(var) + call_source = ast.get_source_segment(source, call_node) or "" if call_node is not None else "" + ctx = ops.OperatorContext( + task_id=task_id, + task_key=task_key, + operator=operator, + kwargs=kwargs, + functions=functions, + source=source, + call_source=call_source, + default_args=visitor.default_args, + ) + builder = ops.OPERATOR_REGISTRY.get(operator, ops.build_placeholder) + activity = builder(ctx) + activity.depends_on = depends_on + # Stamp DAG/task retry + timeout policy (per-task kwargs override default_args). + policy = templating.retry_policy(visitor.default_args, kwargs) + activity.max_retries = policy.get("max_retries") + activity.timeout_seconds = policy.get("timeout_seconds") + activity.min_retry_interval_millis = policy.get("min_retry_interval_millis") + # Convert Airflow Jinja in the activity's parameter fields to DAB refs; collect params. + referenced_params |= _convert_activity_templates(activity) + + if var in visitor.mapped: + # Dynamic mapping (.expand()) -> a for_each_task iterating the mapped operator. + tasks.append(_wrap_in_for_each(activity, task_id, task_key, depends_on, kwargs)) + else: + tasks.append(activity) + + parameters = [{"name": name} for name in sorted(referenced_params)] or None return Pipeline( name=visitor.dag_id or Path(dag_path).stem, tasks=tasks, - schedule=_schedule_from_interval(visitor.schedule_interval), + parameters=parameters, + schedule=schedule, tags={"source": "airflow", "dag_id": visitor.dag_id or ""}, ) +def _wrap_in_for_each( + activity: Activity, + task_id: str, + task_key: str, + depends_on: list[Dependency] | None, + kwargs: dict[str, ast.expr], +) -> ForEachActivity: + """Wraps a dynamically-mapped operator in a ForEachActivity (-> for_each_task). + + Airflow ``.expand(x=[...])`` fans a task out over an iterable. The for_each's + ``inputs`` is the first list-valued expand kwarg (rendered as a JSON array literal + when it is a static list; otherwise ``{{job.parameters...}}`` is left for review). + The mapped operator becomes the single inner activity, re-keyed so it doesn't + collide with the for_each task key. + """ + items = "[]" + for key, node in kwargs.items(): + if key in ("task_id", "group_id"): + continue + value = ops.literal_value(node) + if isinstance(value, list): + items = json.dumps(value) + break + inner = activity + inner.task_key = f"{task_key}_iteration" + inner.name = f"{task_id}_iteration" + inner.depends_on = None + return ForEachActivity( + name=task_id, + task_key=task_key, + depends_on=depends_on, + items_expression=items, + inner_activities=[inner], + ) + + +def _convert_activity_templates(activity: Activity) -> set[str]: + """Converts Airflow Jinja in an activity's parameter fields to DAB refs. + + Mutates ``base_parameters`` (NotebookActivity), ``parameters`` (Spark/Sql/RunJob), + ``job_parameters`` (RunJob), and ``sql`` (SqlActivity) in place, returning the set + of ``{{job.parameters.X}}`` names referenced so the pipeline can declare them. + """ + referenced: set[str] = set() + for attr in ("base_parameters", "job_parameters", "parameters"): + value = getattr(activity, attr, None) + if value: + converted, refs = templating.convert_params(value) + setattr(activity, attr, converted) + referenced |= refs + if isinstance(activity, SqlActivity): + converted_sql, refs = templating.convert_template(activity.sql) + activity.sql = converted_sql + referenced |= refs + # generated_source was already rewritten (Variable.get -> dbutils.widgets.get); collect the + # widget names so the pipeline declares them as job parameters. + generated = getattr(activity, "generated_source", None) + if isinstance(generated, str): + referenced |= set(_WIDGET_GET.findall(generated)) + return referenced + + +_WIDGET_GET = re.compile(r"""dbutils\.widgets\.get\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\)""") + + +def _is_dropped_construct(operator: str, kwargs: dict[str, ast.expr]) -> bool: + """True for constructs that produce no task (lifted to a trigger/schedule or removed). + + Dummy/Empty and file/time sensors always drop. A table sensor drops only when it + names a table (it lifts to a table_update trigger); a table/SQL sensor with no + ``table_name`` is an arbitrary-condition sensor and is kept as a placeholder task + rather than silently vanishing. + """ + if operator in ops.DUMMY_OPERATORS or operator in ops.FILE_SENSORS or operator in ops.TIME_SENSORS: + return True + if operator in ops.TABLE_SENSORS: + return ops.literal_str(kwargs.get("table_name")) is not None + return False + + +def _rewire_dropped(upstreams: dict[str, list[str]], dropped: set[str]) -> dict[str, list[str]]: + """Returns upstream edges with *dropped* vars removed and their edges bridged. + + A downstream of a dropped node inherits the dropped node's (transitive) + non-dropped upstreams, so the DAG stays connected after Dummy/Empty and + lifted sensors are removed. + """ + + def resolve(var: str, seen: set[str]) -> list[str]: + result: list[str] = [] + for up in upstreams.get(var, []): + if up in dropped: + if up not in seen: + result.extend(resolve(up, seen | {up})) + else: + result.append(up) + # De-dup while preserving order. + return list(dict.fromkeys(result)) + + return {var: resolve(var, {var}) for var in upstreams if var not in dropped} + + +def _trigger_from_sensors(operators: dict[str, tuple[str, str, dict[str, ast.expr]]]) -> dict[str, object] | None: + """Builds a job-level trigger from the first eligible sensor. + + File sensors (S3/GCS/File/HDFS) -> ``trigger.file_arrival``; table sensors with a + ``table_name`` -> ``trigger.table_update``. File sensors take precedence when both + are present. Only one trigger is emitted (DABs jobs take one); additional sensors + are left for MIGRATION_NOTES. Returns None when no eligible sensor is present. + """ + for _var, (_task_id, operator, kwargs) in operators.items(): + if operator in ops.FILE_SENSORS: + url = ( + ops.literal_str(kwargs.get("bucket_key")) + or ops.literal_str(kwargs.get("filepath")) + or ops.literal_str(kwargs.get("filepath_")) + or ops.literal_str(kwargs.get("bucket_name")) + or "" + ) + return {"kind": "file_arrival", "url": url, "pause_status": "UNPAUSED"} + for _var, (_task_id, operator, kwargs) in operators.items(): + if operator in ops.TABLE_SENSORS: + table_name = ops.literal_str(kwargs.get("table_name")) + if table_name is not None: + return { + "kind": "table_update", + "table_names": [table_name], + "condition": "ANY_UPDATED", + "pause_status": "UNPAUSED", + } + return None + + +def _build_dbt_factory( + task_id: str, + task_key: str, + kwargs_list: list[dict[str, ast.expr]], + depends_on: list[Dependency] | None, +) -> DbtFactoryActivity: + """Builds a DbtFactoryActivity from cosmos config or a set of dbt CLI operators. + + Extracts project_dir / profiles_dir / target from cosmos ProjectConfig/ProfileConfig + args or dbt operator kwargs. render_mode defaults to static (the flowx-native path); + the manifest is read at package time from project_dir/target/manifest.json. + """ + project_dir = "." + profiles_dir = "dbt_profiles" + target = "dev" + for kwargs in kwargs_list: + # dbt CLI operators pass project_dir/target directly as kwargs. + project_dir = ops.literal_str(kwargs.get("project_dir")) or ops.literal_str(kwargs.get("dir")) or project_dir + profiles_dir = ops.literal_str(kwargs.get("profiles_dir")) or profiles_dir + target = ops.literal_str(kwargs.get("target")) or ops.literal_str(kwargs.get("target_name")) or target + # Cosmos nests config in ProjectConfig(...) / ProfileConfig(...) calls. + project_dir = _cosmos_project_dir(kwargs.get("project_config")) or project_dir + target = _cosmos_target(kwargs.get("profile_config")) or target + return DbtFactoryActivity( + name=task_id, + task_key=task_key, + depends_on=depends_on, + project_dir=project_dir, + profiles_dir=profiles_dir, + target=target, + render_mode="static", + ) + + +def _cosmos_project_dir(node: ast.expr | None) -> str | None: + """Extracts the dbt project path from a cosmos ``ProjectConfig(...)`` call. + + Accepts the path as the first positional arg or as ``dbt_project_path=`` / + ``project_dir=``. Returns None when *node* is not such a call. + """ + if not isinstance(node, ast.Call): + return None + if node.args: + positional = ops.literal_str(node.args[0]) + if positional: + return positional + kwargs = {kw.arg: kw.value for kw in node.keywords if kw.arg} + return ops.literal_str(kwargs.get("dbt_project_path")) or ops.literal_str(kwargs.get("project_dir")) + + +def _cosmos_target(node: ast.expr | None) -> str | None: + """Extracts ``target_name`` from a cosmos ``ProfileConfig(...)`` call.""" + if not isinstance(node, ast.Call): + return None + kwargs = {kw.arg: kw.value for kw in node.keywords if kw.arg} + return ops.literal_str(kwargs.get("target_name")) + + def discover_dags(source_path: Path) -> list[Path]: """Returns the DAG ``.py`` files under *source_path*. @@ -255,41 +659,21 @@ def load_pipelines(source_path: Path, pipeline: str | None = None) -> list[Pipel return pipelines -def _build_activity( - task_id: str, - task_key: str, - operator: str, - kwargs: dict[str, ast.expr], - functions: dict[str, ast.FunctionDef], - depends_on: list[Dependency] | None, - source: str, -) -> Activity: - """Maps one Airflow operator to an Activity IR node.""" - if operator == "PythonOperator": - callable_node = kwargs.get("python_callable") - func = functions.get(callable_node.id) if isinstance(callable_node, ast.Name) else None - if func is not None: - return NotebookActivity( - name=task_id, - task_key=task_key, - depends_on=depends_on, - notebook_path=f"notebooks/{task_key}.py", - generated_source=_notebook_source_from_callable(func, source), - ) - if operator == "BashOperator": - command = _literal_str(kwargs.get("bash_command")) - if command is not None: - return NotebookActivity( - name=task_id, - task_key=task_key, - depends_on=depends_on, - notebook_path=f"notebooks/{task_key}.py", - generated_source=_notebook_source_from_bash(task_id, command), - ) - return PlaceholderActivity( - name=task_id, - task_key=task_key, - depends_on=depends_on, - original_type=operator, - comment=f"Airflow operator '{operator}' has no deterministic flowx mapping yet.", - ) +_HOST_PATTERN = re.compile(r"https://([A-Za-z0-9._-]*(?:azuredatabricks\.net|databricks\.com|cloud\.databricks\.com))") + + +def detect_hosts(source_path: Path) -> list[str]: + """Returns Databricks workspace hosts referenced by the DAG files under *source_path*. + + Scans DAG source text for ``https://.azuredatabricks.net`` / + ``.databricks.com`` URLs (e.g. in a DatabricksNotebook/RunNow operator's host or a + connection default). Returns a sorted, de-duplicated list; empty when none are found. + """ + hosts: set[str] = set() + for dag_path in discover_dags(source_path): + try: + text = dag_path.read_text(encoding="utf-8") + except OSError: + continue + hosts.update(match.rstrip("/") for match in _HOST_PATTERN.findall(text)) + return sorted(hosts) diff --git a/src/flowx/sources/airflow/operators.py b/src/flowx/sources/airflow/operators.py new file mode 100644 index 0000000..718b836 --- /dev/null +++ b/src/flowx/sources/airflow/operators.py @@ -0,0 +1,494 @@ +"""Airflow operator -> flowx IR builders and the dispatch registry. + +Each builder maps one Airflow operator family to an :class:`~flowx.models.ir.Activity` +subclass the flowx bundler can render. flowx emits ``notebook_task`` / +``spark_python_task`` / ``spark_jar_task`` / ``run_job_task`` / ``condition_task`` / +``for_each_task`` today (no ``sql_task``), so SQL operators map to a NotebookActivity +that runs ``spark.sql(...)`` on cluster/serverless compute -- the cluster-backed path. + +Sensors and structural operators (Dummy/Empty) are classified here but handled by +the loader: file sensors lift to a job-level ``file_arrival`` trigger, time sensors +are absorbed into the schedule, and Dummy/Empty are dropped with dependency rewiring. +Operators with no deterministic mapping become a PlaceholderActivity carrying guidance. +""" + +from __future__ import annotations + +import ast +import shlex +import textwrap +from dataclasses import dataclass, field +from typing import Any, Callable + +from flowx.models.ir import ( + Activity, + NotebookActivity, + PlaceholderActivity, + RunJobActivity, + SparkJarActivity, + SparkPythonActivity, + SqlActivity, +) + +# -------------------------------------------------------------------------------------- +# Operator classification (handled specially by the loader, not via a task builder) +# -------------------------------------------------------------------------------------- + +# Removed from the graph; downstream dependencies rewired to the dropped node's upstreams. +DUMMY_OPERATORS: frozenset[str] = frozenset({"DummyOperator", "EmptyOperator"}) + +# Lift to a job-level file_arrival trigger; the sensor task itself is dropped. +FILE_SENSORS: frozenset[str] = frozenset( + {"S3KeySensor", "GCSObjectExistenceSensor", "FileSensor", "HdfsSensor", "WebHdfsSensor"} +) + +# Absorbed into the job schedule (a start-of-DAG delay); dropped with a migration note. +TIME_SENSORS: frozenset[str] = frozenset({"TimeSensor", "TimeDeltaSensor"}) + +# Lift to a job-level table_update trigger; the sensor task itself is dropped. The table +# name is read from the sensor's table_name kwarg (SQL-condition sensors without one fall +# through to a placeholder so their arbitrary condition isn't silently lost). +TABLE_SENSORS: frozenset[str] = frozenset( + {"DatabricksPartitionSensor", "DatabricksSqlSensor", "DatabricksSQLStatementsSensor", "SqlSensor"} +) + +# dbt CLI operators -> a single DbtFactoryActivity (built by the loader, which collapses +# a seed>>run>>test chain into one factory job). +DBT_CLI_OPERATORS: frozenset[str] = frozenset( + { + "DbtOperator", + "DbtRunOperator", + "DbtTestOperator", + "DbtSeedOperator", + "DbtSnapshotOperator", + "DbtBuildOperator", + "DbtDepsOperator", + } +) + +# Cosmos constructs -> DbtFactoryActivity (runtime-rendered, statically unparseable task-by-task). +COSMOS_CONSTRUCTS: frozenset[str] = frozenset({"DbtDag", "DbtTaskGroup"}) + +# dbt CLI command each dbt operator issues (for the factory's enabled types). +DBT_OPERATOR_COMMAND: dict[str, str] = { + "DbtRunOperator": "run", + "DbtTestOperator": "test", + "DbtSeedOperator": "seed", + "DbtSnapshotOperator": "snapshot", + "DbtBuildOperator": "build", +} + + +# -------------------------------------------------------------------------------------- +# AST kwarg extraction helpers +# -------------------------------------------------------------------------------------- + + +def literal_str(node: ast.expr | None) -> str | None: + """Returns the string value of a constant AST node, else None.""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + return None + + +def literal_value(node: ast.expr | None) -> Any: + """Best-effort evaluation of a literal AST node (str/num/bool/list/dict/None). + + Returns ``None`` when the node is not a compile-time literal (e.g. a name or + call), so callers treat "not a literal" and "literal None" the same way -- + acceptable for the kwargs we read. + """ + if node is None: + return None + try: + return ast.literal_eval(node) + except (ValueError, SyntaxError): + return None + + +def callable_name(node: ast.expr | None) -> str | None: + """Returns the referenced function name for ``python_callable=fn``.""" + if isinstance(node, ast.Name): + return node.id + return None + + +@dataclass(slots=True, kw_only=True) +class OperatorContext: + """Everything a builder needs to translate one operator call. + + Attributes: + task_id: The Airflow task_id. + task_key: Sanitized Databricks task key. + operator: Operator class name (e.g. ``KubernetesPodOperator``). + kwargs: The operator call's keyword arguments as AST nodes. + functions: Module-level functions (for resolving python_callable). + source: Full module source text. + call_source: The verbatim source of this operator call, embedded in a + PlaceholderActivity so the agentic-gap round can reason from it (the + Airflow analog of ADF's raw ARM JSON). + """ + + task_id: str + task_key: str + operator: str + kwargs: dict[str, ast.expr] + functions: dict[str, ast.FunctionDef] + source: str + call_source: str = "" + default_args: dict[str, ast.expr] = field(default_factory=dict) + + +# -------------------------------------------------------------------------------------- +# Notebook body generators +# -------------------------------------------------------------------------------------- + + +def _notebook_header(task_id: str, note: str) -> str: + return f"# Databricks notebook source\n# Migrated from Airflow {note} '{task_id}'.\n\n" + + +def notebook_from_callable(func: ast.FunctionDef, source: str) -> str: + """Renders a PythonOperator callable body as a notebook (dedented body statements). + + Airflow ``Variable.get(...)`` / ``BaseHook.get_connection(...)`` calls in the body are + rewritten to ``dbutils.widgets.get`` / ``dbutils.secrets.get`` so the notebook does not + reference a nonexistent Airflow metastore at runtime. + """ + from flowx.sources.airflow import templating + + segments = [ast.get_source_segment(source, stmt) for stmt in func.body] + body = textwrap.dedent("\n\n".join(seg for seg in segments if seg)) + body, _params, _notes = templating.rewrite_airflow_calls(body) + return f"# Databricks notebook source\n# Migrated from Airflow PythonOperator '{func.name}'.\n\n{body}\n" + + +def _sh_notebook(task_id: str, command: str) -> str: + lines = "".join(f"# MAGIC {line}\n" for line in command.splitlines()) + return _notebook_header(task_id, "BashOperator") + "# MAGIC %sh\n" + lines + + +# -------------------------------------------------------------------------------------- +# spark-submit parsing (BashOperator / SSHOperator wrapping spark-submit) +# -------------------------------------------------------------------------------------- + + +@dataclass(slots=True, kw_only=True) +class _SparkSubmit: + application: str | None + java_class: str | None + app_args: list[str] + + +def parse_spark_submit(command: str) -> _SparkSubmit | None: + """Parses a ``spark-submit ...`` command line into its application + args. + + Returns ``None`` when the command is not a spark-submit invocation. + """ + try: + tokens = shlex.split(command) + except ValueError: + return None + if "spark-submit" not in tokens: + return None + tokens = tokens[tokens.index("spark-submit") + 1 :] + + java_class: str | None = None + application: str | None = None + app_args: list[str] = [] + index = 0 + # spark-submit flags that take a value we skip over (cluster-side config, not app args). + valued_flags = {"--master", "--deploy-mode", "--conf", "--name", "--jars", "--packages", "--files", "--py-files"} + while index < len(tokens): + token = tokens[index] + if token == "--class" and index + 1 < len(tokens): + java_class = tokens[index + 1] + index += 2 + continue + if token in valued_flags and index + 1 < len(tokens): + index += 2 + continue + if token.startswith("--"): + index += 1 + continue + # First bare token is the application; the rest are application args. + application = token + app_args = tokens[index + 1 :] + break + return _SparkSubmit(application=application, java_class=java_class, app_args=app_args) + + +def _spark_activity_from_submit(ctx: OperatorContext, submit: _SparkSubmit, note: str) -> Activity: + """Builds a Spark JAR/Python activity from a parsed spark-submit.""" + app = submit.application or "" + if submit.java_class or app.endswith(".jar"): + activity: Activity = SparkJarActivity( + name=ctx.task_id, + task_key=ctx.task_key, + main_class_name=submit.java_class or "UNKNOWN_MAIN_CLASS", + parameters=submit.app_args or None, + libraries=[{"jar": app}] if app else None, + ) + else: + activity = SparkPythonActivity( + name=ctx.task_id, + task_key=ctx.task_key, + python_file=app or f"../src/{ctx.task_key}.py", + parameters=submit.app_args or None, + ) + return activity + + +# -------------------------------------------------------------------------------------- +# Tier 1 builders +# -------------------------------------------------------------------------------------- + + +def _build_python(ctx: OperatorContext) -> Activity: + func = ctx.functions.get(callable_name(ctx.kwargs.get("python_callable")) or "") + generated = notebook_from_callable(func, ctx.source) if func is not None else None + params = literal_value(ctx.kwargs.get("op_kwargs")) + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=generated, + base_parameters={k: str(v) for k, v in params.items()} if isinstance(params, dict) else None, + ) + + +def _build_bash(ctx: OperatorContext) -> Activity: + command = literal_str(ctx.kwargs.get("bash_command")) + if command is not None: + submit = parse_spark_submit(command) + if submit is not None: + return _spark_activity_from_submit(ctx, submit, "BashOperator spark-submit") + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=_sh_notebook(ctx.task_id, command), + ) + return _placeholder(ctx, "BashOperator command is not a string literal; supply the command manually.") + + +def _build_ssh(ctx: OperatorContext) -> Activity: + command = literal_str(ctx.kwargs.get("command")) + if command is not None: + submit = parse_spark_submit(command) + if submit is not None: + # The SSH hop is eliminated -- Databricks runs Spark natively. + return _spark_activity_from_submit(ctx, submit, "SSHOperator spark-submit") + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=_sh_notebook(ctx.task_id, command), + ) + return _placeholder(ctx, "SSHOperator command is not a string literal; supply the command manually.") + + +def _build_spark_submit(ctx: OperatorContext) -> Activity: + application = literal_str(ctx.kwargs.get("application")) or "" + java_class = literal_str(ctx.kwargs.get("java_class")) or literal_str(ctx.kwargs.get("conf")) + app_args = literal_value(ctx.kwargs.get("application_args")) + args = [str(a) for a in app_args] if isinstance(app_args, list) else None + if application.endswith(".jar") or java_class: + return SparkJarActivity( + name=ctx.task_id, + task_key=ctx.task_key, + main_class_name=java_class or "UNKNOWN_MAIN_CLASS", + parameters=args, + libraries=[{"jar": application}] if application else None, + ) + return SparkPythonActivity( + name=ctx.task_id, + task_key=ctx.task_key, + python_file=application or f"../src/{ctx.task_key}.py", + parameters=args, + ) + + +def _build_databricks_notebook(ctx: OperatorContext) -> Activity: + path = literal_str(ctx.kwargs.get("notebook_path")) or f"notebooks/{ctx.task_key}.py" + params = literal_value(ctx.kwargs.get("notebook_params")) + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=path, + base_parameters={k: str(v) for k, v in params.items()} if isinstance(params, dict) else None, + ) + + +def _build_run_now(ctx: OperatorContext) -> Activity: + job_id = literal_value(ctx.kwargs.get("job_id")) + params = ( + literal_value(ctx.kwargs.get("notebook_params")) + or literal_value(ctx.kwargs.get("python_params")) + or literal_value(ctx.kwargs.get("jar_params")) + ) + return RunJobActivity( + name=ctx.task_id, + task_key=ctx.task_key, + job_name=ctx.task_key, + existing_job_id=str(job_id) if job_id is not None else None, + job_parameters={k: str(v) for k, v in params.items()} if isinstance(params, dict) else None, + ) + + +def _build_trigger_dag_run(ctx: OperatorContext) -> Activity: + target = literal_str(ctx.kwargs.get("trigger_dag_id")) or ctx.task_key + conf = literal_value(ctx.kwargs.get("conf")) + return RunJobActivity( + name=ctx.task_id, + task_key=ctx.task_key, + job_name=_sanitize_job_name(target), + job_parameters={k: str(v) for k, v in conf.items()} if isinstance(conf, dict) else None, + ) + + +def _build_databricks_submit_run(ctx: OperatorContext) -> Activity: + """DatabricksSubmitRunOperator: read the notebook_task path out of the json payload.""" + payload = literal_value(ctx.kwargs.get("json")) + if isinstance(payload, dict): + notebook_task = payload.get("notebook_task") + if isinstance(notebook_task, dict) and notebook_task.get("notebook_path"): + base = notebook_task.get("base_parameters") + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=str(notebook_task["notebook_path"]), + base_parameters={k: str(v) for k, v in base.items()} if isinstance(base, dict) else None, + ) + return _placeholder( + ctx, "DatabricksSubmitRunOperator json payload could not be read statically; translate the run spec manually." + ) + + +def _sql_builder(note: str, sql_kwarg: str = "sql") -> Callable[[OperatorContext], Activity]: + """Factory: build a warehouse-backed SqlActivity (sql_task) from an operator's inline SQL.""" + + def build(ctx: OperatorContext) -> Activity: + sql = literal_str(ctx.kwargs.get(sql_kwarg)) or literal_str(ctx.kwargs.get("hql")) + if sql is None: + return _placeholder(ctx, f"{ctx.operator} SQL is not a string literal; extract it manually.") + return SqlActivity(name=ctx.task_id, task_key=ctx.task_key, sql=sql) + + return build + + +def _build_copy_into(ctx: OperatorContext) -> Activity: + table = literal_str(ctx.kwargs.get("table_name")) or "" + location = literal_str(ctx.kwargs.get("file_location")) or "" + file_format = literal_str(ctx.kwargs.get("file_format")) or "CSV" + sql = f"COPY INTO {table}\nFROM '{location}'\nFILEFORMAT = {file_format}" + return SqlActivity(name=ctx.task_id, task_key=ctx.task_key, sql=sql) + + +# -------------------------------------------------------------------------------------- +# Tier 2 builders +# -------------------------------------------------------------------------------------- + + +def _build_branch(ctx: OperatorContext) -> Activity: + # The branch condition lives in a Python callable we can't reduce to left/op/right, so emit + # the evaluation as a notebook that should set a task value; wiring a condition_task on that + # value is a manual follow-up (surfaced in the placeholder comment). + func = ctx.functions.get(callable_name(ctx.kwargs.get("python_callable")) or "") + generated = notebook_from_callable(func, ctx.source) if func is not None else None + activity = NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=generated, + ) + return activity + + +def _build_virtualenv(ctx: OperatorContext) -> Activity: + func = ctx.functions.get(callable_name(ctx.kwargs.get("python_callable")) or "") + requirements = literal_value(ctx.kwargs.get("requirements")) + body = notebook_from_callable(func, ctx.source) if func is not None else _notebook_header(ctx.task_id, ctx.operator) + if isinstance(requirements, list) and requirements: + pip = " ".join(str(r) for r in requirements) + # Insert a %pip install cell after the notebook-source header. + header, _, rest = body.partition("\n\n") + body = f"{header}\n\n# MAGIC %pip install {pip}\n\n{rest}" + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=body, + ) + + +def _build_email(ctx: OperatorContext) -> Activity: + return _placeholder( + ctx, + "EmailOperator: prefer job-level email_notifications on the job/task instead of a task. " + "If a mid-DAG email is required, implement it in a notebook (smtplib) or a webhook notification.", + ) + + +# -------------------------------------------------------------------------------------- +# Fallback +# -------------------------------------------------------------------------------------- + + +def _placeholder(ctx: OperatorContext, comment: str) -> Activity: + # Carry the operator's raw source so the agentic-gap round can reason from it + # (the Airflow analog of ADF's raw ARM JSON), mirroring the ADF placeholder path. + raw_definition = {"operator": ctx.operator, "source": ctx.call_source} if ctx.call_source else None + return PlaceholderActivity( + name=ctx.task_id, + task_key=ctx.task_key, + original_type=ctx.operator, + comment=comment, + raw_definition=raw_definition, + ) + + +def build_placeholder(ctx: OperatorContext) -> Activity: + """Tier 4 fallback: an unmapped operator becomes a placeholder notebook with guidance.""" + return _placeholder( + ctx, f"Airflow operator '{ctx.operator}' has no deterministic flowx mapping; translate manually." + ) + + +def _sanitize_job_name(name: str) -> str: + import re + + key = re.sub(r"[^a-zA-Z0-9_-]", "_", name) + return re.sub(r"_+", "_", key).strip("_") or "job" + + +# -------------------------------------------------------------------------------------- +# Registry: operator name -> builder +# -------------------------------------------------------------------------------------- + +OPERATOR_REGISTRY: dict[str, Callable[[OperatorContext], Activity]] = { + # Tier 1 + "PythonOperator": _build_python, + "BranchPythonOperator": _build_branch, + "ShortCircuitOperator": _build_branch, + "BashOperator": _build_bash, + "SSHOperator": _build_ssh, + "SparkSubmitOperator": _build_spark_submit, + "DatabricksSubmitRunOperator": _build_databricks_submit_run, + "DatabricksSubmitRunDeferrableOperator": _build_databricks_submit_run, + "DatabricksRunNowOperator": _build_run_now, + "DatabricksRunNowDeferrableOperator": _build_run_now, + "DatabricksNotebookOperator": _build_databricks_notebook, + "DatabricksSqlOperator": _sql_builder("DatabricksSqlOperator"), + "DatabricksSQLStatementsOperator": _sql_builder("DatabricksSQLStatementsOperator"), + "DatabricksCopyIntoOperator": _build_copy_into, + "SQLExecuteQueryOperator": _sql_builder("SQLExecuteQueryOperator"), + "PostgresOperator": _sql_builder("PostgresOperator"), + "MySqlOperator": _sql_builder("MySqlOperator"), + "HiveOperator": _sql_builder("HiveOperator", sql_kwarg="hql"), + "TriggerDagRunOperator": _build_trigger_dag_run, + # Tier 2 + "PythonVirtualenvOperator": _build_virtualenv, + "ExternalPythonOperator": _build_virtualenv, + "EmailOperator": _build_email, +} diff --git a/src/flowx/sources/airflow/templating.py b/src/flowx/sources/airflow/templating.py new file mode 100644 index 0000000..44ac454 --- /dev/null +++ b/src/flowx/sources/airflow/templating.py @@ -0,0 +1,231 @@ +"""Airflow Jinja templating, default_args, and trigger_rule -> flowx IR helpers. + +Airflow DAGs template values with Jinja (`{{ ds }}`, `{{ params.x }}`, macros) and +carry cross-cutting task settings in `default_args` (retries, timeouts, email) and +per-edge `trigger_rule`. This module converts those to the shared IR's equivalents: +Databricks dynamic-value references, `max_retries`/`timeout_seconds`, and dependency +`outcome`s (which the preparer reduces to `run_if`). +""" + +from __future__ import annotations + +import ast +import re +from typing import Any + +# Airflow Jinja macros -> Databricks job dynamic-value references. Date macros map to the +# job start time; params/var/dag_run.conf map to job parameters the pipeline should declare. +_MACRO_TO_DAB_REF: dict[str, str] = { + "ds": "{{job.start_time.iso_date}}", + "ds_nodash": "{{job.start_time.[iso_date]}}", + "ts": "{{job.start_time.iso_datetime}}", + "ts_nodash": "{{job.start_time.iso_datetime}}", + "data_interval_start": "{{job.start_time.iso_datetime}}", + "data_interval_end": "{{job.start_time.iso_datetime}}", + "execution_date": "{{job.start_time.iso_datetime}}", + "logical_date": "{{job.start_time.iso_datetime}}", + "run_id": "{{job.id}}", +} + +# {{ params.X }} / {{ var.value.X }} / {{ dag_run.conf['X'] }} -> {{job.parameters.X}} +_PARAM_PATTERNS: list[re.Pattern[str]] = [ + re.compile(r"^params\.([A-Za-z_][A-Za-z0-9_]*)$"), + re.compile(r"^params\[['\"]([^'\"]+)['\"]\]$"), + re.compile(r"^var\.value\.([A-Za-z_][A-Za-z0-9_]*)$"), + re.compile(r"^dag_run\.conf\[['\"]([^'\"]+)['\"]\]$"), +] + +_JINJA = re.compile(r"\{\{\s*(.*?)\s*\}\}") + + +def convert_template(value: str) -> tuple[str, set[str]]: + """Converts Airflow Jinja in *value* to DAB dynamic-value references. + + Returns ``(converted_value, referenced_param_names)``. Date/system macros map + to ``{{job.start_time.*}}`` refs; ``params.X`` / ``var.value.X`` / ``dag_run.conf['X']`` + map to ``{{job.parameters.X}}`` and X is reported so the pipeline can declare it. + An unrecognised expression is left as-is (so nothing is silently corrupted). + """ + params: set[str] = set() + + def _sub(match: re.Match[str]) -> str: + expr = match.group(1).strip() + if expr in _MACRO_TO_DAB_REF: + return _MACRO_TO_DAB_REF[expr] + for pattern in _PARAM_PATTERNS: + m = pattern.match(expr) + if m: + name = m.group(1) + params.add(name) + return "{{job.parameters." + name + "}}" + return match.group(0) # unknown expression: leave untouched + + return _JINJA.sub(_sub, value), params + + +def convert_params(value: Any) -> tuple[Any, set[str]]: + """Recursively converts templates in a str / list / dict value. + + Returns ``(converted, referenced_param_names)``. Non-string leaves pass through. + """ + params: set[str] = set() + if isinstance(value, str): + converted, refs = convert_template(value) + return converted, refs + if isinstance(value, list): + out_list = [] + for item in value: + conv, refs = convert_params(item) + out_list.append(conv) + params |= refs + return out_list, params + if isinstance(value, dict): + out_dict = {} + for key, item in value.items(): + conv, refs = convert_params(item) + out_dict[key] = conv + params |= refs + return out_dict, params + return value, params + + +# -------------------------------------------------------------------------------------- +# default_args (retries / timeouts / email) +# -------------------------------------------------------------------------------------- + + +def _timedelta_seconds(node: ast.expr | None) -> int | None: + """Parses a ``timedelta(...)`` AST call into total seconds (keyword args only).""" + if not isinstance(node, ast.Call): + return None + func = node.func + name = func.attr if isinstance(func, ast.Attribute) else (func.id if isinstance(func, ast.Name) else "") + if name != "timedelta": + return None + units = {"weeks": 604800, "days": 86400, "hours": 3600, "minutes": 60, "seconds": 1, "milliseconds": 0.001} + total = 0.0 + for kw in node.keywords: + if kw.arg in units and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, (int, float)): + total += kw.value.value * units[kw.arg] + return int(total) if total > 0 else None + + +def _literal_int(node: ast.expr | None) -> int | None: + if isinstance(node, ast.Constant) and isinstance(node.value, bool): + return None + if isinstance(node, ast.Constant) and isinstance(node.value, int): + return node.value + return None + + +def retry_policy(dag_default_args: dict[str, ast.expr], task_kwargs: dict[str, ast.expr]) -> dict[str, int]: + """Returns ``max_retries`` / ``timeout_seconds`` / ``min_retry_interval_millis``. + + Per-task kwargs override DAG-level ``default_args``. ``retries`` -> max_retries, + ``execution_timeout=timedelta(...)`` -> timeout_seconds, ``retry_delay=timedelta(...)`` + -> min_retry_interval_millis. Missing values are omitted. + """ + result: dict[str, int] = {} + + def pick(key: str) -> ast.expr | None: + return task_kwargs.get(key, dag_default_args.get(key)) + + retries = _literal_int(pick("retries")) + if retries is not None and retries > 0: + result["max_retries"] = retries + + timeout = _timedelta_seconds(pick("execution_timeout")) + if timeout is not None: + result["timeout_seconds"] = timeout + + retry_delay = _timedelta_seconds(pick("retry_delay")) + if retry_delay is not None: + result["min_retry_interval_millis"] = retry_delay * 1000 + + return result + + +def email_on_failure(dag_default_args: dict[str, ast.expr], task_kwargs: dict[str, ast.expr]) -> list[str]: + """Returns email recipients when email_on_failure is set (for a job-level notification note).""" + on_failure = task_kwargs.get("email_on_failure", dag_default_args.get("email_on_failure")) + if isinstance(on_failure, ast.Constant) and on_failure.value is False: + return [] + email_node = task_kwargs.get("email", dag_default_args.get("email")) + if isinstance(email_node, ast.Constant) and isinstance(email_node.value, str): + return [email_node.value] + if isinstance(email_node, ast.List): + return [e.value for e in email_node.elts if isinstance(e, ast.Constant) and isinstance(e.value, str)] + return [] + + +# -------------------------------------------------------------------------------------- +# trigger_rule -> dependency outcome +# -------------------------------------------------------------------------------------- + +# Map Airflow trigger_rule to the outcome string the preparer's run_if reducer understands +# (Failed -> AT_LEAST_ONE_FAILED, Completed/Skipped -> ALL_DONE). Default all_success -> None. +_TRIGGER_RULE_TO_OUTCOME: dict[str, str | None] = { + "all_success": None, + "all_done": "Completed", + "all_failed": "Failed", + "one_failed": "Failed", + "one_success": None, + "none_failed": None, + "none_failed_min_one_success": None, + "none_failed_or_skipped": None, + "always": "Completed", +} + + +def trigger_rule_outcome(task_kwargs: dict[str, ast.expr]) -> str | None: + """Maps a task's ``trigger_rule`` kwarg to a dependency outcome, or None (all_success).""" + node = task_kwargs.get("trigger_rule") + rule = node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None + if rule is None: + return None + return _TRIGGER_RULE_TO_OUTCOME.get(rule) + + +# -------------------------------------------------------------------------------------- +# Airflow Variable / Connection calls in notebook bodies +# -------------------------------------------------------------------------------------- + +# Variable.get("x") / Variable.get('x', default) -> dbutils.widgets.get("x") (a job parameter). +_VARIABLE_GET = re.compile(r"""Variable\.get\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*(?:,[^)]*)?\)""") +# BaseHook.get_connection("c") / Connection.get_connection_from_secrets("c") -> flagged (needs a +# secret-scope decision), rewritten to a dbutils.secrets.get with a placeholder scope. +_CONNECTION_GET = re.compile( + r"""(?:BaseHook|Connection)\.get_connection(?:_from_secrets)?\(\s*['"]([A-Za-z_][A-Za-z0-9_.\-]*)['"]\s*\)""" +) + + +def rewrite_airflow_calls(source: str) -> tuple[str, set[str], list[str]]: + """Rewrites Airflow Variable/Connection calls in notebook-body *source*. + + - ``Variable.get("x")`` -> ``dbutils.widgets.get("x")`` (a job parameter; ``x`` is + reported so the pipeline declares it and the notebook reads it as a widget). + - ``BaseHook.get_connection("c")`` -> ``dbutils.secrets.get(scope="_scope", key="...")`` + with a note (connections need a manual secret-scope / UC-connection decision). + + Returns ``(rewritten_source, referenced_params, migration_notes)``. Unrecognised + references are left untouched. + """ + params: set[str] = set() + notes: list[str] = [] + + def _var(match: re.Match[str]) -> str: + name = match.group(1) + params.add(name) + return f'dbutils.widgets.get("{name}")' + + def _conn(match: re.Match[str]) -> str: + conn = match.group(1) + notes.append( + f"Airflow connection '{conn}' -> replace with dbutils.secrets.get(scope=..., key=...) " + f"or a Unity Catalog connection; a placeholder secret scope was emitted." + ) + return f'dbutils.secrets.get(scope="{conn}_scope", key="value") # TODO: set real scope/key' + + rewritten = _VARIABLE_GET.sub(_var, source) + rewritten = _CONNECTION_GET.sub(_conn, rewritten) + return rewritten, params, notes diff --git a/tests/unit/test_airflow_adapter_reporting.py b/tests/unit/test_airflow_adapter_reporting.py new file mode 100644 index 0000000..f101b80 --- /dev/null +++ b/tests/unit/test_airflow_adapter_reporting.py @@ -0,0 +1,64 @@ +"""Tests for source-aware inputs prompts and airflow coverage profile columns.""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +from flowx.adapter.session import MigrationInputSession +from flowx.reporting.coverage import COVERAGE_METRIC_COLUMNS, build_coverage_rows +from flowx.sources.airflow.discover import main as discover_main + + +def test_inputs_discover_airflow_prompts_for_dags_not_adf(): + session = MigrationInputSession(phase="discover", source="airflow") + options = {o.option_id: o for o in session.pending().options} + assert "airflow_source_path" in options + assert "adf_source_path" not in options + assert "DAG" in options["airflow_source_path"].prompt + + +def test_inputs_discover_adf_unchanged(): + session = MigrationInputSession(phase="discover", source="adf") + ids = {o.option_id for o in session.pending().options} + assert ids == {"adf_source_path", "adf_resource_url", "output_dir"} + + +def test_inputs_convert_airflow_uses_airflow_source_path(): + session = MigrationInputSession(phase="convert", source="airflow") + ids = {o.option_id for o in session.pending().options} + assert "airflow_source_path" in ids + assert "adf_source_path" not in ids + + +def test_inputs_default_source_is_adf(): + # Back-compat: no source arg -> ADF prompts. + session = MigrationInputSession(phase="discover") + assert any(o.option_id == "adf_source_path" for o in session.pending().options) + + +def test_airflow_profile_csv_has_all_coverage_columns(): + dag = ( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='cov') as dag:\n" + " a = PythonOperator(task_id='a', python_callable=w)\n" + " b = SomeExoticOperator(task_id='b')\n" + " a >> b\n" + ) + with tempfile.TemporaryDirectory() as tmp: + src = Path(tmp) / "dag.py" + src.write_text(dag, encoding="utf-8") + out = Path(tmp) / "out" + assert discover_main(["--source-dir", str(src), "--output-dir", str(out)]) == 0 + rows = build_coverage_rows(out / "metadata") + assert len(rows) == 1 + row = rows[0] + # Every coverage metric column is present (no silent-zero KeyErrors) ... + for column in COVERAGE_METRIC_COLUMNS: + assert column in row + # ... and the computable airflow columns carry real values, not zeros. + assert row["databricks_native_activities"] == 1 # the PythonOperator + assert row["other_activities"] == 1 # the placeholder + assert row["complexity_score"] == 4 # 1*1 + 1*3 diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py new file mode 100644 index 0000000..c02f06b --- /dev/null +++ b/tests/unit/test_airflow_operators.py @@ -0,0 +1,548 @@ +"""Unit tests for Airflow operator -> flowx IR coverage (Tier 1-4).""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +from flowx.models.ir import ( + DbtFactoryActivity, + ForEachActivity, + NotebookActivity, + PlaceholderActivity, + RunJobActivity, + SparkJarActivity, + SparkPythonActivity, + SqlActivity, +) +from flowx.sources.airflow.loader import load_airflow_dag + + +def _load(dag_source: str): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "dag.py" + path.write_text(dag_source, encoding="utf-8") + return load_airflow_dag(path) + + +def _by_key(pipeline): + return {t.task_key: t for t in pipeline.tasks} + + +# -------------------------------------------------------------------------------------- +# Tier 1 +# -------------------------------------------------------------------------------------- + + +def test_python_operator_becomes_generated_notebook(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def work():\n spark.sql('select 1')\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='work', python_callable=work)\n" + ) + task = _by_key(p)["work"] + assert isinstance(task, NotebookActivity) + assert "spark.sql('select 1')" in task.generated_source + + +def test_bash_operator_becomes_sh_notebook(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.bash import BashOperator\n" + "with DAG(dag_id='d') as dag:\n" + " t = BashOperator(task_id='clean', bash_command='rm -rf /tmp/x')\n" + ) + task = _by_key(p)["clean"] + assert isinstance(task, NotebookActivity) + assert "%sh" in task.generated_source + + +def test_bash_operator_wrapping_spark_submit_becomes_spark_task(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.bash import BashOperator\n" + "with DAG(dag_id='d') as dag:\n" + " t = BashOperator(task_id='j', bash_command='spark-submit --master yarn /opt/etl.py --date x')\n" + ) + task = _by_key(p)["j"] + assert isinstance(task, SparkPythonActivity) + assert task.python_file == "/opt/etl.py" + assert task.parameters == ["--date", "x"] + + +def test_spark_submit_python_and_jar(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.apache.spark.operators.spark_submit import SparkSubmitOperator\n" + "with DAG(dag_id='d') as dag:\n" + " a = SparkSubmitOperator(task_id='py', application='/o/e.py', application_args=['--d','1'])\n" + " b = SparkSubmitOperator(task_id='jar', application='/o/a.jar', java_class='com.X')\n" + ) + tasks = _by_key(p) + assert isinstance(tasks["py"], SparkPythonActivity) + assert tasks["py"].parameters == ["--d", "1"] + assert isinstance(tasks["jar"], SparkJarActivity) + assert tasks["jar"].main_class_name == "com.X" + assert tasks["jar"].libraries == [{"jar": "/o/a.jar"}] + + +def test_ssh_operator_spark_submit_drops_the_hop(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.ssh.operators.ssh import SSHOperator\n" + "with DAG(dag_id='d') as dag:\n" + " t = SSHOperator(task_id='r', command='spark-submit --class com.E /o/e.jar --date x')\n" + ) + task = _by_key(p)["r"] + assert isinstance(task, SparkJarActivity) + assert task.main_class_name == "com.E" + + +def test_sql_operator_becomes_sql_task(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator\n" + "with DAG(dag_id='d') as dag:\n" + " t = SQLExecuteQueryOperator(task_id='rep', sql='CREATE TABLE g AS SELECT 1')\n" + ) + task = _by_key(p)["rep"] + assert isinstance(task, SqlActivity) + assert task.sql == "CREATE TABLE g AS SELECT 1" + assert task.warehouse_ref == "${var.warehouse_id}" + + +def test_hive_operator_reads_hql_into_sql_task(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.apache.hive.operators.hive import HiveOperator\n" + "with DAG(dag_id='d') as dag:\n" + " t = HiveOperator(task_id='h', hql='SELECT * FROM t')\n" + ) + task = _by_key(p)["h"] + assert isinstance(task, SqlActivity) + assert task.sql == "SELECT * FROM t" + + +def test_copy_into_operator_becomes_sql_task(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.databricks.operators.databricks_sql import DatabricksCopyIntoOperator\n" + "with DAG(dag_id='d') as dag:\n" + " t = DatabricksCopyIntoOperator(task_id='c', table_name='bronze.raw',\n" + " file_location='s3://l/', file_format='CSV')\n" + ) + task = _by_key(p)["c"] + assert isinstance(task, SqlActivity) + assert "COPY INTO bronze.raw" in task.sql + + +def test_table_sensor_lifts_to_table_update_trigger(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.databricks.sensors.databricks_partition import DatabricksPartitionSensor\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " wait = DatabricksPartitionSensor(task_id='wait', table_name='main.silver.events')\n" + " go = PythonOperator(task_id='go', python_callable=w)\n" + " wait >> go\n" + ) + assert set(_by_key(p)) == {"go"} # sensor is not a task + assert p.schedule == { + "kind": "table_update", + "table_names": ["main.silver.events"], + "condition": "ANY_UPDATED", + "pause_status": "UNPAUSED", + } + + +def test_sql_condition_sensor_without_table_stays_placeholder(): + # A SqlSensor checking an arbitrary condition (no table_name) must NOT vanish; + # it stays as a placeholder task rather than lifting to a table trigger. + p = _load( + "from airflow import DAG\n" + "from airflow.providers.common.sql.sensors.sql import SqlSensor\n" + "with DAG(dag_id='d') as dag:\n" + " s = SqlSensor(task_id='chk', sql='SELECT COUNT(*) FROM t WHERE ready')\n" + ) + task = _by_key(p)["chk"] + assert isinstance(task, PlaceholderActivity) + assert p.schedule is None + + +def test_databricks_run_now_becomes_run_job(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.databricks.operators.databricks import DatabricksRunNowOperator\n" + "with DAG(dag_id='d') as dag:\n" + " t = DatabricksRunNowOperator(task_id='dn', job_id=999)\n" + ) + task = _by_key(p)["dn"] + assert isinstance(task, RunJobActivity) + assert task.existing_job_id == "999" + + +def test_trigger_dag_run_becomes_run_job_by_name(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.trigger_dagrun import TriggerDagRunOperator\n" + "with DAG(dag_id='d') as dag:\n" + " t = TriggerDagRunOperator(task_id='f', trigger_dag_id='other_dag', conf={'k': 'v'})\n" + ) + task = _by_key(p)["f"] + assert isinstance(task, RunJobActivity) + assert task.job_name == "other_dag" + assert task.job_parameters == {"k": "v"} + + +def test_databricks_submit_run_reads_notebook_from_json(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.databricks.operators.databricks import DatabricksSubmitRunOperator\n" + "with DAG(dag_id='d') as dag:\n" + " t = DatabricksSubmitRunOperator(task_id='s', json={'notebook_task': {'notebook_path': '/W/etl'}})\n" + ) + task = _by_key(p)["s"] + assert isinstance(task, NotebookActivity) + assert task.notebook_path == "/W/etl" + + +# -------------------------------------------------------------------------------------- +# Tier 2 +# -------------------------------------------------------------------------------------- + + +def test_dummy_operators_dropped_and_rewired(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.empty import EmptyOperator\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " start = EmptyOperator(task_id='start')\n" + " mid = PythonOperator(task_id='mid', python_callable=w)\n" + " end = EmptyOperator(task_id='end')\n" + " start >> mid >> end\n" + ) + keys = set(_by_key(p)) + assert keys == {"mid"} # start/end dropped + assert p.tasks[0].depends_on is None # mid's dropped upstream rewired away + + +def test_dummy_rewire_bridges_dependencies(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.empty import EmptyOperator\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " a = PythonOperator(task_id='a', python_callable=w)\n" + " gate = EmptyOperator(task_id='gate')\n" + " b = PythonOperator(task_id='b', python_callable=w)\n" + " a >> gate >> b\n" + ) + tasks = _by_key(p) + assert set(tasks) == {"a", "b"} + assert [d.task_key for d in tasks["b"].depends_on] == ["a"] # bridged through dropped gate + + +def test_cosmos_dbt_task_group_becomes_dbt_factory(): + p = _load( + "from airflow import DAG\n" + "from cosmos import DbtTaskGroup, ProjectConfig, ProfileConfig\n" + "with DAG(dag_id='d') as dag:\n" + " dbt = DbtTaskGroup(group_id='t', project_config=ProjectConfig('/opt/proj'),\n" + " profile_config=ProfileConfig(profile_name='p', target_name='prod'))\n" + ) + task = _by_key(p)["t"] + assert isinstance(task, DbtFactoryActivity) + assert task.project_dir == "/opt/proj" + assert task.target == "prod" + assert task.render_mode == "static" + + +def test_dbt_cli_operators_collapse_to_one_factory(): + p = _load( + "from airflow import DAG\n" + "from airflow_dbt.operators.dbt_operator import DbtSeedOperator, DbtRunOperator, DbtTestOperator\n" + "with DAG(dag_id='d') as dag:\n" + " s = DbtSeedOperator(task_id='seed', dir='/opt/proj')\n" + " r = DbtRunOperator(task_id='run', dir='/opt/proj')\n" + " t = DbtTestOperator(task_id='test', dir='/opt/proj')\n" + " s >> r >> t\n" + ) + dbt_tasks = [t for t in p.tasks if isinstance(t, DbtFactoryActivity)] + assert len(dbt_tasks) == 1 # the seed>>run>>test chain collapses into one factory job + assert dbt_tasks[0].project_dir == "/opt/proj" + + +# -------------------------------------------------------------------------------------- +# Tier 3 — sensors +# -------------------------------------------------------------------------------------- + + +def test_file_sensor_lifts_to_file_arrival_trigger(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " wait = S3KeySensor(task_id='wait', bucket_key='s3://landing/in/')\n" + " go = PythonOperator(task_id='go', python_callable=w)\n" + " wait >> go\n" + ) + assert set(_by_key(p)) == {"go"} # sensor is not a task + assert p.schedule == {"kind": "file_arrival", "url": "s3://landing/in/", "pause_status": "UNPAUSED"} + + +def test_explicit_cron_wins_over_sensor_trigger(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d', schedule_interval='0 6 * * *') as dag:\n" + " wait = S3KeySensor(task_id='wait', bucket_key='s3://x/')\n" + " go = PythonOperator(task_id='go', python_callable=w)\n" + " wait >> go\n" + ) + assert p.schedule["kind"] == "schedule" + assert p.schedule["quartz_cron_expression"] == "0 0 6 ? * *" + + +# -------------------------------------------------------------------------------------- +# Tier 4 — fallback +# -------------------------------------------------------------------------------------- + + +def test_unknown_operator_becomes_placeholder(): + p = _load("from airflow import DAG\nwith DAG(dag_id='d') as dag:\n t = SomeExoticOperator(task_id='mystery')\n") + task = _by_key(p)["mystery"] + assert isinstance(task, PlaceholderActivity) + assert task.original_type == "SomeExoticOperator" + + +def test_placeholder_carries_operator_source_for_agentic_round(): + # The placeholder must carry the operator's raw source so the agentic-gap round + # (gaps.json + merge_agentic) can reason from it, like the ADF source's ARM JSON. + p = _load( + "from airflow import DAG\n" + "with DAG(dag_id='d') as dag:\n" + " t = KubernetesPodOperator(task_id='pod', image='python:3.11')\n" + ) + task = _by_key(p)["pod"] + assert isinstance(task, PlaceholderActivity) + assert task.raw_definition is not None + assert task.raw_definition["operator"] == "KubernetesPodOperator" + assert "image='python:3.11'" in task.raw_definition["source"] + + +def test_convert_emits_gaps_json_for_unmapped_operators(): + import json + + from flowx.sources.airflow.convert import main + + with tempfile.TemporaryDirectory() as tmp: + src = Path(tmp) / "dag.py" + src.write_text( + "from airflow import DAG\n" + "with DAG(dag_id='d') as dag:\n" + " t = KubernetesPodOperator(task_id='pod', image='x')\n", + encoding="utf-8", + ) + out = Path(tmp) / "out" + assert main(["--source-dir", str(src), "--output-dir", str(out)]) == 0 + gaps = json.loads((out / ".work" / "gaps.json").read_text()) + assert len(gaps) == 1 + assert gaps[0]["activity_type"] == "KubernetesPodOperator" + assert gaps[0]["raw_definition"]["source"] + + +# -------------------------------------------------------------------------------------- +# Cross-cutting: Jinja templating, default_args, trigger_rule +# -------------------------------------------------------------------------------------- + + +def test_jinja_macros_convert_to_dab_refs_and_collect_params(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='t', python_callable=w,\n" + " op_kwargs={'date': '{{ ds }}', 'env': '{{ params.env }}'})\n" + ) + task = _by_key(p)["t"] + assert task.base_parameters == {"date": "{{job.start_time.iso_date}}", "env": "{{job.parameters.env}}"} + # The referenced param is declared on the pipeline. + assert p.parameters == [{"name": "env"}] + + +def test_default_args_apply_retries_timeout_retry_delay(): + p = _load( + "from datetime import timedelta\n" + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d', default_args={'retries': 3, 'retry_delay': timedelta(minutes=5),\n" + " 'execution_timeout': timedelta(hours=2)}) as dag:\n" + " t = PythonOperator(task_id='t', python_callable=w)\n" + ) + task = _by_key(p)["t"] + assert task.max_retries == 3 + assert task.timeout_seconds == 7200 + assert task.min_retry_interval_millis == 300000 + + +def test_per_task_retries_override_default_args(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.bash import BashOperator\n" + "with DAG(dag_id='d', default_args={'retries': 3}) as dag:\n" + " t = BashOperator(task_id='t', bash_command='echo hi', retries=7)\n" + ) + assert _by_key(p)["t"].max_retries == 7 + + +def test_trigger_rule_maps_to_dependency_outcome(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " a = PythonOperator(task_id='a', python_callable=w)\n" + " cleanup = PythonOperator(task_id='cleanup', python_callable=w, trigger_rule='all_done')\n" + " fail_only = PythonOperator(task_id='fail_only', python_callable=w, trigger_rule='one_failed')\n" + " a >> cleanup\n" + " a >> fail_only\n" + ) + tasks = _by_key(p) + assert tasks["cleanup"].depends_on[0].outcome == "Completed" # all_done -> ALL_DONE + assert tasks["fail_only"].depends_on[0].outcome == "Failed" # one_failed -> AT_LEAST_ONE_FAILED + assert tasks["a"].depends_on is None # default all_success -> no outcome + + +# -------------------------------------------------------------------------------------- +# Dynamic mapping (.expand), TaskGroup prefixing, timezone/timedelta schedules +# -------------------------------------------------------------------------------------- + + +def test_expand_becomes_for_each_task(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def w(i=None):\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " m = PythonOperator.partial(task_id='proc', python_callable=w).expand(\n" + " op_kwargs=[{'i': 1}, {'i': 2}])\n" + ) + task = _by_key(p)["proc"] + assert isinstance(task, ForEachActivity) + assert task.items_expression == '[{"i": 1}, {"i": 2}]' + assert task.inner_activities[0].task_key == "proc_iteration" + + +def test_expand_direct_call_form(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.bash import BashOperator\n" + "with DAG(dag_id='d') as dag:\n" + " m = BashOperator(task_id='run', bash_command='echo').expand(env=[{'a': 1}])\n" + ) + assert isinstance(_by_key(p)["run"], ForEachActivity) + + +def test_task_group_prefixes_member_keys(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "from airflow.utils.task_group import TaskGroup\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " with TaskGroup('extract') as extract:\n" + " r = PythonOperator(task_id='run', python_callable=w)\n" + " with TaskGroup('load') as load:\n" + " r2 = PythonOperator(task_id='run', python_callable=w)\n" + ) + keys = set(_by_key(p)) + assert keys == {"extract__run", "load__run"} # no collision + + +def test_timedelta_schedule_becomes_periodic(): + p = _load( + "from datetime import timedelta\n" + "from airflow import DAG\n" + "with DAG(dag_id='d', schedule_interval=timedelta(days=2)) as dag:\n" + " pass\n" + ) + assert p.schedule == {"kind": "periodic", "interval": 2, "unit": "DAYS", "pause_status": "UNPAUSED"} + + +def test_dag_timezone_extracted_into_cron_schedule(): + p = _load( + "from datetime import datetime\n" + "import pendulum\n" + "from airflow import DAG\n" + "with DAG(dag_id='d', schedule_interval='0 6 * * *',\n" + " start_date=datetime(2024, 1, 1, tzinfo=pendulum.timezone('Europe/Madrid'))) as dag:\n" + " pass\n" + ) + assert p.schedule["timezone_id"] == "Europe/Madrid" + + +# -------------------------------------------------------------------------------------- +# Variables / Connections in notebook bodies +# -------------------------------------------------------------------------------------- + + +def test_variable_get_rewritten_to_widget_and_declared_as_param(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "from airflow.models import Variable\n" + "def ingest():\n" + " env = Variable.get('target_env')\n" + " print(env)\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='ingest', python_callable=ingest)\n" + ) + task = _by_key(p)["ingest"] + assert 'dbutils.widgets.get("target_env")' in task.generated_source + assert "Variable.get" not in task.generated_source + assert {"name": "target_env"} in (p.parameters or []) + + +def test_connection_get_rewritten_to_secrets(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "from airflow.hooks.base import BaseHook\n" + "def ingest():\n" + " conn = BaseHook.get_connection('snowflake_default')\n" + " print(conn.host)\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='ingest', python_callable=ingest)\n" + ) + task = _by_key(p)["ingest"] + assert "dbutils.secrets.get(" in task.generated_source + assert "snowflake_default_scope" in task.generated_source + assert "BaseHook.get_connection" not in task.generated_source + + +def test_airflow_host_detection_from_dag_source(): + from flowx.sources.airflow.loader import detect_hosts + + with tempfile.TemporaryDirectory() as tmp: + src = Path(tmp) / "dag.py" + src.write_text( + "from airflow import DAG\n" + "from airflow.providers.databricks.operators.databricks import DatabricksNotebookOperator\n" + "with DAG(dag_id='h') as dag:\n" + " t = DatabricksNotebookOperator(task_id='n', notebook_path='/W/e',\n" + " host='https://ws.cloud.databricks.com/')\n", + encoding="utf-8", + ) + assert detect_hosts(src) == ["ws.cloud.databricks.com"] diff --git a/tests/unit/test_sql_task_and_table_trigger.py b/tests/unit/test_sql_task_and_table_trigger.py new file mode 100644 index 0000000..ffa87be --- /dev/null +++ b/tests/unit/test_sql_task_and_table_trigger.py @@ -0,0 +1,79 @@ +"""Unit tests for SqlActivity -> sql_task rendering and table_update triggers.""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +import yaml + +from flowx.bundler.dab_writer import pipeline_dict_to_ir, write_bundle +from flowx.ir_serde import pipeline_to_dict +from flowx.models.ir import NotebookActivity, Pipeline, SqlActivity +from flowx.preparer.workflow_preparer import prepare_workflow + + +def _bundle(pipeline: Pipeline) -> tuple[dict, Path]: + wf = prepare_workflow(pipeline) + out = Path(tempfile.mkdtemp(prefix="sqltest_")).resolve() + write_bundle(wf, out, catalog="main", schema="a") + job = yaml.safe_load(next((out / "resources").glob("*.yml")).read_text()) + return job, out + + +def test_sql_activity_renders_sql_task_with_extracted_file(): + pipeline = Pipeline( + name="p", + tasks=[ + SqlActivity( + name="rep", + task_key="rep", + sql="SELECT 1", + parameters={"run_date": "{{job.parameters.run_date}}"}, + ) + ], + ) + job, out = _bundle(pipeline) + task = list(job["resources"]["jobs"].values())[0]["tasks"][0] + assert task["sql_task"]["warehouse_id"] == "${var.warehouse_id}" + assert task["sql_task"]["file"]["path"] == "../src/sql/rep.sql" + assert task["sql_task"]["parameters"] == {"run_date": "{{job.parameters.run_date}}"} + assert (out / "src" / "sql" / "rep.sql").read_text().strip() == "SELECT 1" + + +def test_sql_task_declares_warehouse_id_variable(): + pipeline = Pipeline(name="p", tasks=[SqlActivity(name="rep", task_key="rep", sql="SELECT 1")]) + _, out = _bundle(pipeline) + dby = yaml.safe_load((out / "databricks.yml").read_text()) + assert "warehouse_id" in dby["variables"] + + +def test_sql_activity_round_trips_through_report(): + pipeline = Pipeline( + name="p", tasks=[SqlActivity(name="rep", task_key="rep", sql="SELECT 1", parameters={"d": "x"})] + ) + rehydrated, _ = pipeline_dict_to_ir(pipeline_to_dict(pipeline)) + task = rehydrated.tasks[0] + assert isinstance(task, SqlActivity) + assert task.sql == "SELECT 1" + assert task.parameters == {"d": "x"} + + +def test_table_update_trigger_renders_on_job(): + pipeline = Pipeline( + name="p", + tasks=[NotebookActivity(name="go", task_key="go", notebook_path="notebooks/go.py", generated_source="x")], + schedule={ + "kind": "table_update", + "table_names": ["main.silver.events"], + "condition": "ANY_UPDATED", + "min_time_between_triggers_seconds": 300, + "pause_status": "UNPAUSED", + }, + ) + job, _ = _bundle(pipeline) + jd = list(job["resources"]["jobs"].values())[0] + assert jd["trigger"]["table_update"]["table_names"] == ["main.silver.events"] + assert jd["trigger"]["table_update"]["condition"] == "ANY_UPDATED" + assert jd["trigger"]["table_update"]["min_time_between_triggers_seconds"] == 300 + assert "schedule" not in jd From 8d24caa5d67c966caa7cc9f4527c919e7ed452a4 Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 20 Jul 2026 12:54:10 -0700 Subject: [PATCH 25/77] Route migration sources through MCP and CI --- .github/workflows/push.yml | 2 +- src/flowx/mcp/server.py | 76 ++++++++++++++++----------- tests/unit/test_mcp_source_routing.py | 76 +++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 31 deletions(-) create mode 100644 tests/unit/test_mcp_source_routing.py diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5063847..39cfd50 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -17,7 +17,7 @@ jobs: python-version: "3.12" - name: Scrub internal proxy URLs from uv.lock run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock - - run: uv sync --frozen + - run: uv sync --frozen --extra mcp - run: make test - name: Verify requirements.txt is in sync with the lockfile run: | diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py index f21e8c9..95513ff 100644 --- a/src/flowx/mcp/server.py +++ b/src/flowx/mcp/server.py @@ -64,32 +64,40 @@ def _phase_result(result: runner.AdapterResult, output_dir: Path, **extra: Any) return payload -def _resolve_source(p: dict[str, Any], path_key: str = "adf_source_path") -> tuple[str | None, Callable[[], None]]: - """Resolve the ADF source for a command into a local path the adapter can read. +def _source_name(p: dict[str, Any]) -> str: + """The migration source for a command (``adf`` default; ``airflow`` when requested).""" + return str(p.get("source", "adf")) - Input modes, in priority order — a hosted app can't read the user's files directly, so it relies - on the first three: + +def _resolve_source(p: dict[str, Any], path_key: str | None = None) -> tuple[str | None, Callable[[], None]]: + """Resolve the migration source for a command into a local path the adapter can read. + + ADF input modes, in priority order — a hosted app can't read the user's files directly, so it + relies on the first three: 1. ``adf_volume_path`` — a UC Volume directory; the server downloads it via the SDK Files API. - 2. ``adf_workspace_path`` — a ``/Workspace`` directory (e.g. an ADF Git folder); the server - downloads it via the SDK Workspace API. - Both (1) and (2) scale to large factories — the bytes bypass the agent. Each returns a temp - dir + cleanup. + 2. ``adf_workspace_path`` — a ``/Workspace`` directory (e.g. an ADF Git folder); downloaded via + the SDK Workspace API. Both (1) and (2) scale to large factories — the bytes bypass the agent. 3. ``adf_definitions`` — an inline ARM-JSON payload (small jobs); materialized to a temp dir. - 4. ``path_key`` (``adf_source_path`` / ``source_dir``) — a path the server itself can read - (local hosting or a mounted volume). + 4. ``_source_path`` / explicit ``path_key`` — a path the server itself can read. + + For ``source="airflow"`` the volume/workspace/inline modes are ADF-specific and skipped; the DAG + path is read from ``airflow_source_path`` (or the explicit ``path_key``). """ - if p.get("adf_volume_path"): - src = runner.download_volume_dir(p["adf_volume_path"]) - return src, lambda: runner.cleanup_materialized(src) - if p.get("adf_workspace_path"): - src = runner.download_workspace_dir(p["adf_workspace_path"]) - return src, lambda: runner.cleanup_materialized(src) - definitions = p.get("adf_definitions") - if definitions: - src = runner.materialize_adf_definitions(definitions) - return src, lambda: runner.cleanup_materialized(src) - return p.get(path_key), (lambda: None) + source = _source_name(p) + if source == "adf": + if p.get("adf_volume_path"): + src = runner.download_volume_dir(p["adf_volume_path"]) + return src, lambda: runner.cleanup_materialized(src) + if p.get("adf_workspace_path"): + src = runner.download_workspace_dir(p["adf_workspace_path"]) + return src, lambda: runner.cleanup_materialized(src) + definitions = p.get("adf_definitions") + if definitions: + src = runner.materialize_adf_definitions(definitions) + return src, lambda: runner.cleanup_materialized(src) + default_key = path_key or f"{source}_source_path" + return p.get(default_key), (lambda: None) def _bundle_output(p: dict[str, Any], out: Path) -> dict[str, Any]: @@ -130,17 +138,21 @@ def _pending_options(inspect_result: dict[str, Any]) -> list[dict[str, Any]]: def _cmd_inputs(p: dict[str, Any]) -> dict[str, Any]: - result = runner.run_adapter(["inputs", p["phase"]]) + result = runner.run_adapter(["inputs", p["phase"], "--source", _source_name(p)]) return {"ok": result.ok, "inputs": runner.parse_stdout_json(result), "process": result.as_dict()} def _cmd_discover(p: dict[str, Any]) -> dict[str, Any]: output_dir = p.get("output_dir", "./flowx_output") + source_name = _source_name(p) source, cleanup = _resolve_source(p) if not source: - return {"ok": False, "error": "Provide 'adf_definitions' (inline ARM JSON) or 'adf_source_path'."} + return { + "ok": False, + "error": f"Provide a source path for source '{source_name}' (e.g. '{source_name}_source_path').", + } try: - args = ["discover", "--adf-source-path", source, "--output-dir", output_dir] + args = ["discover", "--source", source_name, "--source-path", source, "--output-dir", output_dir] if p.get("pipeline"): args += ["--pipeline", p["pipeline"]] result = runner.run_adapter(args) @@ -152,11 +164,12 @@ def _cmd_discover(p: dict[str, Any]) -> dict[str, Any]: def _cmd_convert(p: dict[str, Any]) -> dict[str, Any]: output_dir = p.get("output_dir", "./flowx_output") + source_name = _source_name(p) source, cleanup = _resolve_source(p) try: - args = ["convert", "--output-dir", output_dir] + args = ["convert", "--source", source_name, "--output-dir", output_dir] if source: - args += ["--adf-source-path", source] + args += ["--source-path", source] if p.get("pipeline"): args += ["--pipeline", p["pipeline"]] result = runner.run_adapter(args) @@ -200,7 +213,7 @@ def _cmd_materialize_lookup(p: dict[str, Any]) -> dict[str, Any]: def _cmd_workspace_paths(p: dict[str, Any]) -> dict[str, Any]: - args: list[Any] = ["workspace-paths", p["report_path"]] + args: list[Any] = ["workspace-paths", p["report_path"], "--source", _source_name(p)] source, cleanup = _resolve_source(p, path_key="source_dir") try: if source: @@ -252,6 +265,7 @@ def _cmd_migrate(p: dict[str, Any]) -> dict[str, Any]: prompt and package with defaults. """ output_dir = p.get("output_dir", "./flowx_output") + source_name = _source_name(p) catalog = p.get("catalog", "main") schema = p.get("schema", "default") pipeline = p.get("pipeline") @@ -272,10 +286,12 @@ def _cmd_migrate(p: dict[str, Any]) -> dict[str, Any]: return { "ok": False, "error": ( - "Provide 'adf_volume_path' / 'adf_workspace_path' / 'adf_definitions' / 'adf_source_path'." + f"Provide a source path for source '{source_name}' " + "(adf: adf_volume_path / adf_workspace_path / adf_definitions / adf_source_path; " + "airflow: airflow_source_path)." ), } - discover_args = ["discover", "--adf-source-path", source, "--output-dir", output_dir] + discover_args = ["discover", "--source", source_name, "--source-path", source, "--output-dir", output_dir] if pipeline: discover_args += ["--pipeline", pipeline] discover_res = runner.run_adapter(discover_args) @@ -283,7 +299,7 @@ def _cmd_migrate(p: dict[str, Any]) -> dict[str, Any]: if not discover_res.ok: return {"ok": False, "status": "failed", "failed_phase": "discover", "steps": steps} - convert_args = ["convert", "--output-dir", output_dir, "--adf-source-path", source] + convert_args = ["convert", "--source", source_name, "--output-dir", output_dir, "--source-path", source] if pipeline: convert_args += ["--pipeline", pipeline] convert_res = runner.run_adapter(convert_args) diff --git a/tests/unit/test_mcp_source_routing.py b/tests/unit/test_mcp_source_routing.py new file mode 100644 index 0000000..5ecdfd5 --- /dev/null +++ b/tests/unit/test_mcp_source_routing.py @@ -0,0 +1,76 @@ +"""Tests that the MCP dispatcher threads --source to the adapter for both sources. + +Guards the P0 regression where the adapter began requiring --source but the MCP commands +never passed it (breaking ADF and never supporting Airflow). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytest.importorskip("mcp") + +from flowx.mcp import runner, server # noqa: E402 + + +class _FakeResult: + ok = True + stdout = "" + stderr = "" + returncode = 0 + + def as_dict(self) -> dict[str, object]: + return {"returncode": 0, "stdout": "", "stderr": ""} + + +@pytest.fixture +def captured(monkeypatch): + """Records the full argv of every adapter invocation.""" + calls: list[list[str]] = [] + + def fake_run_adapter(args, **_kwargs): + calls.append([str(a) for a in args]) + return _FakeResult() + + monkeypatch.setattr(runner, "run_adapter", fake_run_adapter) + monkeypatch.setattr(runner, "summarize_inventory", lambda out: {}) + monkeypatch.setattr(runner, "summarize_translation", lambda out: {}) + return calls + + +def _argv(calls: list[list[str]], subcommand: str) -> list[str]: + return next(argv for argv in calls if argv and argv[0] == subcommand) + + +def test_discover_defaults_to_adf_source(captured, tmp_path: Path): + server._cmd_discover({"adf_source_path": str(tmp_path), "output_dir": str(tmp_path / "o")}) + argv = _argv(captured, "discover") + assert "--source" in argv and argv[argv.index("--source") + 1] == "adf" + assert "--source-path" in argv + + +def test_discover_routes_airflow_source(captured, tmp_path: Path): + server._cmd_discover({"source": "airflow", "airflow_source_path": str(tmp_path), "output_dir": str(tmp_path / "o")}) + argv = _argv(captured, "discover") + assert argv[argv.index("--source") + 1] == "airflow" + assert argv[argv.index("--source-path") + 1] == str(tmp_path) + + +def test_convert_threads_source(captured, tmp_path: Path): + server._cmd_convert({"source": "airflow", "airflow_source_path": str(tmp_path), "output_dir": str(tmp_path)}) + argv = _argv(captured, "convert") + assert argv[argv.index("--source") + 1] == "airflow" + + +def test_inputs_threads_source(captured): + server._cmd_inputs({"phase": "discover", "source": "airflow"}) + argv = _argv(captured, "inputs") + assert argv[argv.index("--source") + 1] == "airflow" + + +def test_discover_missing_source_path_errors_clearly(captured, tmp_path: Path): + result = server._cmd_discover({"source": "airflow", "output_dir": str(tmp_path)}) + assert result["ok"] is False + assert "airflow" in result["error"] From 84a7f02b8c476773949c0d8541dc0d8276f0c05b Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 20 Jul 2026 12:54:16 -0700 Subject: [PATCH 26/77] Validate and package generated multi-pipeline bundles --- src/flowx/bundler/dab_writer.py | 33 ++++++++++++- src/flowx/validate/bundle_invariants.py | 49 +++++++++++++++++++- tests/unit/test_bundle_invariants.py | 21 +++++++++ tests/unit/test_package_invariants.py | 61 +++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_package_invariants.py diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index bef8a33..15e1384 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -444,6 +444,22 @@ def main(argv: list[str] | None = None) -> int: all_created.extend(created) print(f" [{index + 1}/{len(workflows)}] {workflow.name}: {len(created)} files") + # Tier-0 structural check over the emitted bundle(s): duplicate task keys / job params, + # dangling depends_on, undeclared {{job.parameters.X}}, leaked YAML anchors. Source-agnostic. + from flowx.validate.bundle_invariants import check_bundle_dir, format_result + + bundle_dirs = ( + [args.output_dir / normalize_task_key(workflow.name) for workflow in workflows] + if len(workflows) > 1 + else [args.output_dir] + ) + invariant_violations = 0 + for bundle_dir in bundle_dirs: + result = check_bundle_dir(bundle_dir) + if not result.ok or result.warnings: + print(format_result(result), file=sys.stderr) + invariant_violations += len(result.violations) + if not args.keep_intermediates: work_dir = args.output_dir / ".work" if work_dir.is_dir(): @@ -453,12 +469,18 @@ def main(argv: list[str] | None = None) -> int: print(f"Pruned transient {work_dir}") print(f"\nBundle generation complete: {len(all_created)} files written to {args.output_dir}") + if invariant_violations: + print( + f"\nWARNING: {invariant_violations} bundle-invariant violation(s) above — " + "fix before `databricks bundle validate`.", + file=sys.stderr, + ) print("\nNext steps:") print(" 1. Review the generated notebooks in src/") print(" 2. Run the setup notebooks to create secrets and volumes") print(" 3. Validate the bundle: databricks bundle validate") print(" 4. Deploy: databricks bundle deploy -t dev") - return 0 + return 1 if invariant_violations else 0 def _warn(task_key: str, message: str) -> None: @@ -1394,6 +1416,15 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]: workflows.append(workflow) return workflows + if "pipelines" in report and isinstance(report["pipelines"], list): + # Multi-pipeline report ({"pipelines": [, ...]}), emitted when a source + # converts more than one pipeline/DAG at once. Each entry is a full pipeline IR dict, so it + # routes through the same single-pipeline machinery. + for entry in report["pipelines"]: + if isinstance(entry, dict) and "tasks" in entry and "name" in entry: + workflows.append(_pipeline_dict_to_workflow(entry)) + return workflows + if "translations" in report: # Aggregated translation_report.json: ``translations`` is a flat list of {pipeline, ir, status}. # Group by pipeline and route each group through _pipeline_dict_to_workflow (same machinery as the diff --git a/src/flowx/validate/bundle_invariants.py b/src/flowx/validate/bundle_invariants.py index c822802..c457a16 100644 --- a/src/flowx/validate/bundle_invariants.py +++ b/src/flowx/validate/bundle_invariants.py @@ -3,8 +3,9 @@ These guard against output that is valid YAML / valid Python but invalid as a Databricks job -- e.g. a job parameter declared twice (the duplicate-``region`` regression), a duplicate task key, a ``{{job.parameters.X}}`` reference to an -undeclared parameter, a ``depends_on`` edge to a missing task, or a leaked YAML -anchor/alias (the fingerprint of a shared mutable object reaching serialization). +undeclared parameter, a ``depends_on`` edge to a missing task, a dependency +cycle, or a leaked YAML anchor/alias (the fingerprint of a shared mutable object +reaching serialization). Run :func:`check_bundle_dir` over a generated bundle in tests (and optionally as a Tier-0 prepare step) so these never ship silently. @@ -133,9 +134,53 @@ def check_job(job_key: str, job: dict[str, Any]) -> list[BundleFinding]: message=f"depends_on references unknown task '{target}'.", ) ) + + # 5. The task dependency graph is acyclic (a cycle fails `databricks bundle validate`). + if _has_dependency_cycle(job.get("tasks") or []): + findings.append( + BundleFinding( + code="dependency_cycle", + location=where, + message="The job's task dependency graph contains a cycle.", + ) + ) return findings +def _has_dependency_cycle(tasks: list[dict[str, Any]]) -> bool: + """Returns True when the top-level ``depends_on`` graph has a cycle (Kahn's algorithm). + + Source-agnostic: operates on the emitted job's task keys and depends_on edges, so it + guards every source's output. Edges to unknown tasks are ignored here (surfaced + separately as ``dangling_depends_on``). + """ + keys: list[str] = [ + task["task_key"] for task in tasks if isinstance(task, dict) and isinstance(task.get("task_key"), str) + ] + key_set = set(keys) + in_degree: dict[str, int] = {key: 0 for key in keys} + adjacency: dict[str, set[str]] = {key: set() for key in keys} + for task in tasks: + downstream = task.get("task_key") + if not isinstance(downstream, str) or downstream not in key_set: + continue + for dep in task.get("depends_on") or []: + upstream = dep.get("task_key") + if isinstance(upstream, str) and upstream in key_set and downstream not in adjacency[upstream]: + adjacency[upstream].add(downstream) + in_degree[downstream] += 1 + queue = [key for key in keys if in_degree[key] == 0] + visited = 0 + while queue: + node = queue.pop() + visited += 1 + for successor in adjacency[node]: + in_degree[successor] -= 1 + if in_degree[successor] == 0: + queue.append(successor) + return visited != len(keys) + + def check_resource_text(text: str, *, filename: str = "") -> list[BundleFinding]: """Check one resource YAML document (raw text): anchors + per-job invariants.""" findings: list[BundleFinding] = [] diff --git a/tests/unit/test_bundle_invariants.py b/tests/unit/test_bundle_invariants.py index aed8ebc..d61a12d 100644 --- a/tests/unit/test_bundle_invariants.py +++ b/tests/unit/test_bundle_invariants.py @@ -59,3 +59,24 @@ def test_yaml_anchor_smell_flagged(): assert "yaml_anchor" in codes # and the parsed structure also trips the duplicate-parameter invariant assert "duplicate_job_parameter" in codes + + +def test_dependency_cycle_flagged(): + job = { + "tasks": [ + {"task_key": "a", "depends_on": [{"task_key": "b"}]}, + {"task_key": "b", "depends_on": [{"task_key": "a"}]}, + ] + } + assert "dependency_cycle" in _codes(check_job("p", job)) + + +def test_acyclic_chain_has_no_cycle_finding(): + job = { + "tasks": [ + {"task_key": "a"}, + {"task_key": "b", "depends_on": [{"task_key": "a"}]}, + {"task_key": "c", "depends_on": [{"task_key": "b"}]}, + ] + } + assert "dependency_cycle" not in _codes(check_job("p", job)) diff --git a/tests/unit/test_package_invariants.py b/tests/unit/test_package_invariants.py new file mode 100644 index 0000000..6a0e8ab --- /dev/null +++ b/tests/unit/test_package_invariants.py @@ -0,0 +1,61 @@ +"""Tests that the package phase runs bundle invariants (Tier-0) over its output.""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path + +from flowx.bundler.dab_writer import main as package_main + + +def _run_package(report: dict) -> int: + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + work = out / ".work" + work.mkdir(parents=True) + (work / "translation_report.json").write_text(json.dumps(report), encoding="utf-8") + return package_main(["--output-dir", str(out)]) + + +def _notebook_task(name: str, task_key: str) -> dict: + return { + "name": name, + "task_key": task_key, + "type": "NotebookActivity", + "notebook_path": f"notebooks/{name}.py", + "generated_source": "# Databricks notebook source\nprint('x')\n", + } + + +def test_package_passes_invariants_for_clean_bundle(): + report = {"name": "clean", "tasks": [_notebook_task("a", "a"), _notebook_task("b", "b")]} + assert _run_package(report) == 0 + + +def test_package_fails_on_duplicate_task_key(): + # Two tasks sharing a task_key -> duplicate_task_key violation -> non-zero exit. + report = {"name": "bad", "tasks": [_notebook_task("a", "dup"), _notebook_task("b", "dup")]} + assert _run_package(report) == 1 + + +def test_package_loads_multi_pipeline_report(): + # A {"pipelines": [...]} report (emitted for multi-DAG conversion) must package all pipelines, + # not silently produce "no pipelines found". Guards the P0 multi-DAG load crash. + from flowx.bundler.dab_writer import _load_report + + report = { + "pipelines": [ + {"name": "first", "tasks": [_notebook_task("x", "x")]}, + {"name": "second", "tasks": [_notebook_task("y", "y")]}, + ] + } + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + work = out / ".work" + work.mkdir(parents=True) + report_path = work / "translation_report.json" + report_path.write_text(json.dumps(report), encoding="utf-8") + workflows = _load_report(report_path) + assert [w.name for w in workflows] == ["first", "second"] + assert package_main(["--output-dir", str(out)]) == 0 From d77985049b42ccd9c05153e8dc6299e6e793e76b Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 20 Jul 2026 12:54:28 -0700 Subject: [PATCH 27/77] Emit runnable Airflow callables, sensors, and TaskFlow jobs --- src/flowx/preparer/workflow_preparer.py | 15 +- .../sources/airflow/callable_notebook.py | 198 ++++++ src/flowx/sources/airflow/loader.py | 669 ++++++++++++++++-- src/flowx/sources/airflow/operators.py | 364 +++++++++- src/flowx/sources/airflow/templating.py | 82 ++- .../integration/test_airflow_golden_bundle.py | 122 ++++ .../resources/airflow/golden_pipeline_dag.py | 78 ++ tests/unit/test_airflow_operators.py | 396 ++++++++++- 8 files changed, 1775 insertions(+), 149 deletions(-) create mode 100644 src/flowx/sources/airflow/callable_notebook.py create mode 100644 tests/integration/test_airflow_golden_bundle.py create mode 100644 tests/resources/airflow/golden_pipeline_dag.py diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py index 49095ff..72e782a 100644 --- a/src/flowx/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -70,11 +70,24 @@ class PreparedWorkflow: schedule: dict[str, Any] | None = None +# The DAB job ``run_if`` vocabulary. Airflow maps ``trigger_rule`` straight to one of these +# constants (see flowx.sources.airflow.templating), so they arrive as dependency outcomes and are +# passed through here. ADF's outcome vocabulary (Succeeded/Failed/Completed/Skipped) is disjoint. +_DAB_RUN_IF: frozenset[str] = frozenset( + {"ALL_SUCCESS", "ALL_DONE", "AT_LEAST_ONE_FAILED", "ALL_FAILED", "NONE_FAILED", "AT_LEAST_ONE_SUCCESS"} +) + + def run_if_from_adf_outcomes(outcomes: list[str | None]) -> str | None: - """Maps a set of ADF dependency-edge outcomes to a single DAB ``run_if``.""" + """Maps a set of dependency-edge outcomes to a single DAB ``run_if`` (None = ALL_SUCCESS).""" normalised = [outcome for outcome in outcomes if outcome] if not normalised: return None + # A DAB run_if constant (Airflow trigger_rule) passes through directly. A task's edges share one + # trigger_rule, so these are uniform; the default ALL_SUCCESS collapses to None (no run_if key). + dab = [outcome for outcome in normalised if outcome in _DAB_RUN_IF] + if dab: + return None if dab[0] == "ALL_SUCCESS" else dab[0] if any(outcome in ("Completed", "Skipped") for outcome in normalised): return "ALL_DONE" if any(outcome == "Failed" for outcome in normalised): diff --git a/src/flowx/sources/airflow/callable_notebook.py b/src/flowx/sources/airflow/callable_notebook.py new file mode 100644 index 0000000..75c584b --- /dev/null +++ b/src/flowx/sources/airflow/callable_notebook.py @@ -0,0 +1,198 @@ +"""Render an Airflow PythonOperator callable into a valid, runnable Databricks notebook. + +The callable's complete ``def`` is preserved (so early ``return``s stay legal), its +transitive module-level dependencies (helper functions, literal constants, non-Airflow +imports) are carried, and ``op_args`` / ``op_kwargs`` are passed as JSON widgets and +splatted into a call. Airflow/provider imports are dropped (they fail on Databricks); +Variable/connection access is rewritten by :func:`flowx.sources.airflow.templating.rewrite_airflow_calls`. +""" + +from __future__ import annotations + +import ast + +from flowx.sources.airflow import templating + +# Import roots that don't exist on Databricks -- never copy these into the notebook. +_AIRFLOW_IMPORT_ROOTS: frozenset[str] = frozenset({"airflow", "cosmos", "airflow_dbt"}) + + +def _module_symbols(module: ast.Module) -> tuple[dict[str, ast.stmt], dict[str, ast.stmt]]: + """Returns ``(defs, assigns)`` -- module-level function/class defs and simple constant assigns.""" + defs: dict[str, ast.stmt] = {} + assigns: dict[str, ast.stmt] = {} + for node in module.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + defs[node.name] = node + elif isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + assigns[target.id] = node + return defs, assigns + + +def _import_bindings(module: ast.Module) -> dict[str, tuple[ast.stmt, str]]: + """Maps each imported name -> (import stmt, root module) for non-Airflow import filtering.""" + bindings: dict[str, tuple[ast.stmt, str]] = {} + for node in module.body: + if isinstance(node, ast.Import): + for alias in node.names: + bound = (alias.asname or alias.name).split(".")[0] + root = alias.name.split(".")[0] + bindings[bound] = (node, root) + elif isinstance(node, ast.ImportFrom): + root = (node.module or "").split(".")[0] + for alias in node.names: + bindings[alias.asname or alias.name] = (node, root) + return bindings + + +def _names_used(node: ast.AST) -> set[str]: + """Every bare Name id loaded anywhere in *node*.""" + return {n.id for n in ast.walk(node) if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Load)} + + +def _closure( + func: ast.FunctionDef, defs: dict[str, ast.stmt], assigns: dict[str, ast.stmt] +) -> tuple[list[str], set[str]]: + """Returns transitively-referenced module symbols (defs+assigns) in source order, plus all names used. + + Walks the callable, then any helper defs/constants it references, collecting their names + too (BFS), so a helper that calls another helper is carried. + """ + ordered: list[str] = [] + seen: set[str] = set() + all_names: set[str] = set() + queue = [func.name] + seen.add(func.name) + while queue: + name = queue.pop(0) + node = defs.get(name) or assigns.get(name) + if node is None: + continue + used = _names_used(node) + all_names |= used + if name != func.name: + ordered.append(name) + for used_name in used: + if used_name not in seen and (used_name in defs or used_name in assigns): + seen.add(used_name) + queue.append(used_name) + return ordered, all_names + + +def render_definitions(func: ast.FunctionDef, source: str, *, note: str) -> str: + """Renders the callable's ``def`` plus its transitive deps as a notebook prelude (no invocation). + + Carried: an ``import json`` line, the non-Airflow imports the callable/helpers use, the + referenced module-level helpers/constants, and *func* verbatim. Variable/connection access is + rewritten. The caller appends its own invocation (a splatted call, a poll loop, ...). + """ + module = ast.parse(source) + defs, assigns = _module_symbols(module) + imports = _import_bindings(module) + + dep_names, used_names = _closure(func, defs, assigns) + + lines: list[str] = ["# Databricks notebook source", f"# Migrated from Airflow {note} '{func.name}'.", ""] + + # 1. Carried non-Airflow imports the callable/helpers actually use. + import_segments: list[str] = [] + emitted_import_nodes: set[int] = set() + for name in sorted(used_names): + binding = imports.get(name) + if binding is None: + continue + stmt, root = binding + if root in _AIRFLOW_IMPORT_ROOTS or id(stmt) in emitted_import_nodes: + continue + emitted_import_nodes.add(id(stmt)) + segment = ast.get_source_segment(source, stmt) + if segment: + import_segments.append(segment) + lines.append("import json") + lines.extend(sorted(import_segments)) + lines.append("") + + # 2. Carried helper defs / constants, in module source order. + for name in dep_names: + node = defs.get(name) or assigns.get(name) + segment = ast.get_source_segment(source, node) if node is not None else None + if segment: + lines.append(segment) + lines.append("") + + # 3. The callable itself, verbatim (keeps early returns valid). + func_segment = ast.get_source_segment(source, func) or "" + lines.append(func_segment) + lines.append("") + + prelude = "\n".join(lines) + "\n" + # Rewrite Variable.get / BaseHook.get_connection in the emitted definitions. + rewritten, _params, _notes = templating.rewrite_airflow_calls(prelude) + return rewritten + + +def render(func: ast.FunctionDef, source: str, *, op_args: bool, op_kwargs: bool) -> str: + """Renders *func* (a PythonOperator callable) as a notebook body. + + Args: + func: The callable's FunctionDef. + source: Full DAG module source (for slicing dependency segments). + op_args / op_kwargs: Whether the operator supplied op_args / op_kwargs (drives the + JSON-widget call form). + + Returns: + Notebook source: carried imports + constants + helpers + the ``def`` + a widget-driven call. + """ + prelude = render_definitions(func, source, note="PythonOperator") + + lines: list[str] = [] + # Widget-driven invocation. op_args/op_kwargs arrive as JSON so lists/dicts survive. + call_prefix = "result = " if _returns_value(func) else "" + if op_args: + lines.append("op_args = json.loads(dbutils.widgets.get('__flowx_op_args'))") + if op_kwargs: + lines.append("op_kwargs = json.loads(dbutils.widgets.get('__flowx_op_kwargs'))") + call_args = ", ".join(filter(None, ["*op_args" if op_args else "", "**op_kwargs" if op_kwargs else ""])) + lines.append(f"{call_prefix}{func.name}({call_args})") + if call_prefix: + lines.append("dbutils.jobs.taskValues.set(key='return_value', value=result)") + + return prelude + "\n".join(lines) + "\n" + + +def _returns_value(func: ast.FunctionDef) -> bool: + """True when the callable has a ``return `` (a value consumed downstream).""" + for node in ast.walk(func): + if isinstance(node, ast.Return) and node.value is not None: + return True + return False + + +# Airflow injects execution context (the templated context dict, the task instance ``ti``, XCom) +# into a callable at runtime. flowx runs the callable as a plain notebook with no Airflow runtime, +# so a callable that reads task context or XCom cannot be lowered deterministically. +_TASK_CONTEXT_PARAMS: frozenset[str] = frozenset({"ti", "task_instance"}) +_XCOM_METHODS: frozenset[str] = frozenset({"xcom_pull", "xcom_push"}) + + +def task_context_reason(func: ast.FunctionDef) -> str | None: + """Returns a short reason if *func* depends on Airflow task context / XCom, else None. + + Detects a ``**context`` / ``**kwargs`` catch-all (Airflow passes the whole templated context + dict there), a ``ti`` / ``task_instance`` parameter, and ``xcom_pull`` / ``xcom_push`` calls. + These make the callable unrunnable as a plain notebook, so the caller routes it to a placeholder + for manual/agentic translation rather than emitting code that fails at runtime. + """ + args = func.args + if args.kwarg is not None: + return f"callable takes **{args.kwarg.arg} (Airflow task context)" + named = {a.arg for a in (args.posonlyargs + args.args + args.kwonlyargs)} + hit = sorted(named & _TASK_CONTEXT_PARAMS) + if hit: + return f"callable takes the '{hit[0]}' task-instance parameter" + for node in ast.walk(func): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr in _XCOM_METHODS: + return f"callable calls {node.func.attr}() (XCom)" + return None diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py index fce3bf4..123778d 100644 --- a/src/flowx/sources/airflow/loader.py +++ b/src/flowx/sources/airflow/loader.py @@ -11,7 +11,8 @@ :mod:`flowx.sources.airflow.operators`): Tier 1 direct mappings (Python/Bash, Spark-submit, Databricks provider, SQL, dbt CLI), Tier 2 semantic (branch/virtualenv, cosmos ``DbtTaskGroup`` -> DbtFactoryActivity, Dummy/Empty -dropped + rewired), Tier 3 sensors (file sensors -> ``file_arrival`` trigger, +dropped + rewired), Tier 3 sensors (a root file/table sensor with no schedule -> +``file_arrival`` / ``table_update`` trigger, otherwise retained as a polling task; time sensors -> schedule), and Tier 4 (unmapped -> PlaceholderActivity). ``>>`` / ``<<`` dependencies and cron ``schedule_interval`` -> Quartz are handled here. @@ -22,7 +23,9 @@ import ast import json import re +from dataclasses import dataclass, field from pathlib import Path +from typing import Any from flowx.models.ir import ( Activity, @@ -32,8 +35,25 @@ Pipeline, SqlActivity, ) +from flowx.sources.airflow import callable_notebook, templating from flowx.sources.airflow import operators as ops -from flowx.sources.airflow import templating + + +@dataclass(slots=True) +class _TaskFlowTask: + """A TaskFlow ``@task`` invocation captured from a ``@dag`` body. + + ``positional_deps`` / ``keyword_deps`` map each argument position / keyword the callable was + invoked with to the upstream task var it references (TaskFlow's implicit XCom data flow), so the + emitted notebook can read that upstream's return value via ``dbutils.jobs.taskValues``. Literal + args are ignored (the callable's own defaults apply). + """ + + task_id: str + def_name: str + decorator: str + positional_deps: dict[int, str] = field(default_factory=dict) + keyword_deps: dict[str, str] = field(default_factory=dict) def _sanitize_task_key(name: str) -> str: @@ -45,18 +65,68 @@ def _sanitize_task_key(name: str) -> str: return key or "unnamed" +def _param_default(node: ast.expr) -> Any: + """The default value of a DAG ``params`` entry: a bare literal or ``Param(default=...)``. + + Returns ``None`` when no literal default can be read (the caller emits an empty-string default so + the job parameter still validates). + """ + if isinstance(node, ast.Call): + func = node.func + name = func.attr if isinstance(func, ast.Attribute) else (func.id if isinstance(func, ast.Name) else "") + if name == "Param": + for kw in node.keywords: + if kw.arg == "default": + return ops.literal_value(kw.value) + if node.args: + return ops.literal_value(node.args[0]) + return None + return ops.literal_value(node) + + +def _shift_weekday_field(dow: str) -> str: + """Shifts Unix-cron day-of-week numbering (0-6, Sun=0) to Quartz (1-7, Sun=1). + + Airflow/Unix: 0=Sun..6=Sat (7 also = Sun). Quartz: 1=Sun..7=Sat. Each numeric token is + shifted +1, with 7 -> 1. Ranges/lists/steps (e.g. ``1-5``, ``0,3``, ``*/2``) have their + numeric components shifted individually; ``*`` / ``?`` and named days pass through. + """ + + def _shift_token(token: str) -> str: + if token.isdigit(): + n = int(token) + return "1" if n == 7 else str(n + 1) if 0 <= n <= 6 else token + return token + + # Split on commas (lists), then on '/' (steps) and '-' (ranges), shifting numeric pieces. + def _shift_part(part: str) -> str: + step = "" + if "/" in part: + part, _, step = part.partition("/") + step = "/" + step + if "-" in part: + lo, _, hi = part.partition("-") + return f"{_shift_token(lo)}-{_shift_token(hi)}{step}" + return f"{_shift_token(part)}{step}" + + return ",".join(_shift_part(p) for p in dow.split(",")) + + def _cron_to_quartz(cron: str) -> str | None: """Converts a 5-field Unix cron to a 6-field Quartz expression. Quartz is ``second minute hour day-of-month month day-of-week``; Unix cron is ``minute hour day-of-month month day-of-week``. Prepend the seconds - field and reconcile the day-of-month / day-of-week wildcard (Quartz rejects - ``*`` in both simultaneously -- one must be ``?``). + field, shift the day-of-week from Unix (0-6) to Quartz (1-7) numbering, and + reconcile the day-of-month / day-of-week wildcard (Quartz rejects ``*`` in + both simultaneously -- one must be ``?``). """ fields = cron.split() if len(fields) != 5: return None minute, hour, dom, month, dow = fields + if dow not in ("*", "?"): + dow = _shift_weekday_field(dow) if dow == "*" and dom != "*": dow = "?" elif dom == "*": @@ -161,7 +231,7 @@ class _DagVisitor(ast.NodeVisitor): def __init__(self, module: ast.Module) -> None: self._functions: dict[str, ast.FunctionDef] = { - node.name: node for node in module.body if isinstance(node, ast.FunctionDef) + node.name: node for node in _iter_functions(module) if isinstance(node, ast.FunctionDef) } # task variable name -> (task_id, operator, kwargs) self.operators: dict[str, tuple[str, str, dict[str, ast.expr]]] = {} @@ -173,15 +243,49 @@ def __init__(self, module: ast.Module) -> None: self.schedule_node: ast.expr | None = None self.timezone: str | None = None self.default_args: dict[str, ast.expr] = {} + # DAG-level params={...} defaults (param name -> literal default), so emitted job parameters + # carry a Databricks-required default rather than an empty placeholder. + self.dag_params: dict[str, Any] = {} # task variable name -> TaskGroup id prefix (for task-key namespacing) self.groups: dict[str, str] = {} self._group_stack: list[str] = [] + # `with TaskGroup(...) as tg:` binding -> the group's prefix, so a group-level edge + # (tg >> other) can expand to edges between the groups' boundary tasks. + self.group_vars: dict[str, str] = {} # task variable names defined via dynamic mapping (.expand()) -> wrapped in a for_each self.mapped: set[str] = set() + # TaskFlow: function name -> (FunctionDef, decorator dotted-name) for @task-decorated defs. + # Pre-scanned so a @task def defined after the @dag body that uses it is still resolved. + self.taskflow_defs: dict[str, tuple[ast.FunctionDef, str]] = {} + for fn in _iter_functions(module): + decorator = next( + (_decorator_name(d) for d in fn.decorator_list if _decorator_name(d) in _TASK_DECORATORS), None + ) + if decorator is not None: + self.taskflow_defs[fn.name] = (fn, decorator) + # TaskFlow task instances: var name -> _TaskFlowTask (id, def-name, decorator, arg bindings). + self.taskflow_tasks: dict[str, _TaskFlowTask] = {} + self._taskflow_counter = 0 + # A @dag-decorated function was found (so a bare `@task` file is still recognized as a DAG). + self.is_taskflow_dag: bool = False def functions(self) -> dict[str, ast.FunctionDef]: return self._functions + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + # A @task-decorated function defines a task from its callable; its body is task logic, not DAG + # structure, so don't descend. @dag marks the DAG-defining function: read its config off the + # decorator, then descend so the body's task instances / edges are collected. + if _has_decorator(node, _TASK_DECORATORS): + return + if _has_decorator(node, _DAG_DECORATORS): + self.is_taskflow_dag = True + dag_kwargs = _decorator_kwargs(node.decorator_list, _DAG_DECORATORS) + self._apply_dag_kwargs(dag_kwargs) + if self.dag_id is None: + self.dag_id = ops.literal_str(dag_kwargs.get("dag_id")) or node.name + self.generic_visit(node) + def visit_Assign(self, node: ast.Assign) -> None: if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name) and isinstance(node.value, ast.Call): var = node.targets[0].id @@ -198,8 +302,80 @@ def visit_Assign(self, node: ast.Assign) -> None: self.mapped.add(var) if self._group_stack: self.groups[var] = "__".join(self._group_stack) + elif self._register_taskflow_call(node.value, var): + pass # a `x = mytask(...)` TaskFlow invocation, captured with var as its key self.generic_visit(node) + def _taskflow_def_name(self, call: ast.Call) -> tuple[str | None, bool, str | None]: + """Resolves a call's underlying ``@task`` def name, unwrapping ``.expand`` / ``.override``. + + Returns ``(def_name_or_None, is_mapped, override_task_id)``. + """ + func = call.func + mapped = False + override_id: str | None = None + while isinstance(func, ast.Attribute): + if func.attr == "expand": + mapped = True + elif func.attr == "override": + override_id = ops.literal_str({kw.arg: kw.value for kw in call.keywords if kw.arg}.get("task_id")) + func = func.value + if isinstance(func, ast.Name) and func.id in self.taskflow_defs: + return func.id, mapped, override_id + return None, mapped, override_id + + def _register_taskflow_call(self, call: ast.Call, var: str) -> bool: + """Records a TaskFlow ``@task`` invocation as a task instance keyed by *var*. + + Binds each call argument that references (or nests) another ``@task`` to that upstream task + var -- TaskFlow's implicit XCom data flow (``transform(extract())`` wires extract -> + transform). Nested calls (``load(transform(extract()))``) register their own instances + recursively. A ``.override(task_id=...)`` renames the task. Returns True when captured. + """ + def_name, mapped, override_id = self._taskflow_def_name(call) + if def_name is None: + return False + _fn, decorator = self.taskflow_defs[def_name] + task = _TaskFlowTask(task_id=override_id or var, def_name=def_name, decorator=decorator) + self.taskflow_tasks[var] = task + self.calls[var] = call + if mapped: + self.mapped.add(var) + if self._group_stack: + self.groups[var] = "__".join(self._group_stack) + # Bind each arg that resolves to an upstream task var, and add the data-flow edge. + for index, arg in enumerate(call.args): + dep = self._resolve_taskflow_arg(arg) + if dep is not None: + task.positional_deps[index] = dep + self.edges.append((dep, var)) + for kw in call.keywords: + if kw.arg is None: + continue + dep = self._resolve_taskflow_arg(kw.value) + if dep is not None: + task.keyword_deps[kw.arg] = dep + self.edges.append((dep, var)) + return True + + def _resolve_taskflow_arg(self, arg: ast.expr) -> str | None: + """Returns the upstream task var an argument refers to, else None (a literal / unknown). + + A bare ``Name`` is an existing task var. A nested ``@task`` call (``transform(extract())``) + is registered as its own synthetic task instance and its var returned, so the whole + expression tree becomes a chain of task instances. + """ + if isinstance(arg, ast.Name): + return arg.id + if isinstance(arg, ast.Call): + def_name, _mapped, _override = self._taskflow_def_name(arg) + if def_name is not None: + self._taskflow_counter += 1 + synthetic = f"{def_name}__tf{self._taskflow_counter}" + self._register_taskflow_call(arg, synthetic) + return synthetic + return None + def visit_With(self, node: ast.With) -> None: pushed_group = False for item in node.items: @@ -217,6 +393,10 @@ def visit_With(self, node: ast.With) -> None: ) self._group_stack.append(_sanitize_task_key(group_id)) pushed_group = True + # Record the `as tg` binding (with the full nested prefix) so a group-level + # edge on `tg` resolves to the group's member tasks. + if isinstance(item.optional_vars, ast.Name): + self.group_vars[item.optional_vars.id] = "__".join(self._group_stack) elif _is_task_construct(call.func.id) and item.optional_vars is not None: # `with DbtTaskGroup(...) as g:` — a cosmos group bound to a name. if isinstance(item.optional_vars, ast.Name): @@ -232,6 +412,9 @@ def visit_With(self, node: ast.With) -> None: def _read_dag_kwargs(self, call: ast.Call) -> None: kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg} self.dag_id = ops.literal_str(kwargs.get("dag_id")) + self._apply_dag_kwargs(kwargs) + + def _apply_dag_kwargs(self, kwargs: dict[str, ast.expr]) -> None: self.schedule_node = kwargs.get("schedule_interval") or kwargs.get("schedule") self.schedule_interval = ops.literal_str(kwargs.get("schedule_interval")) or ops.literal_str( kwargs.get("schedule") @@ -245,34 +428,168 @@ def _read_dag_kwargs(self, call: ast.Call) -> None: for key, val in zip(default_args.keys, default_args.values) if isinstance(key, ast.Constant) and isinstance(key.value, str) } + # params={...} supplies DAG parameter defaults; each value is a literal or a Param(default=...). + params = kwargs.get("params") + if isinstance(params, ast.Dict): + for key, val in zip(params.keys, params.values): + if isinstance(key, ast.Constant) and isinstance(key.value, str): + self.dag_params[key.value] = _param_default(val) def visit_Expr(self, node: ast.Expr) -> None: - # Capture `a >> b >> c` and `a << b` dependency chains. - if isinstance(node.value, ast.BinOp) and isinstance(node.value.op, (ast.RShift, ast.LShift)): - self._collect_shift_chain(node.value) + # Dependency edges come from two forms: + # - shift chains: `a >> b >> c`, `a >> [b, c]`, `[a, b] >> c`, `a << b` + # - method calls: `a.set_upstream(b)` / `a.set_downstream([b, c])` + value = node.value + if isinstance(value, ast.BinOp) and isinstance(value.op, (ast.RShift, ast.LShift)): + self._collect_shift_chain(value) + elif isinstance(value, ast.Call): + # A bare TaskFlow call (`extract()` with no assignment) is a task instance keyed by its + # def name; otherwise it may be a set_upstream/set_downstream dependency call. + func = value.func + if isinstance(func, ast.Name) and func.id in self.taskflow_defs and func.id not in self.taskflow_tasks: + self._register_taskflow_call(value, func.id) + else: + self._collect_set_dependency(value) self.generic_visit(node) def _collect_shift_chain(self, binop: ast.BinOp) -> None: - names = _flatten_shift(binop) - if not names: + # Each chain position is a *group* of task names (a bare name, a [list], or an inline + # TaskFlow call like `extract()`); adjacent groups are connected as a cross-product so + # `a >> [b, c]` yields a->b and a->c. + groups = [self._shift_position_names(node) for node in _flatten_shift_nodes(binop)] + groups = [g for g in groups if g] + if len(groups) < 2: return - pairs = zip(names, names[1:]) - for left, right in pairs: - if isinstance(binop.op, ast.RShift): - self.edges.append((left, right)) - else: - self.edges.append((right, left)) + rightward = isinstance(binop.op, ast.RShift) + for upstream_group, downstream_group in zip(groups, groups[1:]): + up, down = (upstream_group, downstream_group) if rightward else (downstream_group, upstream_group) + for u in up: + for d in down: + self.edges.append((u, d)) + + def _shift_position_names(self, node: ast.expr) -> list[str]: + # A shift-chain position resolves to task vars. An inline TaskFlow call (`extract()`) is + # registered as its own instance so `prep >> finalize()` doesn't drop finalize. + if isinstance(node, (ast.List, ast.Tuple)): + names: list[str] = [] + for elt in node.elts: + names.extend(self._shift_position_names(elt)) + return names + if isinstance(node, ast.Name): + return [node.id] + if isinstance(node, ast.Call): + def_name, _mapped, _override = self._taskflow_def_name(node) + if def_name is not None: + self._taskflow_counter += 1 + synthetic = f"{def_name}__tf{self._taskflow_counter}" + self._register_taskflow_call(node, synthetic) + return [synthetic] + return [] + + def _collect_set_dependency(self, call: ast.Call) -> None: + # `x.set_upstream(y)` / `x.set_downstream(y)` where y is a Name or a list of Names. + func = call.func + if not (isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name) and call.args): + return + this = func.value.id + others = _names_in(call.args[0]) + if func.attr == "set_downstream": + self.edges.extend((this, other) for other in others) + elif func.attr == "set_upstream": + self.edges.extend((other, this) for other in others) + + +def _expand_group_edges( + edges: list[tuple[str, str]], + operators: dict[str, tuple[str, str, dict[str, ast.expr]]], + groups: dict[str, str], + group_vars: dict[str, str], +) -> list[tuple[str, str]]: + """Rewrites edges whose endpoint is a ``TaskGroup`` var into task-to-task edges. + + A group endpoint expands to its boundary tasks: as an upstream, the group's *leaves* (members + with no downstream inside the group); as a downstream, the group's *roots* (members with no + upstream inside the group). Airflow connects leaves(upstream) -> roots(downstream). A non-group + var resolves to itself. Membership includes nested subgroups (prefix match). + """ + if not group_vars: + return edges + + # Group prefix -> member task vars (a member's group prefix equals or nests under the group's). + def _members(prefix: str) -> list[str]: + return [var for var, gp in groups.items() if gp == prefix or gp.startswith(prefix + "__")] + + # Intra-group edges decide which members are roots (no in-group upstream) / leaves (no + # in-group downstream). Edges here are still in var terms. + def _roots_leaves(prefix: str) -> tuple[list[str], list[str]]: + members = set(_members(prefix)) + has_in_up = {v: False for v in members} + has_in_down = {v: False for v in members} + for up, down in edges: + if up in members and down in members: + has_in_down[up] = True + has_in_up[down] = True + roots = [v for v in members if not has_in_up[v]] + leaves = [v for v in members if not has_in_down[v]] + return roots or list(members), leaves or list(members) + + def _resolve(var: str, *, as_upstream: bool) -> list[str]: + prefix = group_vars.get(var) + if prefix is None: + return [var] + roots, leaves = _roots_leaves(prefix) + return leaves if as_upstream else roots + + expanded: list[tuple[str, str]] = [] + for up, down in edges: + if up not in group_vars and down not in group_vars: + expanded.append((up, down)) + continue + for u in _resolve(up, as_upstream=True): + for d in _resolve(down, as_upstream=False): + if u != d: + expanded.append((u, d)) + return expanded + + +def _iter_functions(module: ast.Module) -> list[ast.FunctionDef]: + """All FunctionDefs in *module*, including those nested inside a ``@dag`` function body. + + TaskFlow ``@task`` defs are often nested inside the ``@dag`` function, so a top-level-only scan + would miss them. Async defs are skipped (flowx renders sync notebooks). + """ + found: list[ast.FunctionDef] = [] + for node in ast.walk(module): + if isinstance(node, ast.FunctionDef): + found.append(node) + return found -def _flatten_shift(node: ast.expr) -> list[str]: - """Flattens a chain of ``>>`` / ``<<`` Name nodes into an ordered list.""" +def _referenced_names(node: ast.expr) -> set[str]: + """Every bare Name id loaded anywhere in *node* (for TaskFlow data-flow edge detection).""" + return {n.id for n in ast.walk(node) if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Load)} + + +def _names_in(node: ast.expr) -> list[str]: + """Returns the task-variable names in a Name or a ``[Name, ...]`` list node.""" if isinstance(node, ast.Name): return [node.id] - if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.RShift, ast.LShift)): - return _flatten_shift(node.left) + _flatten_shift(node.right) + if isinstance(node, (ast.List, ast.Tuple)): + return [elt.id for elt in node.elts if isinstance(elt, ast.Name)] return [] +def _flatten_shift_nodes(node: ast.expr) -> list[ast.expr]: + """Flattens a ``>>`` / ``<<`` chain into its per-position operand nodes, left to right. + + ``a >> [b, c] >> d()`` becomes ``[Name('a'), List([b, c]), Call(d)]``; the caller resolves each + position to task vars (registering an inline TaskFlow call along the way). + """ + if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.RShift, ast.LShift)): + return _flatten_shift_nodes(node.left) + _flatten_shift_nodes(node.right) + return [node] + + def _direct_operator_call(node: ast.Call) -> ast.Call | None: """Returns *node* if it is a direct ``SomeOperator(...)`` / ``SomeSensor(...)`` call.""" if isinstance(node.func, ast.Name) and _is_task_construct(node.func.id): @@ -321,6 +638,40 @@ def _is_task_construct(name: str) -> bool: return name.endswith("Operator") or name.endswith("Sensor") or name in ops.COSMOS_CONSTRUCTS +def _decorator_name(node: ast.expr) -> str: + """Dotted name of a decorator, ignoring call args: ``@task`` / ``@task.branch()`` -> 'task.branch'.""" + if isinstance(node, ast.Call): + node = node.func + parts: list[str] = [] + while isinstance(node, ast.Attribute): + parts.append(node.attr) + node = node.value + if isinstance(node, ast.Name): + parts.append(node.id) + return ".".join(reversed(parts)) + + +def _decorator_kwargs(decorators: list[ast.expr], names: frozenset[str]) -> dict[str, ast.expr]: + """Merged keyword args of the first decorator whose dotted name is in *names* (if it's a call).""" + for dec in decorators: + if _decorator_name(dec) in names and isinstance(dec, ast.Call): + return {kw.arg: kw.value for kw in dec.keywords if kw.arg} + return {} + + +# TaskFlow decorators. ``@dag`` marks a DAG-defining function; ``@task`` (and its variants) mark a +# task-defining function. The bare ``task`` and dotted forms (``task.branch`` / ``task.virtualenv`` / +# ``task.short_circuit`` / ``task.sensor``) all define one task from the decorated callable. +_DAG_DECORATORS: frozenset[str] = frozenset({"dag"}) +_TASK_DECORATORS: frozenset[str] = frozenset( + {"task", "task.branch", "task.virtualenv", "task.short_circuit", "task.sensor", "task.external_python"} +) + + +def _has_decorator(func: ast.FunctionDef, names: frozenset[str]) -> bool: + return any(_decorator_name(dec) in names for dec in func.decorator_list) + + def load_airflow_dag(dag_path: Path) -> Pipeline: """Parses an Airflow DAG file into a flowx Pipeline IR. @@ -347,23 +698,47 @@ def _task_key(var: str, task_id: str) -> str: key = _sanitize_task_key(task_id) return f"{visitor.groups[var]}__{key}" if var in visitor.groups else key - var_to_task_key = {var: _task_key(var, task_id) for var, (task_id, _, _) in visitor.operators.items()} + # TaskFlow @task instances share the task table with classic operators (both are just tasks with + # a task_key and dependency edges downstream). + var_task_ids: dict[str, str] = {var: tid for var, (tid, _, _) in visitor.operators.items()} + var_task_ids.update({var: tf.task_id for var, tf in visitor.taskflow_tasks.items()}) + var_to_task_key = {var: _task_key(var, tid) for var, tid in var_task_ids.items()} + + # Expand group-level edges (`group_a >> group_b`, `task >> group`, ...) into edges between the + # groups' boundary tasks: leaves of the upstream group -> roots of the downstream group, matching + # Airflow's TaskGroup dependency semantics. A non-group var resolves to itself. + edges = _expand_group_edges(visitor.edges, visitor.operators, visitor.groups, visitor.group_vars) # Build the upstream adjacency in dependency terms, then drop structural nodes # (Dummy/Empty, file/time sensors) by rewiring their downstreams to their upstreams. - upstreams: dict[str, list[str]] = {var: [] for var in visitor.operators} - for upstream_var, downstream_var in visitor.edges: + upstreams: dict[str, list[str]] = {var: [] for var in var_task_ids} + for upstream_var, downstream_var in edges: if downstream_var in upstreams and upstream_var in var_to_task_key: upstreams[downstream_var].append(upstream_var) - dropped = {var for var, (_, op, kw) in visitor.operators.items() if _is_dropped_construct(op, kw)} - upstreams = _rewire_dropped(upstreams, dropped) - - # Lift file/time sensors to a job-level trigger / schedule note (they don't become tasks). + # Sensor / schedule precedence. Airflow semantics are "run on schedule, THEN wait for data", + # and Databricks treats schedule / file_arrival / table_update as mutually-exclusive job trigger + # types -- so a data sensor lifts to a file_arrival/table_update *trigger* only when it stands at + # the DAG root AND no cron/timedelta schedule is present. With a schedule (cron AND-THEN wait) or + # mid-DAG (an ordering gate, not the DAG's entry condition), the sensor is retained as a polling + # task instead of being silently dropped. schedule = _schedule_from_interval(visitor.schedule_interval, node=visitor.schedule_node, timezone=visitor.timezone) - trigger = _trigger_from_sensors(visitor.operators) - if trigger is not None and schedule is None: - schedule = trigger + has_schedule = schedule is not None + + # Dummy/Empty and time sensors always drop (structural / absorbed into the schedule as a delay). + dropped = { + var + for var, (_, op, _) in visitor.operators.items() + if op in ops.DUMMY_OPERATORS or op in ops.TIME_SENSORS + } + if not has_schedule: + trigger_var = _root_trigger_sensor(visitor.operators, upstreams) + if trigger_var is not None: + trigger = _trigger_from_sensor(*visitor.operators[trigger_var][1:]) + if trigger is not None: + schedule = trigger + dropped.add(trigger_var) + upstreams = _rewire_dropped(upstreams, dropped) # Collapse all dbt CLI operators over the one project into a single DbtFactoryActivity. dbt_vars = [var for var, (_, op, _) in visitor.operators.items() if op in ops.DBT_CLI_OPERATORS] @@ -419,7 +794,26 @@ def _task_key(var: str, task_id: str) -> str: else: tasks.append(activity) - parameters = [{"name": name} for name in sorted(referenced_params)] or None + # TaskFlow @task instances: emit each as a notebook that reads upstream return values via + # dbutils.jobs.taskValues, calls the decorated function, and sets its own return value. + for var, tf in visitor.taskflow_tasks.items(): + task_key = var_to_task_key[var] + dep_keys = {var_to_task_key[u] for u in upstreams.get(var, []) if u in var_to_task_key} + dep_keys.discard(task_key) + depends_on = [Dependency(task_key=k) for k in sorted(dep_keys)] or None + activity = _build_taskflow_task(tf, var_to_task_key, functions, source, task_key) + activity.depends_on = depends_on + referenced_params |= _convert_activity_templates(activity) + tasks.append(activity) + + # Declare every job parameter -- those referenced in templates plus any from the DAG's + # params={...} -- each with a default (Databricks requires one): the params={...} default when + # present, else an empty string so the bundle still validates. + param_names = referenced_params | set(visitor.dag_params) + parameters = [ + {"name": name, "default": visitor.dag_params[name] if visitor.dag_params.get(name) is not None else ""} + for name in sorted(param_names) + ] or None return Pipeline( name=visitor.dag_id or Path(dag_path).stem, tasks=tasks, @@ -465,6 +859,115 @@ def _wrap_in_for_each( ) +# TaskFlow decorators that gate downstream tasks at runtime -- can't lower to a notebook (same +# reason BranchPythonOperator/ShortCircuitOperator route to the agentic round). +_TASKFLOW_BRANCHING = frozenset({"task.branch", "task.short_circuit"}) + + +def _build_taskflow_task( + tf: _TaskFlowTask, + var_to_task_key: dict[str, str], + functions: dict[str, ast.FunctionDef], + source: str, + task_key: str, +) -> Activity: + """Builds an Activity for one TaskFlow ``@task`` instance. + + The callable is rendered as a notebook that reads each upstream task's return value via + ``dbutils.jobs.taskValues.get`` (TaskFlow's implicit XCom data flow), invokes the function with + those bound arguments, and publishes its own return value. Callables that read Airflow task + context/XCom, or use a branching decorator, route to a placeholder for the agentic round. + """ + from flowx.models.ir import NotebookActivity, PlaceholderActivity + + func = functions.get(tf.def_name) + if func is None: + return PlaceholderActivity( + name=tf.task_id, + task_key=task_key, + original_type=f"@{tf.decorator}", + comment=f"TaskFlow @{tf.decorator} '{tf.def_name}' could not be resolved; translate manually.", + ) + if tf.decorator in _TASKFLOW_BRANCHING: + return PlaceholderActivity( + name=tf.task_id, + task_key=task_key, + original_type=f"@{tf.decorator}", + comment=( + f"TaskFlow @{tf.decorator} '{tf.def_name}' selects downstream tasks at runtime. " + "Translate to a Databricks condition_task and gate each downstream branch with a " + "true/false outcome dependency; do NOT run all branches." + ), + raw_definition={"operator": f"@{tf.decorator}", "source": ast.get_source_segment(source, func) or ""}, + ) + reason = callable_notebook.task_context_reason(func) + if reason is not None: + return PlaceholderActivity( + name=tf.task_id, + task_key=task_key, + original_type=f"@{tf.decorator}", + comment=( + f"TaskFlow @{tf.decorator} '{tf.def_name}' {reason}. flowx has no Airflow runtime to " + "supply it; pass upstream data via job parameters or map XCom to dbutils.jobs.taskValues." + ), + raw_definition={"operator": f"@{tf.decorator}", "source": ast.get_source_segment(source, func) or ""}, + ) + + prelude = callable_notebook.render_definitions(func, source, note=f"TaskFlow @{tf.decorator}") + body = _taskflow_invocation(func, tf, var_to_task_key) + return NotebookActivity( + name=tf.task_id, + task_key=task_key, + notebook_path=f"notebooks/{task_key}.py", + generated_source=prelude + body, + ) + + +def _taskflow_invocation(func: ast.FunctionDef, tf: _TaskFlowTask, var_to_task_key: dict[str, str]) -> str: + """The invocation cell for a TaskFlow task: read upstream taskValues, call, publish return. + + Each bound upstream task's ``return_value`` is fetched with ``dbutils.jobs.taskValues.get`` and + passed in the argument position/keyword it was wired to. Unbound parameters fall back to the + callable's own defaults. + """ + lines: list[str] = [] + call_positional: list[str] = [] + call_keywords: list[str] = [] + + def _reader(dep_var: str) -> str: + dep_key = var_to_task_key.get(dep_var, dep_var) + return f"dbutils.jobs.taskValues.get(taskKey='{dep_key}', key='return_value', debugValue=None)" + + # Positional args must stay contiguous from index 0 -- only bind a leading run of positions so a + # gap doesn't shift later args. Any remaining bound positions are passed by parameter name. + param_names = [a.arg for a in (func.args.posonlyargs + func.args.args)] + index = 0 + while index in tf.positional_deps: + var = f"_upstream_{index}" + lines.append(f"{var} = {_reader(tf.positional_deps[index])}") + call_positional.append(var) + index += 1 + for pos, dep_var in sorted(tf.positional_deps.items()): + if pos < index or pos >= len(param_names): + continue + name = param_names[pos] + var = f"_upstream_{name}" + lines.append(f"{var} = {_reader(dep_var)}") + call_keywords.append(f"{name}={var}") + for name, dep_var in tf.keyword_deps.items(): + var = f"_upstream_{name}" + lines.append(f"{var} = {_reader(dep_var)}") + call_keywords.append(f"{name}={var}") + + call_args = ", ".join(call_positional + call_keywords) + returns = any(isinstance(n, ast.Return) and n.value is not None for n in ast.walk(func)) + prefix = "result = " if returns else "" + lines.append(f"{prefix}{func.name}({call_args})") + if returns: + lines.append("dbutils.jobs.taskValues.set(key='return_value', value=result)") + return "\n".join(lines) + "\n" + + def _convert_activity_templates(activity: Activity) -> set[str]: """Converts Airflow Jinja in an activity's parameter fields to DAB refs. @@ -480,33 +983,24 @@ def _convert_activity_templates(activity: Activity) -> set[str]: setattr(activity, attr, converted) referenced |= refs if isinstance(activity, SqlActivity): - converted_sql, refs = templating.convert_template(activity.sql) - activity.sql = converted_sql - referenced |= refs + # SQL dynamic refs must go through :name markers + sql_task.parameters, not inline text. + marked_sql, sql_params = templating.convert_sql_template(activity.sql) + activity.sql = marked_sql + activity.parameters = {**(activity.parameters or {}), **sql_params} + # sql_task.parameters values that resolve to {{job.parameters.X}} need X declared. + for value in sql_params.values(): + referenced |= set(_JOB_PARAM_REF.findall(value)) # generated_source was already rewritten (Variable.get -> dbutils.widgets.get); collect the - # widget names so the pipeline declares them as job parameters. + # widget names so the pipeline declares them as job parameters. Skip the internal __flowx_* + # widgets (op_args/op_kwargs) -- those are fed by the task's base_parameters, not job params. generated = getattr(activity, "generated_source", None) if isinstance(generated, str): - referenced |= set(_WIDGET_GET.findall(generated)) + referenced |= {name for name in _WIDGET_GET.findall(generated) if not name.startswith("__flowx_")} return referenced _WIDGET_GET = re.compile(r"""dbutils\.widgets\.get\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\)""") - - -def _is_dropped_construct(operator: str, kwargs: dict[str, ast.expr]) -> bool: - """True for constructs that produce no task (lifted to a trigger/schedule or removed). - - Dummy/Empty and file/time sensors always drop. A table sensor drops only when it - names a table (it lifts to a table_update trigger); a table/SQL sensor with no - ``table_name`` is an arbitrary-condition sensor and is kept as a placeholder task - rather than silently vanishing. - """ - if operator in ops.DUMMY_OPERATORS or operator in ops.FILE_SENSORS or operator in ops.TIME_SENSORS: - return True - if operator in ops.TABLE_SENSORS: - return ops.literal_str(kwargs.get("table_name")) is not None - return False +_JOB_PARAM_REF = re.compile(r"\{\{\s*job\.parameters\.([A-Za-z0-9_]+)\s*\}\}") def _rewire_dropped(upstreams: dict[str, list[str]], dropped: set[str]) -> dict[str, list[str]]: @@ -531,34 +1025,55 @@ def resolve(var: str, seen: set[str]) -> list[str]: return {var: resolve(var, {var}) for var in upstreams if var not in dropped} -def _trigger_from_sensors(operators: dict[str, tuple[str, str, dict[str, ast.expr]]]) -> dict[str, object] | None: - """Builds a job-level trigger from the first eligible sensor. +def _root_trigger_sensor( + operators: dict[str, tuple[str, str, dict[str, ast.expr]]], + upstreams: dict[str, list[str]], +) -> str | None: + """Returns the var of a root sensor eligible to become a job-level trigger, else None. - File sensors (S3/GCS/File/HDFS) -> ``trigger.file_arrival``; table sensors with a - ``table_name`` -> ``trigger.table_update``. File sensors take precedence when both - are present. Only one trigger is emitted (DABs jobs take one); additional sensors - are left for MIGRATION_NOTES. Returns None when no eligible sensor is present. + Only a sensor with no upstreams (the DAG's entry gate) can lift to a file_arrival / + table_update trigger: mid-DAG sensors are ordering gates within the run and must stay + as tasks. File sensors win over table sensors when both sit at the root (Databricks jobs + take a single trigger). A table/SQL sensor lifts only when it names a literal table; one + without a ``table_name`` is an arbitrary-condition sensor kept as a polling task. """ - for _var, (_task_id, operator, kwargs) in operators.items(): - if operator in ops.FILE_SENSORS: - url = ( - ops.literal_str(kwargs.get("bucket_key")) - or ops.literal_str(kwargs.get("filepath")) - or ops.literal_str(kwargs.get("filepath_")) - or ops.literal_str(kwargs.get("bucket_name")) - or "" - ) - return {"kind": "file_arrival", "url": url, "pause_status": "UNPAUSED"} - for _var, (_task_id, operator, kwargs) in operators.items(): - if operator in ops.TABLE_SENSORS: - table_name = ops.literal_str(kwargs.get("table_name")) - if table_name is not None: - return { - "kind": "table_update", - "table_names": [table_name], - "condition": "ANY_UPDATED", - "pause_status": "UNPAUSED", - } + file_roots = [ + var + for var, (_id, op, _kw) in operators.items() + if op in ops.FILE_SENSORS and not upstreams.get(var) + ] + if file_roots: + return file_roots[0] + for var, (_id, op, kw) in operators.items(): + if op in ops.TABLE_SENSORS and not upstreams.get(var) and ops.literal_str(kw.get("table_name")) is not None: + return var + return None + + +def _trigger_from_sensor(operator: str, kwargs: dict[str, ast.expr]) -> dict[str, object] | None: + """Builds a job-level trigger dict from a single sensor's operator + kwargs. + + File sensors (S3/GCS/File/HDFS) -> ``trigger.file_arrival``; table sensors with a literal + ``table_name`` -> ``trigger.table_update``. Returns None when the sensor can't lift. + """ + if operator in ops.FILE_SENSORS: + url = ( + ops.literal_str(kwargs.get("bucket_key")) + or ops.literal_str(kwargs.get("filepath")) + or ops.literal_str(kwargs.get("filepath_")) + or ops.literal_str(kwargs.get("bucket_name")) + or "" + ) + return {"kind": "file_arrival", "url": url, "pause_status": "UNPAUSED"} + if operator in ops.TABLE_SENSORS: + table_name = ops.literal_str(kwargs.get("table_name")) + if table_name is not None: + return { + "kind": "table_update", + "table_names": [table_name], + "condition": "ANY_UPDATED", + "pause_status": "UNPAUSED", + } return None diff --git a/src/flowx/sources/airflow/operators.py b/src/flowx/sources/airflow/operators.py index 718b836..e45140a 100644 --- a/src/flowx/sources/airflow/operators.py +++ b/src/flowx/sources/airflow/operators.py @@ -1,22 +1,22 @@ """Airflow operator -> flowx IR builders and the dispatch registry. Each builder maps one Airflow operator family to an :class:`~flowx.models.ir.Activity` -subclass the flowx bundler can render. flowx emits ``notebook_task`` / -``spark_python_task`` / ``spark_jar_task`` / ``run_job_task`` / ``condition_task`` / -``for_each_task`` today (no ``sql_task``), so SQL operators map to a NotebookActivity -that runs ``spark.sql(...)`` on cluster/serverless compute -- the cluster-backed path. - -Sensors and structural operators (Dummy/Empty) are classified here but handled by -the loader: file sensors lift to a job-level ``file_arrival`` trigger, time sensors -are absorbed into the schedule, and Dummy/Empty are dropped with dependency rewiring. -Operators with no deterministic mapping become a PlaceholderActivity carrying guidance. +subclass the flowx bundler can render (``notebook_task`` / ``spark_python_task`` / +``spark_jar_task`` / ``sql_task`` / ``run_job_task`` / ``condition_task`` / ``for_each_task``). + +Structural operators (Dummy/Empty) and time sensors are classified here but dropped by the +loader (Dummy/Empty with dependency rewiring; time sensors absorbed into the schedule). A file +or table sensor at the DAG root with no schedule lifts to a job-level ``file_arrival`` / +``table_update`` trigger; otherwise (mid-DAG, or under a schedule) it is retained as a polling +notebook task via :func:`_build_file_sensor` / :func:`_build_table_sensor`. Operators with no +deterministic mapping become a PlaceholderActivity carrying guidance. """ from __future__ import annotations import ast +import json as _json import shlex -import textwrap from dataclasses import dataclass, field from typing import Any, Callable @@ -29,6 +29,7 @@ SparkPythonActivity, SqlActivity, ) +from flowx.sources.airflow import callable_notebook # -------------------------------------------------------------------------------------- # Operator classification (handled specially by the loader, not via a task builder) @@ -37,7 +38,8 @@ # Removed from the graph; downstream dependencies rewired to the dropped node's upstreams. DUMMY_OPERATORS: frozenset[str] = frozenset({"DummyOperator", "EmptyOperator"}) -# Lift to a job-level file_arrival trigger; the sensor task itself is dropped. +# File sensors: a root file sensor with no schedule lifts to a job-level file_arrival trigger; +# otherwise it is retained as a dbutils.fs polling task (_build_file_sensor). FILE_SENSORS: frozenset[str] = frozenset( {"S3KeySensor", "GCSObjectExistenceSensor", "FileSensor", "HdfsSensor", "WebHdfsSensor"} ) @@ -45,9 +47,9 @@ # Absorbed into the job schedule (a start-of-DAG delay); dropped with a migration note. TIME_SENSORS: frozenset[str] = frozenset({"TimeSensor", "TimeDeltaSensor"}) -# Lift to a job-level table_update trigger; the sensor task itself is dropped. The table -# name is read from the sensor's table_name kwarg (SQL-condition sensors without one fall -# through to a placeholder so their arbitrary condition isn't silently lost). +# Table/SQL sensors: a root table sensor naming a literal table with no schedule lifts to a +# job-level table_update trigger; otherwise it is retained as a spark.sql polling task +# (_build_table_sensor). A sensor with no literal sql/table_name becomes a placeholder. TABLE_SENSORS: frozenset[str] = frozenset( {"DatabricksPartitionSensor", "DatabricksSqlSensor", "DatabricksSQLStatementsSensor", "SqlSensor"} ) @@ -148,19 +150,16 @@ def _notebook_header(task_id: str, note: str) -> str: return f"# Databricks notebook source\n# Migrated from Airflow {note} '{task_id}'.\n\n" -def notebook_from_callable(func: ast.FunctionDef, source: str) -> str: - """Renders a PythonOperator callable body as a notebook (dedented body statements). +def notebook_from_callable( + func: ast.FunctionDef, source: str, *, op_args: bool = False, op_kwargs: bool = False +) -> str: + """Renders a PythonOperator callable as a valid, runnable Databricks notebook. - Airflow ``Variable.get(...)`` / ``BaseHook.get_connection(...)`` calls in the body are - rewritten to ``dbutils.widgets.get`` / ``dbutils.secrets.get`` so the notebook does not - reference a nonexistent Airflow metastore at runtime. + Preserves the callable's full ``def`` (early returns stay legal), carries its transitive + module-level dependencies (helpers / constants / non-Airflow imports), and invokes it with + ``op_args`` / ``op_kwargs`` read from JSON widgets. Variable/connection access is rewritten. """ - from flowx.sources.airflow import templating - - segments = [ast.get_source_segment(source, stmt) for stmt in func.body] - body = textwrap.dedent("\n\n".join(seg for seg in segments if seg)) - body, _params, _notes = templating.rewrite_airflow_calls(body) - return f"# Databricks notebook source\n# Migrated from Airflow PythonOperator '{func.name}'.\n\n{body}\n" + return callable_notebook.render(func, source, op_args=op_args, op_kwargs=op_kwargs) def _sh_notebook(task_id: str, command: str) -> str: @@ -168,6 +167,252 @@ def _sh_notebook(task_id: str, command: str) -> str: return _notebook_header(task_id, "BashOperator") + "# MAGIC %sh\n" + lines +# Airflow sensor defaults (seconds): poke every 60s, give up after 7 days. +_DEFAULT_POKE_INTERVAL = 60 +_DEFAULT_SENSOR_TIMEOUT = 604800 + + +def _poke_settings(kwargs: dict[str, ast.expr]) -> tuple[int, int]: + """Reads ``poke_interval`` / ``timeout`` (seconds) from a sensor's kwargs, with Airflow defaults.""" + interval = literal_value(kwargs.get("poke_interval")) + timeout = literal_value(kwargs.get("timeout")) + poke = int(interval) if isinstance(interval, (int, float)) and interval > 0 else _DEFAULT_POKE_INTERVAL + limit = int(timeout) if isinstance(timeout, (int, float)) and timeout > 0 else _DEFAULT_SENSOR_TIMEOUT + return poke, limit + + +def _poll_body(operator: str, check_expr: str, description: str, poke: int, timeout: int) -> str: + """The polling loop for a retained sensor (no notebook header / imports; callers add those). + + ``check_expr`` is a Python expression (evaluated each poke) that returns truthy when the + awaited condition holds. The loop honours the sensor's poke_interval / timeout and raises on + expiry so the task fails rather than passing silently. + """ + return ( + f"POKE_INTERVAL = {poke} # seconds\n" + + f"TIMEOUT = {timeout} # seconds\n" + + f'DESCRIPTION = "{description}"\n\n' + + "def _condition_met():\n" + + f" # {operator} poke: returns truthy once the awaited condition holds.\n" + + f" return {check_expr}\n\n" + + "deadline = time.monotonic() + TIMEOUT\n" + + "while not _condition_met():\n" + + " if time.monotonic() >= deadline:\n" + + ' raise TimeoutError(f"Sensor timed out after {TIMEOUT}s waiting for: {DESCRIPTION}")\n' + + " time.sleep(POKE_INTERVAL)\n" + + 'print(f"Condition met: {DESCRIPTION}")\n' + ) + + +def _file_sensor_path(kwargs: dict[str, ast.expr]) -> str | None: + """Best-effort literal storage path a file sensor waits on (S3/GCS/File/HDFS).""" + bucket_key = literal_str(kwargs.get("bucket_key")) + bucket_name = literal_str(kwargs.get("bucket_name")) + if bucket_key is not None: + if "://" in bucket_key or bucket_name is None: + return bucket_key + return f"s3://{bucket_name}/{bucket_key.lstrip('/')}" + obj = literal_str(kwargs.get("object")) + bucket = literal_str(kwargs.get("bucket")) + if obj is not None and bucket is not None: + return f"gs://{bucket}/{obj.lstrip('/')}" + return literal_str(kwargs.get("filepath")) or literal_str(kwargs.get("filepath_")) + + +def _build_file_sensor(ctx: OperatorContext) -> Activity: + """A retained file sensor -> a notebook that polls dbutils.fs for the awaited path.""" + path = _file_sensor_path(ctx.kwargs) + if path is None: + return _placeholder( + ctx, + f"{ctx.operator} path is not a string literal; implement the wait (poll dbutils.fs.ls " + "for the awaited object) manually, or lift it to a file_arrival trigger if it gates the DAG.", + ) + poke, timeout = _poke_settings(ctx.kwargs) + header = _notebook_header(ctx.task_id, ctx.operator) + ( + "import time\n\n" + "def _path_exists(path):\n" + " try:\n" + " dbutils.fs.ls(path)\n" + " return True\n" + " except Exception:\n" + " return False\n\n" + ) + loop = _poll_body(ctx.operator, f'_path_exists("{path}")', f"file at {path}", poke, timeout) + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=header + loop, + ) + + +def _build_table_sensor(ctx: OperatorContext) -> Activity: + """A retained table/SQL sensor -> a notebook that polls a spark.sql condition.""" + poke, timeout = _poke_settings(ctx.kwargs) + sql = literal_str(ctx.kwargs.get("sql")) + table_name = literal_str(ctx.kwargs.get("table_name")) + if sql is not None: + # SqlSensor semantics: run the query, take the first row; ready unless there are no rows or + # the first cell is falsy (0 / "0" / "" / None), matching Airflow's default success criteria. + check = "_sql_sensor_ready(SENSOR_SQL)" + header = ( + _notebook_header(ctx.task_id, ctx.operator) + + "import time\n\n" + + f"SENSOR_SQL = {sql!r}\n\n" + + "def _sql_sensor_ready(query):\n" + + " rows = spark.sql(query).take(1)\n" + + " if not rows:\n" + + " return False\n" + + " first = rows[0][0]\n" + + ' return first not in (0, "0", "", None, False)\n\n' + ) + desc = "SQL sensor condition" + elif table_name is not None: + check = f'spark.catalog.tableExists("{table_name}")' + header = _notebook_header(ctx.task_id, ctx.operator) + "import time\n\n" + desc = f"table {table_name}" + else: + return _placeholder( + ctx, + f"{ctx.operator} has no literal sql/table_name; implement the wait (poll spark.sql for the " + "awaited condition) manually.", + ) + loop = _poll_body(ctx.operator, check, desc, poke, timeout) + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=header + loop, + ) + + +def _build_external_task_sensor(ctx: OperatorContext) -> Activity: + """ExternalTaskSensor -> a notebook that waits for the external DAG's Databricks job to succeed. + + The external DAG becomes a sibling Databricks job (one bundle per DAG); this task polls that + job's most recent run via the Jobs API until it reaches a successful terminal state. The job is + referenced by the sanitized DAG name, matching TriggerDagRunOperator's RunJobActivity naming. + """ + external_dag = literal_str(ctx.kwargs.get("external_dag_id")) + if external_dag is None: + return _placeholder( + ctx, + f"{ctx.operator} external_dag_id is not a string literal; implement the cross-DAG wait " + "(poll the upstream job's run state) manually.", + ) + poke, timeout = _poke_settings(ctx.kwargs) + job_name = _sanitize_job_name(external_dag) + external_task = literal_str(ctx.kwargs.get("external_task_id")) + scope = f"task '{external_task}' in " if external_task else "" + header = _notebook_header(ctx.task_id, ctx.operator) + ( + "import time\n\n" + "from databricks.sdk import WorkspaceClient\n\n" + f"EXTERNAL_JOB_NAME = {job_name!r}\n" + "w = WorkspaceClient()\n\n" + "def _external_job_succeeded():\n" + " jobs = list(w.jobs.list(name=EXTERNAL_JOB_NAME))\n" + " if not jobs:\n" + " raise RuntimeError(f\"No Databricks job named {EXTERNAL_JOB_NAME!r}; deploy the \"\n" + " \"migrated upstream DAG's bundle first.\")\n" + " runs = list(w.jobs.list_runs(job_id=jobs[0].job_id, limit=1, completed_only=True))\n" + " if not runs:\n" + " return False\n" + " state = runs[0].state\n" + ' return state is not None and str(state.result_state) == "RunResultState.SUCCESS"\n\n' + ) + loop = _poll_body(ctx.operator, "_external_job_succeeded()", f"{scope}DAG '{external_dag}'", poke, timeout) + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=header + loop, + ) + + +def _build_http_sensor(ctx: OperatorContext) -> Activity: + """HttpSensor -> a notebook that polls an HTTP endpoint until it returns 2xx.""" + endpoint = literal_str(ctx.kwargs.get("endpoint")) or "" + if not endpoint: + return _placeholder( + ctx, + f"{ctx.operator} endpoint is not a string literal; implement the HTTP poll manually " + "(the http_conn_id base URL also needs wiring).", + ) + poke, timeout = _poke_settings(ctx.kwargs) + header = _notebook_header(ctx.task_id, ctx.operator) + ( + "import time\n\n" + "import requests\n\n" + f"ENDPOINT = {endpoint!r} # TODO: prefix with the http_conn_id base URL\n\n" + "def _endpoint_ready():\n" + " try:\n" + " return requests.get(ENDPOINT, timeout=30).ok\n" + " except requests.RequestException:\n" + " return False\n\n" + ) + loop = _poll_body(ctx.operator, "_endpoint_ready()", f"HTTP endpoint {endpoint}", poke, timeout) + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=header + loop, + ) + + +def _build_python_sensor(ctx: OperatorContext) -> Activity: + """PythonSensor -> a notebook that polls its python_callable until it returns truthy.""" + func = ctx.functions.get(callable_name(ctx.kwargs.get("python_callable")) or "") + if func is None: + return _placeholder( + ctx, + f"{ctx.operator} python_callable could not be resolved; implement the poll manually.", + ) + reason = callable_notebook.task_context_reason(func) + if reason is not None: + return _placeholder( + ctx, + f"Airflow {ctx.operator} {reason}. flowx has no Airflow runtime to supply it; implement " + "the poll condition manually.", + ) + poke, timeout = _poke_settings(ctx.kwargs) + # Emit the callable's def + deps, then poll its return value (no eager one-shot invocation). + prelude = callable_notebook.render_definitions(func, ctx.source, note=ctx.operator) + loop = _poll_body(ctx.operator, f"{func.name}()", f"{func.name}() condition", poke, timeout) + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=prelude + "import time\n\n" + loop, + ) + + +def _build_datetime_sensor(ctx: OperatorContext) -> Activity: + """DateTimeSensor -> a notebook that sleeps until a target datetime.""" + target = literal_str(ctx.kwargs.get("target_time")) + if target is None: + return _placeholder( + ctx, + f"{ctx.operator} target_time is not a string literal; implement the wait-until manually.", + ) + _poke, timeout = _poke_settings(ctx.kwargs) + header = _notebook_header(ctx.task_id, ctx.operator) + ( + "import time\n" + "from datetime import datetime, timezone\n\n" + f"TARGET_TIME = {target!r}\n\n" + "def _target_reached():\n" + " target = datetime.fromisoformat(TARGET_TIME)\n" + " now = datetime.now(target.tzinfo or timezone.utc)\n" + " return now >= target\n\n" + ) + loop = _poll_body(ctx.operator, "_target_reached()", f"datetime {target}", 60, timeout) + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=header + loop, + ) + + # -------------------------------------------------------------------------------------- # spark-submit parsing (BashOperator / SSHOperator wrapping spark-submit) # -------------------------------------------------------------------------------------- @@ -246,14 +491,37 @@ def _spark_activity_from_submit(ctx: OperatorContext, submit: _SparkSubmit, note def _build_python(ctx: OperatorContext) -> Activity: func = ctx.functions.get(callable_name(ctx.kwargs.get("python_callable")) or "") - generated = notebook_from_callable(func, ctx.source) if func is not None else None - params = literal_value(ctx.kwargs.get("op_kwargs")) + # A callable that reads Airflow task context (**context / ti) or XCom can't run as a plain + # notebook -- route it to the agentic-gap round instead of emitting code that fails at runtime. + if func is not None: + reason = callable_notebook.task_context_reason(func) + if reason is not None: + return _placeholder( + ctx, + f"Airflow {ctx.operator} {reason}. flowx has no Airflow runtime to supply it; " + "translate manually -- pass upstream data via job parameters or map XCom to " + "dbutils.jobs.taskValues (set in the producer, get in the consumer).", + ) + op_kwargs = literal_value(ctx.kwargs.get("op_kwargs")) + op_args = literal_value(ctx.kwargs.get("op_args")) + has_kwargs = isinstance(op_kwargs, dict) + has_args = isinstance(op_args, list) + generated = ( + notebook_from_callable(func, ctx.source, op_args=has_args, op_kwargs=has_kwargs) if func is not None else None + ) + # op_args/op_kwargs pass as JSON widgets so lists/numbers/nested objects survive; the notebook + # json.loads() them and splats into the call. + base_parameters: dict[str, str] = {} + if has_args: + base_parameters["__flowx_op_args"] = _json.dumps(op_args) + if has_kwargs: + base_parameters["__flowx_op_kwargs"] = _json.dumps(op_kwargs) return NotebookActivity( name=ctx.task_id, task_key=ctx.task_key, notebook_path=f"notebooks/{ctx.task_key}.py", generated_source=generated, - base_parameters={k: str(v) for k, v in params.items()} if isinstance(params, dict) else None, + base_parameters=base_parameters or None, ) @@ -391,18 +659,17 @@ def _build_copy_into(ctx: OperatorContext) -> Activity: def _build_branch(ctx: OperatorContext) -> Activity: - # The branch condition lives in a Python callable we can't reduce to left/op/right, so emit - # the evaluation as a notebook that should set a task value; wiring a condition_task on that - # value is a manual follow-up (surfaced in the placeholder comment). - func = ctx.functions.get(callable_name(ctx.kwargs.get("python_callable")) or "") - generated = notebook_from_callable(func, ctx.source) if func is not None else None - activity = NotebookActivity( - name=ctx.task_id, - task_key=ctx.task_key, - notebook_path=f"notebooks/{ctx.task_key}.py", - generated_source=generated, + # Airflow Branch/ShortCircuit gate *sibling* tasks on a Python callable's return, which flowx + # can't statically lower to a condition_task's left/op/right plus per-branch true/false outcome + # wiring. Emitting it as an ordinary notebook would silently let every downstream branch run, so + # route it to the agentic-gap round with the callable source instead (a real translation, not a + # wrong-but-quiet one). Full Branch->condition_task remains a scoped follow-up. + return _placeholder( + ctx, + f"Airflow {ctx.operator} selects downstream tasks at runtime. Translate to a Databricks " + "condition_task (or a task that sets a task value read by a condition_task) and gate each " + "downstream branch with a true/false outcome dependency; do NOT run all branches.", ) - return activity def _build_virtualenv(ctx: OperatorContext) -> Activity: @@ -492,3 +759,22 @@ def _sanitize_job_name(name: str) -> str: "ExternalPythonOperator": _build_virtualenv, "EmailOperator": _build_email, } + +# File/table sensors retained as tasks (mid-DAG, or under a schedule) poll for their condition. Root +# instances without a schedule are lifted to a job trigger by the loader before dispatch reaches here. +OPERATOR_REGISTRY.update({name: _build_file_sensor for name in FILE_SENSORS}) +OPERATOR_REGISTRY.update({name: _build_table_sensor for name in TABLE_SENSORS}) + +# Sensors that always become polling tasks (never triggers): cross-DAG, HTTP, arbitrary-callable, +# and wait-until-datetime. +OPERATOR_REGISTRY.update( + { + "ExternalTaskSensor": _build_external_task_sensor, + "ExternalTaskSensorAsync": _build_external_task_sensor, + "HttpSensor": _build_http_sensor, + "HttpSensorAsync": _build_http_sensor, + "PythonSensor": _build_python_sensor, + "DateTimeSensor": _build_datetime_sensor, + "DateTimeSensorAsync": _build_datetime_sensor, + } +) diff --git a/src/flowx/sources/airflow/templating.py b/src/flowx/sources/airflow/templating.py index 44ac454..2cc0e84 100644 --- a/src/flowx/sources/airflow/templating.py +++ b/src/flowx/sources/airflow/templating.py @@ -13,18 +13,19 @@ import re from typing import Any -# Airflow Jinja macros -> Databricks job dynamic-value references. Date macros map to the -# job start time; params/var/dag_run.conf map to job parameters the pipeline should declare. +# Airflow Jinja macros -> Databricks job dynamic-value references. Only macros with an exact +# Databricks equivalent are mapped; date macros -> the job start time, run_id -> the run id. +# ``ds_nodash``/``ts_nodash`` have no dashless dynamic-value form, so they are intentionally NOT +# mapped -- they're left untouched (surfaced as an unresolved reference) rather than emitting an +# invalid ref. _MACRO_TO_DAB_REF: dict[str, str] = { "ds": "{{job.start_time.iso_date}}", - "ds_nodash": "{{job.start_time.[iso_date]}}", "ts": "{{job.start_time.iso_datetime}}", - "ts_nodash": "{{job.start_time.iso_datetime}}", "data_interval_start": "{{job.start_time.iso_datetime}}", "data_interval_end": "{{job.start_time.iso_datetime}}", "execution_date": "{{job.start_time.iso_datetime}}", "logical_date": "{{job.start_time.iso_datetime}}", - "run_id": "{{job.id}}", + "run_id": "{{job.run_id}}", } # {{ params.X }} / {{ var.value.X }} / {{ dag_run.conf['X'] }} -> {{job.parameters.X}} @@ -63,6 +64,48 @@ def _sub(match: re.Match[str]) -> str: return _JINJA.sub(_sub, value), params +# Airflow macro -> the sql_task.parameters name + the DAB dynamic value it resolves to. Databricks +# requires dynamic references in SQL to go through named :markers + sql_task.parameters, never inline. +_SQL_MACRO_PARAM: dict[str, tuple[str, str]] = { + "ds": ("run_date", "{{job.start_time.iso_date}}"), + "ts": ("run_timestamp", "{{job.start_time.iso_datetime}}"), + "data_interval_start": ("data_interval_start", "{{job.start_time.iso_datetime}}"), + "data_interval_end": ("data_interval_end", "{{job.start_time.iso_datetime}}"), + "execution_date": ("execution_date", "{{job.start_time.iso_datetime}}"), + "logical_date": ("logical_date", "{{job.start_time.iso_datetime}}"), + "run_id": ("run_id", "{{job.run_id}}"), +} + + +def convert_sql_template(sql: str) -> tuple[str, dict[str, str]]: + """Rewrites Airflow Jinja in *sql* to ``:name`` markers + a ``sql_task.parameters`` map. + + Databricks requires dynamic references in a ``sql_task`` to be passed through named parameters, + not interpolated into the SQL text. ``{{ ds }}`` -> ``:run_date`` with + ``{"run_date": "{{job.start_time.iso_date}}"}``; ``{{ params.x }}`` -> ``:x`` with + ``{"x": "{{job.parameters.x}}"}``. Unknown expressions are left untouched. + + Returns ``(sql_with_markers, parameters)``. + """ + parameters: dict[str, str] = {} + + def _sub(match: re.Match[str]) -> str: + expr = match.group(1).strip() + if expr in _SQL_MACRO_PARAM: + name, ref = _SQL_MACRO_PARAM[expr] + parameters[name] = ref + return f":{name}" + for pattern in _PARAM_PATTERNS: + m = pattern.match(expr) + if m: + name = m.group(1) + parameters[name] = "{{job.parameters." + name + "}}" + return f":{name}" + return match.group(0) + + return _JINJA.sub(_sub, sql), parameters + + def convert_params(value: Any) -> tuple[Any, set[str]]: """Recursively converts templates in a str / list / dict value. @@ -162,28 +205,31 @@ def email_on_failure(dag_default_args: dict[str, ast.expr], task_kwargs: dict[st # trigger_rule -> dependency outcome # -------------------------------------------------------------------------------------- -# Map Airflow trigger_rule to the outcome string the preparer's run_if reducer understands -# (Failed -> AT_LEAST_ONE_FAILED, Completed/Skipped -> ALL_DONE). Default all_success -> None. -_TRIGGER_RULE_TO_OUTCOME: dict[str, str | None] = { +# Map Airflow trigger_rule -> the DAB job ``run_if`` constant carried as a dependency outcome. The +# preparer's reducer passes these through unchanged. Rules with no exact DAB equivalent fall back to +# the closest safe constant: none_failed_min_one_success -> AT_LEAST_ONE_SUCCESS (both require >=1 +# success and no upstream failure); none_failed_or_skipped -> NONE_FAILED (skips are non-failures). +# Airflow's default all_success maps to None (no run_if key -> Databricks default ALL_SUCCESS). +_TRIGGER_RULE_TO_RUN_IF: dict[str, str | None] = { "all_success": None, - "all_done": "Completed", - "all_failed": "Failed", - "one_failed": "Failed", - "one_success": None, - "none_failed": None, - "none_failed_min_one_success": None, - "none_failed_or_skipped": None, - "always": "Completed", + "all_done": "ALL_DONE", + "all_failed": "ALL_FAILED", + "one_failed": "AT_LEAST_ONE_FAILED", + "one_success": "AT_LEAST_ONE_SUCCESS", + "none_failed": "NONE_FAILED", + "none_failed_min_one_success": "AT_LEAST_ONE_SUCCESS", + "none_failed_or_skipped": "NONE_FAILED", + "always": "ALL_DONE", } def trigger_rule_outcome(task_kwargs: dict[str, ast.expr]) -> str | None: - """Maps a task's ``trigger_rule`` kwarg to a dependency outcome, or None (all_success).""" + """Maps a task's ``trigger_rule`` kwarg to a DAB ``run_if`` constant, or None (ALL_SUCCESS).""" node = task_kwargs.get("trigger_rule") rule = node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None if rule is None: return None - return _TRIGGER_RULE_TO_OUTCOME.get(rule) + return _TRIGGER_RULE_TO_RUN_IF.get(rule) # -------------------------------------------------------------------------------------- diff --git a/tests/integration/test_airflow_golden_bundle.py b/tests/integration/test_airflow_golden_bundle.py new file mode 100644 index 0000000..063d881 --- /dev/null +++ b/tests/integration/test_airflow_golden_bundle.py @@ -0,0 +1,122 @@ +"""Golden-bundle test for the Airflow source. + +Converts a single representative DAG (tests/resources/airflow/golden_pipeline_dag.py) all the +way to a DAB bundle on disk and pins the emitted job YAML + notebooks. It guards the Phase-2 +conversion behaviours together, end-to-end, so a regression in any one of them fails here: + + - cron schedule + a root file sensor -> schedule kept AND sensor retained as a polling task + (schedule / file_arrival triggers are mutually exclusive on a Databricks job) + - a mid-DAG table sensor -> polling task, never a trigger + - trigger_rule -> DAB run_if constants (ALL_DONE, AT_LEAST_ONE_FAILED, ...) + - params={...} -> job-parameter defaults; {{ params.x }} -> {{job.parameters.x}} + - Unix cron day-of-week -> Quartz (Mon: 1 -> 2) + - >> chains and set_upstream() dependency forms +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest +import yaml + +from flowx.bundler.dab_writer import write_bundle +from flowx.preparer.workflow_preparer import prepare_workflow +from flowx.sources.airflow.loader import load_airflow_dag +from flowx.validate.bundle_invariants import check_bundle_dir + +_DAG = Path(__file__).parent.parent / "resources" / "airflow" / "golden_pipeline_dag.py" + + +@pytest.fixture(scope="module") +def bundle_dir(tmp_path_factory) -> Path: + out = tmp_path_factory.mktemp("golden_bundle") + workflow = prepare_workflow(load_airflow_dag(_DAG)) + write_bundle(workflow, out) + return out + + +@pytest.fixture(scope="module") +def job_def(bundle_dir: Path) -> dict: + doc = yaml.safe_load((bundle_dir / "resources" / "golden_pipeline.yml").read_text()) + return doc["resources"]["jobs"]["golden_pipeline"] + + +def _task(job_def: dict, key: str) -> dict: + return next(t for t in job_def["tasks"] if t["task_key"] == key) + + +def test_all_expected_tasks_present(job_def: dict): + # Both sensors are retained as tasks (not dropped, not lifted to triggers). + keys = {t["task_key"] for t in job_def["tasks"]} + assert keys == { + "wait_landing", + "ingest_orders", + "wait_partition", + "publish_metrics", + "cleanup", + "alert_on_failure", + } + + +def test_cron_schedule_kept_with_quartz_weekday_shift(job_def: dict): + # cron survives the presence of the root sensor; Unix DOW 1 (Mon) -> Quartz 2. + schedule = job_def["schedule"] + assert schedule["quartz_cron_expression"] == "0 0 6 ? * 2" + assert schedule["timezone_id"] == "UTC" + # No mutually-exclusive job trigger was emitted alongside the schedule. + assert "trigger" not in job_def + + +def test_root_file_sensor_is_polling_task(bundle_dir: Path): + src = (bundle_dir / "src" / "notebooks" / "wait_landing.py").read_text() + assert "dbutils.fs.ls" in src + assert "s3://acme-orders/landing/" in src + assert "POKE_INTERVAL = 120" in src + assert "TIMEOUT = 3600" in src + ast.parse(src) + + +def test_mid_dag_table_sensor_is_polling_task(bundle_dir: Path): + src = (bundle_dir / "src" / "notebooks" / "wait_partition.py").read_text() + assert 'spark.catalog.tableExists("main.analytics.raw_orders")' in src + assert "POKE_INTERVAL = 60" in src + ast.parse(src) + + +def test_trigger_rules_map_to_run_if(job_def: dict): + assert _task(job_def, "cleanup")["run_if"] == "ALL_DONE" + assert _task(job_def, "alert_on_failure")["run_if"] == "AT_LEAST_ONE_FAILED" + # Default all_success tasks carry no run_if key. + assert "run_if" not in _task(job_def, "ingest_orders") + + +def test_dependencies_from_both_shift_and_set_upstream(job_def: dict): + # >> chain + assert [d["task_key"] for d in _task(job_def, "ingest_orders")["depends_on"]] == ["wait_landing"] + assert [d["task_key"] for d in _task(job_def, "wait_partition")["depends_on"]] == ["ingest_orders"] + # set_upstream() edges + assert [d["task_key"] for d in _task(job_def, "cleanup")["depends_on"]] == ["publish_metrics"] + assert [d["task_key"] for d in _task(job_def, "alert_on_failure")["depends_on"]] == ["publish_metrics"] + + +def test_job_parameters_carry_defaults(job_def: dict): + params = {p["name"]: p["default"] for p in job_def["parameters"]} + assert params["target_env"] == "prod" # from Param("prod") + assert params["threshold"] == 100 # bare literal default + + +def test_templated_param_becomes_dab_ref(job_def: dict): + base = _task(job_def, "ingest_orders")["notebook_task"]["base_parameters"] + assert base["__flowx_op_kwargs"] == '{"target_env": "{{job.parameters.target_env}}"}' + + +def test_all_notebooks_are_valid_python(bundle_dir: Path): + for notebook in (bundle_dir / "src" / "notebooks").glob("*.py"): + ast.parse(notebook.read_text()) + + +def test_bundle_passes_invariants(bundle_dir: Path): + result = check_bundle_dir(bundle_dir) + assert result.ok, "\n".join(f"{f.severity}: {f.message}" for f in result.findings) diff --git a/tests/resources/airflow/golden_pipeline_dag.py b/tests/resources/airflow/golden_pipeline_dag.py new file mode 100644 index 0000000..a77ec84 --- /dev/null +++ b/tests/resources/airflow/golden_pipeline_dag.py @@ -0,0 +1,78 @@ +"""Golden-bundle fixture DAG for the airflow source. + +Exercises the Phase-2 conversion behaviours in one representative DAG so the golden test +pins their end-to-end bundle output: + - cron schedule AND a root file sensor -> schedule kept + sensor retained as a polling task + - a mid-DAG table sensor -> polling task (not a trigger) + - trigger_rule variety -> DAB run_if constants + - params={...} -> job-parameter defaults; {{ params.x }} -> {{job.parameters.x}} + - >> and set_upstream dependency forms +Parsed statically by flowx.sources.airflow.loader (no Airflow install required). +""" + +from datetime import datetime + +from airflow import DAG +from airflow.models.param import Param +from airflow.operators.python import PythonOperator +from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor +from airflow.providers.databricks.sensors.databricks_partition import DatabricksPartitionSensor + + +def ingest_orders(target_env=None): + df = spark.read.json(f"s3://acme-orders/{target_env}/raw/") + df.write.mode("append").saveAsTable("main.analytics.raw_orders") + + +def publish_metrics(): + daily = spark.table("main.analytics.raw_orders").groupBy("order_date").count() + daily.write.mode("overwrite").saveAsTable("main.analytics.daily_order_metrics") + + +with DAG( + dag_id="golden_pipeline", + schedule_interval="0 6 * * 1", + start_date=datetime(2024, 1, 1), + catchup=False, + params={"target_env": Param("prod"), "threshold": 100}, +) as dag: + wait_landing = S3KeySensor( + task_id="wait_landing", + bucket_key="s3://acme-orders/landing/", + poke_interval=120, + timeout=3600, + ) + + ingest = PythonOperator( + task_id="ingest_orders", + python_callable=ingest_orders, + op_kwargs={"target_env": "{{ params.target_env }}"}, + ) + + wait_partition = DatabricksPartitionSensor( + task_id="wait_partition", + table_name="main.analytics.raw_orders", + poke_interval=60, + timeout=1800, + ) + + publish = PythonOperator( + task_id="publish_metrics", + python_callable=publish_metrics, + ) + + cleanup = PythonOperator( + task_id="cleanup", + python_callable=publish_metrics, + trigger_rule="all_done", + ) + + alert_on_failure = PythonOperator( + task_id="alert_on_failure", + python_callable=publish_metrics, + trigger_rule="one_failed", + ) + + wait_landing >> ingest >> wait_partition >> publish + cleanup.set_upstream(publish) + alert_on_failure.set_upstream(publish) diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py index c02f06b..219c9e9 100644 --- a/tests/unit/test_airflow_operators.py +++ b/tests/unit/test_airflow_operators.py @@ -47,6 +47,76 @@ def test_python_operator_becomes_generated_notebook(): assert "spark.sql('select 1')" in task.generated_source +def test_python_operator_notebook_is_valid_python(): + # A callable with an early return, a helper, a constant, and a non-Airflow import must + # produce a notebook that compiles (the review's P1 blind spot: top-level return / undefined names). + p = _load( + "from datetime import datetime\n" + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "CONST = 10\n" + "def _double(x):\n return x * 2\n" + "def process(factor=1):\n" + " n = _double(factor) + CONST\n" + " if n > 100:\n return 'big'\n" + " return datetime.now().isoformat()\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='proc', python_callable=process, op_kwargs={'factor': 5})\n" + ) + nb = _by_key(p)["proc"].generated_source + compile(nb, "", "exec") # raises SyntaxError if invalid + assert "def process(factor=1):" in nb # def preserved (early returns stay legal) + assert "def _double(x):" in nb # transitive helper carried + assert "CONST = 10" in nb # constant carried + assert "from datetime import datetime" in nb # non-Airflow import carried + assert "from airflow" not in nb # Airflow imports dropped + assert "result = process(**op_kwargs)" in nb # invoked with op_kwargs + assert "taskValues.set" in nb # return value captured + + +def test_python_operator_with_context_kwarg_becomes_placeholder(): + # A callable taking **context can't run without the Airflow runtime; route to a gap + # rather than emitting a notebook that fails at runtime. + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def work(**context):\n print(context['ds'])\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='work', python_callable=work)\n" + ) + task = _by_key(p)["work"] + assert isinstance(task, PlaceholderActivity) + assert "context" in task.comment + assert task.raw_definition is not None # carries source for the agentic round + + +def test_python_operator_with_ti_param_becomes_placeholder(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def work(ti):\n ti.xcom_push(key='k', value=1)\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='work', python_callable=work)\n" + ) + task = _by_key(p)["work"] + assert isinstance(task, PlaceholderActivity) + + +def test_python_operator_with_xcom_pull_becomes_placeholder(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def work(data=None):\n" + " prev = work.xcom_pull(task_ids='up')\n" + " print(prev)\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='work', python_callable=work)\n" + ) + task = _by_key(p)["work"] + assert isinstance(task, PlaceholderActivity) + assert "XCom" in task.comment + + def test_bash_operator_becomes_sh_notebook(): p = _load( "from airflow import DAG\n" @@ -158,20 +228,118 @@ def test_table_sensor_lifts_to_table_update_trigger(): } -def test_sql_condition_sensor_without_table_stays_placeholder(): - # A SqlSensor checking an arbitrary condition (no table_name) must NOT vanish; - # it stays as a placeholder task rather than lifting to a table trigger. +def test_sql_condition_sensor_becomes_polling_task(): + # A SqlSensor checking an arbitrary condition (no table_name) must NOT vanish and must NOT lift + # to a table trigger; it becomes a polling notebook task running the query on a poke loop. p = _load( "from airflow import DAG\n" "from airflow.providers.common.sql.sensors.sql import SqlSensor\n" "with DAG(dag_id='d') as dag:\n" - " s = SqlSensor(task_id='chk', sql='SELECT COUNT(*) FROM t WHERE ready')\n" + " s = SqlSensor(task_id='chk', sql='SELECT COUNT(*) FROM t WHERE ready', poke_interval=30, timeout=600)\n" + ) + task = _by_key(p)["chk"] + assert isinstance(task, NotebookActivity) + assert p.schedule is None + src = task.generated_source + assert "SELECT COUNT(*) FROM t WHERE ready" in src + assert "POKE_INTERVAL = 30" in src + assert "TIMEOUT = 600" in src + # Generated polling notebook must be valid Python. + compile(src, "", "exec") + + +def test_sql_condition_sensor_without_literal_sql_stays_placeholder(): + # No literal sql/table_name to poll -> a placeholder task with guidance, never a silent drop. + p = _load( + "from airflow import DAG\n" + "from airflow.providers.common.sql.sensors.sql import SqlSensor\n" + "with DAG(dag_id='d') as dag:\n" + " s = SqlSensor(task_id='chk', sql=build_query())\n" ) task = _by_key(p)["chk"] assert isinstance(task, PlaceholderActivity) assert p.schedule is None +def test_external_task_sensor_becomes_cross_dag_wait(): + p = _load( + "from airflow import DAG\n" + "from airflow.sensors.external_task import ExternalTaskSensor\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " wait = ExternalTaskSensor(task_id='wait_up', external_dag_id='upstream dag',\n" + " poke_interval=45, timeout=900)\n" + " go = PythonOperator(task_id='go', python_callable=w)\n" + " wait >> go\n" + ) + task = _by_key(p)["wait_up"] + assert isinstance(task, NotebookActivity) + src = task.generated_source + assert "WorkspaceClient" in src + assert "EXTERNAL_JOB_NAME = 'upstream_dag'" in src # sanitized to the sibling job name + assert "POKE_INTERVAL = 45" in src + compile(src, "", "exec") + + +def test_http_sensor_becomes_polling_task(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.http.sensors.http import HttpSensor\n" + "with DAG(dag_id='d') as dag:\n" + " s = HttpSensor(task_id='h', endpoint='api/ready', poke_interval=10, timeout=120)\n" + ) + task = _by_key(p)["h"] + assert isinstance(task, NotebookActivity) + src = task.generated_source + assert "requests.get" in src + assert "api/ready" in src + compile(src, "", "exec") + + +def test_python_sensor_polls_callable_without_eager_call(): + p = _load( + "from airflow import DAG\n" + "from airflow.sensors.python import PythonSensor\n" + "def is_ready():\n return spark.table('t').count() > 0\n" + "with DAG(dag_id='d') as dag:\n" + " s = PythonSensor(task_id='chk', python_callable=is_ready, poke_interval=25, timeout=500)\n" + ) + task = _by_key(p)["chk"] + assert isinstance(task, NotebookActivity) + src = task.generated_source + assert "def is_ready():" in src # callable carried + assert "return is_ready()" in src # polled inside the loop + assert "taskValues.set" not in src # NOT invoked eagerly as a one-shot + compile(src, "", "exec") + + +def test_python_sensor_with_context_stays_placeholder(): + p = _load( + "from airflow import DAG\n" + "from airflow.sensors.python import PythonSensor\n" + "def is_ready(**context):\n return context['ti'].xcom_pull('x')\n" + "with DAG(dag_id='d') as dag:\n" + " s = PythonSensor(task_id='chk', python_callable=is_ready)\n" + ) + assert isinstance(_by_key(p)["chk"], PlaceholderActivity) + + +def test_datetime_sensor_becomes_wait_until_task(): + p = _load( + "from airflow import DAG\n" + "from airflow.sensors.date_time import DateTimeSensor\n" + "with DAG(dag_id='d') as dag:\n" + " s = DateTimeSensor(task_id='wait', target_time='2026-01-01T00:00:00+00:00')\n" + ) + task = _by_key(p)["wait"] + assert isinstance(task, NotebookActivity) + src = task.generated_source + assert "datetime.fromisoformat" in src + assert "2026-01-01T00:00:00+00:00" in src + compile(src, "", "exec") + + def test_databricks_run_now_becomes_run_job(): p = _load( "from airflow import DAG\n" @@ -298,19 +466,72 @@ def test_file_sensor_lifts_to_file_arrival_trigger(): assert p.schedule == {"kind": "file_arrival", "url": "s3://landing/in/", "pause_status": "UNPAUSED"} -def test_explicit_cron_wins_over_sensor_trigger(): +def test_cron_and_sensor_keeps_both_schedule_and_polling_task(): + # cron AND-THEN wait: the cron becomes the schedule and the sensor is retained as a polling + # task (never silently dropped), because schedule and file_arrival triggers are mutually exclusive. p = _load( "from airflow import DAG\n" "from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor\n" "from airflow.operators.python import PythonOperator\n" "def w():\n pass\n" "with DAG(dag_id='d', schedule_interval='0 6 * * *') as dag:\n" - " wait = S3KeySensor(task_id='wait', bucket_key='s3://x/')\n" + " wait = S3KeySensor(task_id='wait', bucket_key='s3://x/in/', poke_interval=30, timeout=600)\n" " go = PythonOperator(task_id='go', python_callable=w)\n" " wait >> go\n" ) assert p.schedule["kind"] == "schedule" assert p.schedule["quartz_cron_expression"] == "0 0 6 ? * *" + tasks = _by_key(p) + assert set(tasks) == {"wait", "go"} # sensor retained as a task + wait = tasks["wait"] + assert isinstance(wait, NotebookActivity) + assert "dbutils.fs.ls" in wait.generated_source + assert "s3://x/in/" in wait.generated_source + assert "POKE_INTERVAL = 30" in wait.generated_source + assert "TIMEOUT = 600" in wait.generated_source + compile(wait.generated_source, "", "exec") + # `go` still depends on the retained sensor task (ordering preserved). + assert [d.task_key for d in tasks["go"].depends_on] == ["wait"] + + +def test_mid_dag_sensor_retained_as_polling_task(): + # A sensor that is not the DAG's entry gate (has an upstream) is an ordering gate within the run, + # so it stays a polling task rather than lifting to a job-level trigger. + p = _load( + "from airflow import DAG\n" + "from airflow.providers.databricks.sensors.databricks_partition import DatabricksPartitionSensor\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " prep = PythonOperator(task_id='prep', python_callable=w)\n" + " wait = DatabricksPartitionSensor(task_id='wait', table_name='main.silver.events')\n" + " go = PythonOperator(task_id='go', python_callable=w)\n" + " prep >> wait >> go\n" + ) + assert p.schedule is None # mid-DAG sensor does not become a trigger + tasks = _by_key(p) + assert set(tasks) == {"prep", "wait", "go"} + wait = tasks["wait"] + assert isinstance(wait, NotebookActivity) + assert 'spark.catalog.tableExists("main.silver.events")' in wait.generated_source + compile(wait.generated_source, "", "exec") + + +def test_dag_params_supply_job_parameter_defaults(): + p = _load( + "from airflow import DAG\n" + "from airflow.models.param import Param\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d', params={'env': 'prod', 'threshold': Param(10)}) as dag:\n" + " t = PythonOperator(task_id='t', python_callable=w,\n" + " op_kwargs={'e': '{{ params.env }}'})\n" + ) + params = {entry["name"]: entry["default"] for entry in (p.parameters or [])} + # Referenced param picks up its params={...} default; a declared-but-unreferenced param is + # still emitted (with its default) so the job parameter validates. + assert params["env"] == "prod" + assert params["threshold"] == 10 # -------------------------------------------------------------------------------------- @@ -370,15 +591,20 @@ def test_jinja_macros_convert_to_dab_refs_and_collect_params(): p = _load( "from airflow import DAG\n" "from airflow.operators.python import PythonOperator\n" - "def w():\n pass\n" + "def w(date=None, env=None):\n pass\n" "with DAG(dag_id='d') as dag:\n" " t = PythonOperator(task_id='t', python_callable=w,\n" " op_kwargs={'date': '{{ ds }}', 'env': '{{ params.env }}'})\n" ) task = _by_key(p)["t"] - assert task.base_parameters == {"date": "{{job.start_time.iso_date}}", "env": "{{job.parameters.env}}"} - # The referenced param is declared on the pipeline. - assert p.parameters == [{"name": "env"}] + # op_kwargs are JSON-encoded into the internal __flowx_op_kwargs widget; Jinja inside the + # values is still converted to DAB refs. + kwargs_json = task.base_parameters["__flowx_op_kwargs"] + assert "{{job.start_time.iso_date}}" in kwargs_json + assert "{{job.parameters.env}}" in kwargs_json + # The referenced param is declared on the pipeline with a (Databricks-required) default; the + # internal __flowx_ widget is NOT declared. + assert p.parameters == [{"name": "env", "default": ""}] def test_default_args_apply_retries_timeout_retry_delay(): @@ -407,7 +633,7 @@ def test_per_task_retries_override_default_args(): assert _by_key(p)["t"].max_retries == 7 -def test_trigger_rule_maps_to_dependency_outcome(): +def test_trigger_rule_maps_to_run_if_constant(): p = _load( "from airflow import DAG\n" "from airflow.operators.python import PythonOperator\n" @@ -416,12 +642,22 @@ def test_trigger_rule_maps_to_dependency_outcome(): " a = PythonOperator(task_id='a', python_callable=w)\n" " cleanup = PythonOperator(task_id='cleanup', python_callable=w, trigger_rule='all_done')\n" " fail_only = PythonOperator(task_id='fail_only', python_callable=w, trigger_rule='one_failed')\n" + " only_fail = PythonOperator(task_id='only_fail', python_callable=w, trigger_rule='all_failed')\n" + " any_ok = PythonOperator(task_id='any_ok', python_callable=w, trigger_rule='one_success')\n" + " no_fail = PythonOperator(task_id='no_fail', python_callable=w, trigger_rule='none_failed')\n" " a >> cleanup\n" " a >> fail_only\n" + " a >> only_fail\n" + " a >> any_ok\n" + " a >> no_fail\n" ) tasks = _by_key(p) - assert tasks["cleanup"].depends_on[0].outcome == "Completed" # all_done -> ALL_DONE - assert tasks["fail_only"].depends_on[0].outcome == "Failed" # one_failed -> AT_LEAST_ONE_FAILED + # trigger_rule maps straight to the DAB run_if constant, carried as the dependency outcome. + assert tasks["cleanup"].depends_on[0].outcome == "ALL_DONE" + assert tasks["fail_only"].depends_on[0].outcome == "AT_LEAST_ONE_FAILED" + assert tasks["only_fail"].depends_on[0].outcome == "ALL_FAILED" + assert tasks["any_ok"].depends_on[0].outcome == "AT_LEAST_ONE_SUCCESS" + assert tasks["no_fail"].depends_on[0].outcome == "NONE_FAILED" assert tasks["a"].depends_on is None # default all_success -> no outcome @@ -471,6 +707,32 @@ def test_task_group_prefixes_member_keys(): assert keys == {"extract__run", "load__run"} # no collision +def test_task_group_level_dependencies_expand_to_boundary_tasks(): + # `start >> etl >> pub >> end` where etl/pub are TaskGroups: the group-level edges must expand to + # leaf(upstream) -> root(downstream), not be silently dropped. + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "from airflow.utils.task_group import TaskGroup\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " start = PythonOperator(task_id='start', python_callable=w)\n" + " with TaskGroup('etl') as etl:\n" + " a = PythonOperator(task_id='a', python_callable=w)\n" + " b = PythonOperator(task_id='b', python_callable=w)\n" + " a >> b\n" + " with TaskGroup('publish') as pub:\n" + " c = PythonOperator(task_id='c', python_callable=w)\n" + " end = PythonOperator(task_id='end', python_callable=w)\n" + " start >> etl >> pub >> end\n" + ) + deps = {k: sorted(d.task_key for d in (t.depends_on or [])) for k, t in _by_key(p).items()} + assert deps["etl__a"] == ["start"] # start -> root of etl + assert deps["etl__b"] == ["etl__a"] # intra-group edge preserved + assert deps["publish__c"] == ["etl__b"] # leaf of etl -> root of publish + assert deps["end"] == ["publish__c"] # leaf of publish -> end + + def test_timedelta_schedule_becomes_periodic(): p = _load( "from datetime import timedelta\n" @@ -512,7 +774,7 @@ def test_variable_get_rewritten_to_widget_and_declared_as_param(): task = _by_key(p)["ingest"] assert 'dbutils.widgets.get("target_env")' in task.generated_source assert "Variable.get" not in task.generated_source - assert {"name": "target_env"} in (p.parameters or []) + assert {"name": "target_env", "default": ""} in (p.parameters or []) def test_connection_get_rewritten_to_secrets(): @@ -546,3 +808,109 @@ def test_airflow_host_detection_from_dag_source(): encoding="utf-8", ) assert detect_hosts(src) == ["ws.cloud.databricks.com"] + + +# -------------------------------------------------------------------------------------- +# TaskFlow API (@dag / @task) +# -------------------------------------------------------------------------------------- + + +def test_taskflow_dag_and_tasks_are_detected(): + # A pure-TaskFlow DAG must yield tasks (not silently drop them), with the @dag config picked up. + p = _load( + "from airflow.decorators import dag, task\n" + "from datetime import datetime\n" + "@task\n" + "def extract():\n return [1, 2, 3]\n" + "@task\n" + "def transform(data):\n return [x * 2 for x in data]\n" + "@task\n" + "def load(data):\n print(sum(data))\n" + "@dag(schedule='0 6 * * *', start_date=datetime(2024, 1, 1), dag_id='etl_flow')\n" + "def pipeline():\n" + " load(transform(extract()))\n" + "pipeline()\n" + ) + assert p.name == "etl_flow" + assert p.schedule["quartz_cron_expression"] == "0 0 6 ? * *" + kinds = sorted(type(t).__name__ for t in p.tasks) + assert kinds == ["NotebookActivity", "NotebookActivity", "NotebookActivity"] + # The nested chain load(transform(extract())) wires extract -> transform -> load. + transform_task = next(t for t in p.tasks if t.task_key.startswith("transform")) + extract_key = next(t.task_key for t in p.tasks if t.task_key.startswith("extract")) + load_task = next(t for t in p.tasks if t.task_key.startswith("load")) + assert transform_task.depends_on[0].task_key == extract_key + assert load_task.depends_on[0].task_key == transform_task.task_key + + +def test_taskflow_data_flow_reads_upstream_taskvalue(): + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def extract():\n return 5\n" + "@task\n" + "def transform(data):\n return data * 2\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " raw = extract()\n" + " transform(raw)\n" + "pipeline()\n" + ) + tasks = _by_key(p) + assert set(tasks) == {"raw", "transform"} + src = tasks["transform"].generated_source + compile(src, "", "exec") + assert "def transform(data):" in src # callable carried + assert "dbutils.jobs.taskValues.get(taskKey='raw', key='return_value'" in src # reads upstream + assert "result = transform(_upstream_0)" in src # bound to positional arg + assert "dbutils.jobs.taskValues.set(key='return_value', value=result)" in src # publishes + + +def test_taskflow_branch_decorator_becomes_placeholder(): + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def extract():\n return 1\n" + "@task.branch\n" + "def choose(data):\n return 'a' if data else 'b'\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " choose(extract())\n" + "pipeline()\n" + ) + choose = next(t for t in p.tasks if t.task_key.startswith("choose")) + assert isinstance(choose, PlaceholderActivity) + assert "condition_task" in choose.comment + + +def test_taskflow_task_with_context_becomes_placeholder(): + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def work(**context):\n print(context['ds'])\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " work()\n" + "pipeline()\n" + ) + work = next(t for t in p.tasks if t.task_key.startswith("work")) + assert isinstance(work, PlaceholderActivity) + + +def test_taskflow_mixed_with_classic_operator(): + # A @dag body mixing a classic operator and a @task: both become tasks, wired by >>. + p = _load( + "from airflow.decorators import dag, task\n" + "from airflow.operators.bash import BashOperator\n" + "@task\n" + "def finalize():\n print('done')\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " prep = BashOperator(task_id='prep', bash_command='echo hi')\n" + " prep >> finalize()\n" + "pipeline()\n" + ) + tasks = _by_key(p) + assert "prep" in tasks + finalize = next(t for t in p.tasks if t.task_key.startswith("finalize")) + assert finalize.depends_on[0].task_key == "prep" From adc4d504e48eb4a7b25d9fcad19be310bdaa4284 Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 20 Jul 2026 12:54:37 -0700 Subject: [PATCH 28/77] Wire Airflow dbt factories through static and PyDABs modes --- src/flowx/bundler/dab_writer.py | 76 ++++++++++++++---- src/flowx/bundler/prereqs_writer.py | 32 ++++++++ .../activity_preparers/dbt_factory.py | 4 +- src/flowx/sources/airflow/convert.py | 9 ++- src/flowx/sources/airflow/loader.py | 52 +++++++++--- tests/unit/test_airflow_operators.py | 80 +++++++++++++++++++ tests/unit/test_dbt_factory_preparer.py | 45 ++++++++++- 7 files changed, 270 insertions(+), 28 deletions(-) diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 15e1384..3fbe618 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -129,6 +129,10 @@ def write_bundle( "warehouse_id", {"description": "SQL warehouse id for sql_task queries"} ) + # dbt-factory PyDABs hooks: each `resources._dbt_job:load_resources` module must be + # registered under the `python.resources` block so `bundle deploy` runs it to build the dbt job. + pydabs_resource_entries = _collect_pydabs_resource_entries(workflow) + # 1. Write databricks.yml. When any task runs on classic compute, spark_version / node_type_id # defaults come from the ADF linked-service configs; when every task is serverless, they're omitted. databricks_yml_path = output_dir / "databricks.yml" @@ -141,6 +145,7 @@ def write_bundle( node_type_id=inferred_node_type_id, include_cluster_variables=bundle_uses_classic_cluster, extra_variables=pipeline_variable_declarations, + pydabs_resources=pydabs_resource_entries, ) databricks_yml_path.write_text( yaml.dump( @@ -206,10 +211,21 @@ def write_bundle( ) created_files.append(resource_yml_path.resolve()) - # 3. Write generated notebooks + # 3. Write generated notebooks. PyDABs hook modules (relative_path under ``resources/``) are + # Python resources the bundle imports as ``resources.`` from the bundle root, so they + # go to output_dir; all other generated notebooks go under ``src/``. src_dir = output_dir / "src" + + def _write_generated(notebooks: list[DabNotebook]) -> None: + hooks = [nb for nb in notebooks if nb.relative_path.startswith("resources/")] + rest = [nb for nb in notebooks if not nb.relative_path.startswith("resources/")] + if rest: + created_files.extend(write_notebooks(rest, src_dir)) + if hooks: + created_files.extend(write_notebooks(hooks, output_dir)) + if workflow.notebooks: - created_files.extend(write_notebooks(workflow.notebooks, src_dir)) + _write_generated(workflow.notebooks) # 4. Generate and write setup notebooks (create-scope, create-volume, etc.) — the executable # provisioning artifacts; SETUP.md (below) is the human-readable companion. @@ -225,7 +241,7 @@ def write_bundle( # Collect notebooks from inner workflows for inner in workflow.inner_workflows: if inner.notebooks: - created_files.extend(write_notebooks(inner.notebooks, src_dir)) + _write_generated(inner.notebooks) inner_setup = generate_setup_tasks( secrets=inner.secrets, setup_tasks=inner.setup_tasks, @@ -262,7 +278,11 @@ def write_bundle( task.config for task in workflow.setup_tasks if task.type == "manual_schedule_time_of_day" ] manual_credential_configs = [task.config for task in workflow.setup_tasks if task.type == "manual_credential"] + pydabs_dbt_factory_configs = [task.config for task in workflow.setup_tasks if task.type == "pydabs_dbt_factory"] for inner in workflow.inner_workflows: + pydabs_dbt_factory_configs.extend( + task.config for task in inner.setup_tasks if task.type == "pydabs_dbt_factory" + ) dynamic_dispatch_configs.extend( task.config for task in inner.setup_tasks if task.type == "dynamic_notebook_dispatch" ) @@ -296,6 +316,7 @@ def write_bundle( manual_schedule_time_of_day=manual_schedule_time_of_day_configs, manual_credentials=manual_credential_configs, neutralized_conditions=list(_neutralized_conditions), + pydabs_dbt_factories=pydabs_dbt_factory_configs, ) setup_path = output_dir / "SETUP.md" setup_path.write_text(render_setup_md(prereqs, bundle_name=effective_name), encoding="utf-8") @@ -600,6 +621,7 @@ def _build_databricks_yml( node_type_id: str = _DEFAULT_NODE_TYPE_ID, include_cluster_variables: bool = True, extra_variables: dict[str, Any] | None = None, + pydabs_resources: list[str] | None = None, ) -> dict[str, Any]: """Builds the root ``databricks.yml`` configuration as a dict. @@ -618,6 +640,9 @@ def _build_databricks_yml( extra_variables: Additional variable declarations (name -> DAB declaration dict) to merge into the ``variables`` block, e.g. the source-side variables a Lakeflow Connect pipeline references. + pydabs_resources: ``python.resources`` entries (``:load_resources``) + for dbt-factory PyDABs hooks. When present, a ``python:`` block is + emitted so ``bundle deploy`` runs each hook to build its dbt job. Returns: Dict ready for YAML serialization. @@ -656,7 +681,7 @@ def _build_databricks_yml( "the default here." ), } - return { + config: dict[str, Any] = { "bundle": { "name": bundle_name, }, @@ -664,18 +689,23 @@ def _build_databricks_yml( "include": [ "resources/*.yml", ], - "targets": { - "dev": { - "mode": "development", - }, - "staging": { - "mode": "production", - }, - "prod": { - "mode": "production", - }, + } + if pydabs_resources: + # PyDABs hooks build dbt jobs at deploy time; venv_path points at the project's own venv + # (created by `make setup` / `uv sync`), which must have `databricks-dbt-factory` installed. + config["python"] = {"venv_path": ".venv", "resources": list(pydabs_resources)} + config["targets"] = { + "dev": { + "mode": "development", + }, + "staging": { + "mode": "production", + }, + "prod": { + "mode": "production", }, } + return config def _build_default_job_clusters( @@ -793,6 +823,24 @@ def _collect_pipeline_resources(workflow: PreparedWorkflow) -> list[dict[str, An return resources +def _collect_pydabs_resource_entries(workflow: PreparedWorkflow) -> list[str]: + """Returns the ``python.resources`` entries for every dbt-factory PyDABs hook in *workflow*. + + Each ``pydabs_dbt_factory`` SetupTask carries a ``hook_module`` (e.g. + ``resources.orders_dbt_job``); the databricks.yml ``python.resources`` list needs + ``:load_resources`` so ``bundle deploy`` runs the hook to build the dbt job. + """ + entries: list[str] = [] + for wf in [workflow, *workflow.inner_workflows]: + for task in wf.setup_tasks: + if task.type == "pydabs_dbt_factory": + module = task.config.get("hook_module") + if module: + entries.append(f"{module}:load_resources") + # De-dup while preserving order. + return list(dict.fromkeys(entries)) + + def _wrap_pipeline_resource(resource: dict[str, Any]) -> dict[str, Any]: """Wraps a pipeline definition in the DAB ``resources.pipelines`` envelope. diff --git a/src/flowx/bundler/prereqs_writer.py b/src/flowx/bundler/prereqs_writer.py index 0149507..c4ee335 100644 --- a/src/flowx/bundler/prereqs_writer.py +++ b/src/flowx/bundler/prereqs_writer.py @@ -131,6 +131,9 @@ class Prereqs: # C-43 (CF5-001 / CF5-002): condition_task operands blanked because they referenced a task in another # job ({task_key, field, original_ref}); a blanked operand is always-true, so the user must re-wire it. neutralized_conditions: list[dict[str, str]] = field(default_factory=list) + # dbt-factory PyDABs hooks; each entry is the SetupTask config dict ({hook_module, job_key, + # manifest_path, note}). The user must `pip install databricks-dbt-factory` before deploy. + pydabs_dbt_factories: list[dict[str, Any]] = field(default_factory=list) def is_empty(self) -> bool: """Return ``True`` when nothing needs to happen before ``bundle run``.""" @@ -150,6 +153,7 @@ def is_empty(self) -> bool: and not self.manual_schedule_time_of_day and not self.manual_credentials and not self.neutralized_conditions + and not self.pydabs_dbt_factories ) @@ -361,6 +365,7 @@ def build_prereqs( manual_schedule_time_of_day: list[dict[str, Any]] | None = None, manual_credentials: list[dict[str, Any]] | None = None, neutralized_conditions: list[dict[str, str]] | None = None, + pydabs_dbt_factories: list[dict[str, Any]] | None = None, ) -> Prereqs: """Assemble a :class:`Prereqs` from the bundle's generated artifacts. @@ -408,6 +413,7 @@ def build_prereqs( manual_schedule_time_of_day=list(manual_schedule_time_of_day or []), manual_credentials=list(manual_credentials or []), neutralized_conditions=list(neutralized_conditions or []), + pydabs_dbt_factories=list(pydabs_dbt_factories or []), ) @@ -760,4 +766,30 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append(f"| {label} | `{endpoint.target}` | {endpoint.notes} |") lines.append("") + if prereqs.pydabs_dbt_factories: + lines.append("## dbt factory (PyDABs mode)") + lines.append("") + lines.append( + "This bundle builds its dbt job(s) at deploy time via a PyDABs hook. `databricks.yml` " + "already registers each hook under `python.resources`; complete the environment so " + "`databricks bundle deploy` can run them:" + ) + lines.append("") + lines.append("1. Install the generator and PyDABs into the bundle's venv (the `python.venv_path`):") + lines.append("") + lines.append("```bash") + lines.append("uv venv .venv && uv pip install databricks-dbt-factory databricks-bundles") + lines.append("```") + lines.append("") + lines.append("2. Ensure each dbt project's `manifest.json` exists (run `dbt parse`/`dbt compile`).") + lines.append("") + lines.append("| dbt job | Hook module | Manifest |") + lines.append("|---|---|---|") + for entry in prereqs.pydabs_dbt_factories: + job_key = entry.get("job_key", "") + module = entry.get("hook_module", "") + manifest = entry.get("manifest_path", "") + lines.append(f"| `{job_key}` | `{module}:load_resources` | `{manifest}` |") + lines.append("") + return "\n".join(lines) diff --git a/src/flowx/preparer/activity_preparers/dbt_factory.py b/src/flowx/preparer/activity_preparers/dbt_factory.py index 6945ca1..a330a03 100644 --- a/src/flowx/preparer/activity_preparers/dbt_factory.py +++ b/src/flowx/preparer/activity_preparers/dbt_factory.py @@ -165,6 +165,8 @@ def _prepare_pydabs(activity: DbtFactoryActivity) -> PreparedActivity: hook_relative_path = f"resources/{activity.task_key}_dbt_job.py" hook_notebook = DabNotebook(relative_path=hook_relative_path, content=_pydabs_hook_source(activity)) + # `resources` must be an importable package for `python.resources: resources.` to resolve. + package_marker = DabNotebook(relative_path="resources/__init__.py", content="") setup_task = SetupTask( type="pydabs_dbt_factory", config={ @@ -177,7 +179,7 @@ def _prepare_pydabs(activity: DbtFactoryActivity) -> PreparedActivity: ), }, ) - return PreparedActivity(task=parent_task, notebooks=[hook_notebook], setup_tasks=[setup_task]) + return PreparedActivity(task=parent_task, notebooks=[hook_notebook, package_marker], setup_tasks=[setup_task]) def prepare(activity: DbtFactoryActivity, *, scope: str = "") -> PreparedActivity: diff --git a/src/flowx/sources/airflow/convert.py b/src/flowx/sources/airflow/convert.py index c2fd769..0d9c443 100644 --- a/src/flowx/sources/airflow/convert.py +++ b/src/flowx/sources/airflow/convert.py @@ -27,11 +27,18 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--source-dir", required=True, type=Path, help="A DAG .py file or directory of DAGs.") parser.add_argument("--output-dir", type=Path, default=Path("./flowx_output"), help="Shared migration output dir.") parser.add_argument("--pipeline", type=str, default=None, help="Translate only the named DAG (default: all).") + parser.add_argument( + "--dbt-mode", + choices=("static", "pydabs"), + default="static", + help="dbt-factory render mode: 'static' (inner job of per-node tasks, default) or 'pydabs' " + "(a deploy-time PyDABs hook that builds the dbt job from the live manifest).", + ) args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") - pipelines = load_pipelines(args.source_dir, pipeline=args.pipeline) + pipelines = load_pipelines(args.source_dir, pipeline=args.pipeline, dbt_mode=args.dbt_mode) if not pipelines: logger.error("No Airflow DAGs found under %s (or none matched --pipeline).", args.source_dir) return 1 diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py index 123778d..c984b03 100644 --- a/src/flowx/sources/airflow/loader.py +++ b/src/flowx/sources/airflow/loader.py @@ -672,11 +672,13 @@ def _has_decorator(func: ast.FunctionDef, names: frozenset[str]) -> bool: return any(_decorator_name(dec) in names for dec in func.decorator_list) -def load_airflow_dag(dag_path: Path) -> Pipeline: +def load_airflow_dag(dag_path: Path, *, dbt_mode: str = "static") -> Pipeline: """Parses an Airflow DAG file into a flowx Pipeline IR. Args: dag_path: Path to a ``.py`` DAG module. + dbt_mode: dbt-factory render mode for any dbt workload -- ``"static"`` (default, + an inner job of per-node tasks) or ``"pydabs"`` (a deploy-time hook module). Returns: A :class:`~flowx.models.ir.Pipeline`. Mapped operators become their IR @@ -740,8 +742,17 @@ def _task_key(var: str, task_id: str) -> str: dropped.add(trigger_var) upstreams = _rewire_dropped(upstreams, dropped) - # Collapse all dbt CLI operators over the one project into a single DbtFactoryActivity. + # Collapse all dbt CLI operators over the one project into a single DbtFactoryActivity emitted at + # the first dbt task's position. Every dbt var's task_key remaps to that single key, so a + # downstream task that depended on a later dbt op (e.g. `dbt_test`) points at the factory task + # rather than a task_key that was never emitted (which would dangle). dbt_vars = [var for var, (_, op, _) in visitor.operators.items() if op in ops.DBT_CLI_OPERATORS] + dbt_factory_key = var_to_task_key[dbt_vars[0]] if dbt_vars else None + dbt_key_remap = {var_to_task_key[v]: dbt_factory_key for v in dbt_vars} if dbt_factory_key else {} + + def _dep(upstream_var: str, outcome: str | None) -> str: + key = var_to_task_key[upstream_var] + return dbt_key_remap.get(key, key) tasks: list[Activity] = [] referenced_params: set[str] = set() @@ -751,10 +762,14 @@ def _task_key(var: str, task_id: str) -> str: continue task_key = var_to_task_key[var] outcome = templating.trigger_rule_outcome(kwargs) - depends_on = [Dependency(task_key=var_to_task_key[u], outcome=outcome) for u in upstreams[var]] or None + # Remap dbt-chain upstreams to the single factory key and drop self-edges (a dbt op + # depending on another dbt op in the same collapsed chain). + dep_keys = {_dep(u, outcome) for u in upstreams[var]} + dep_keys.discard(task_key if operator not in ops.DBT_CLI_OPERATORS else dbt_factory_key) + depends_on = [Dependency(task_key=k, outcome=outcome) for k in sorted(dep_keys)] or None if operator in ops.COSMOS_CONSTRUCTS: - tasks.append(_build_dbt_factory(task_id, task_key, [kwargs], depends_on)) + tasks.append(_build_dbt_factory(task_id, task_key, [kwargs], depends_on, dbt_mode)) continue if operator in ops.DBT_CLI_OPERATORS: # Emit one factory job for the whole dbt chain, at the first dbt task's position. @@ -762,7 +777,7 @@ def _task_key(var: str, task_id: str) -> str: continue emitted_dbt = True dbt_kwargs = [visitor.operators[v][2] for v in dbt_vars] - tasks.append(_build_dbt_factory(task_id, task_key, dbt_kwargs, depends_on)) + tasks.append(_build_dbt_factory(task_id, task_key, dbt_kwargs, depends_on, dbt_mode)) continue call_node = visitor.calls.get(var) @@ -1082,16 +1097,18 @@ def _build_dbt_factory( task_key: str, kwargs_list: list[dict[str, ast.expr]], depends_on: list[Dependency] | None, + dbt_mode: str = "static", ) -> DbtFactoryActivity: """Builds a DbtFactoryActivity from cosmos config or a set of dbt CLI operators. Extracts project_dir / profiles_dir / target from cosmos ProjectConfig/ProfileConfig - args or dbt operator kwargs. render_mode defaults to static (the flowx-native path); + args or dbt operator kwargs. ``dbt_mode`` selects the render mode (static | pydabs); the manifest is read at package time from project_dir/target/manifest.json. """ project_dir = "." profiles_dir = "dbt_profiles" target = "dev" + manifest_path: str | None = None for kwargs in kwargs_list: # dbt CLI operators pass project_dir/target directly as kwargs. project_dir = ops.literal_str(kwargs.get("project_dir")) or ops.literal_str(kwargs.get("dir")) or project_dir @@ -1100,6 +1117,13 @@ def _build_dbt_factory( # Cosmos nests config in ProjectConfig(...) / ProfileConfig(...) calls. project_dir = _cosmos_project_dir(kwargs.get("project_config")) or project_dir target = _cosmos_target(kwargs.get("profile_config")) or target + manifest_path = _cosmos_manifest_path(kwargs.get("project_config")) or manifest_path + # The static preparer needs a manifest to explode into tasks. Point it at the per-target + # manifest `make manifest` produces (target//manifest.json), rooted at the project dir, + # unless cosmos gave an explicit manifest_path. Without this the child job would be empty. + if manifest_path is None: + base = project_dir.rstrip("/") if project_dir not in ("", ".") else "." + manifest_path = f"{base}/target/{target}/manifest.json" if base != "." else f"target/{target}/manifest.json" return DbtFactoryActivity( name=task_id, task_key=task_key, @@ -1107,7 +1131,8 @@ def _build_dbt_factory( project_dir=project_dir, profiles_dir=profiles_dir, target=target, - render_mode="static", + manifest_path=manifest_path, + render_mode="pydabs" if dbt_mode == "pydabs" else "static", ) @@ -1135,6 +1160,14 @@ def _cosmos_target(node: ast.expr | None) -> str | None: return ops.literal_str(kwargs.get("target_name")) +def _cosmos_manifest_path(node: ast.expr | None) -> str | None: + """Extracts an explicit ``manifest_path`` from a cosmos ``ProjectConfig(...)`` call, if any.""" + if not isinstance(node, ast.Call): + return None + kwargs = {kw.arg: kw.value for kw in node.keywords if kw.arg} + return ops.literal_str(kwargs.get("manifest_path")) + + def discover_dags(source_path: Path) -> list[Path]: """Returns the DAG ``.py`` files under *source_path*. @@ -1157,18 +1190,19 @@ def discover_dags(source_path: Path) -> list[Path]: return dags -def load_pipelines(source_path: Path, pipeline: str | None = None) -> list[Pipeline]: +def load_pipelines(source_path: Path, pipeline: str | None = None, *, dbt_mode: str = "static") -> list[Pipeline]: """Loads every DAG under *source_path* into Pipeline IR. Args: source_path: A DAG ``.py`` file or a directory of them. pipeline: When set, keep only the pipeline whose name (dag_id) matches. + dbt_mode: dbt-factory render mode -- ``"static"`` (default) or ``"pydabs"``. Returns: One :class:`~flowx.models.ir.Pipeline` per discovered DAG, filtered to *pipeline* when provided. """ - pipelines = [load_airflow_dag(dag_path) for dag_path in discover_dags(source_path)] + pipelines = [load_airflow_dag(dag_path, dbt_mode=dbt_mode) for dag_path in discover_dags(source_path)] if pipeline is not None: pipelines = [p for p in pipelines if p.name == pipeline] return pipelines diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py index 219c9e9..a354ab4 100644 --- a/tests/unit/test_airflow_operators.py +++ b/tests/unit/test_airflow_operators.py @@ -431,6 +431,24 @@ def test_cosmos_dbt_task_group_becomes_dbt_factory(): assert task.render_mode == "static" +def test_dbt_mode_pydabs_sets_render_mode(): + # `--dbt-mode pydabs` (threaded through load_airflow_dag) makes the factory reachable in PyDABs mode. + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "dag.py" + path.write_text( + "from airflow import DAG\n" + "from cosmos import DbtTaskGroup, ProjectConfig, ProfileConfig\n" + "with DAG(dag_id='d') as dag:\n" + " dbt = DbtTaskGroup(group_id='t', project_config=ProjectConfig('/opt/proj'),\n" + " profile_config=ProfileConfig(profile_name='p', target_name='prod'))\n", + encoding="utf-8", + ) + p = load_airflow_dag(path, dbt_mode="pydabs") + task = _by_key(p)["t"] + assert isinstance(task, DbtFactoryActivity) + assert task.render_mode == "pydabs" + + def test_dbt_cli_operators_collapse_to_one_factory(): p = _load( "from airflow import DAG\n" @@ -444,6 +462,68 @@ def test_dbt_cli_operators_collapse_to_one_factory(): dbt_tasks = [t for t in p.tasks if isinstance(t, DbtFactoryActivity)] assert len(dbt_tasks) == 1 # the seed>>run>>test chain collapses into one factory job assert dbt_tasks[0].project_dir == "/opt/proj" + # A manifest_path must be set or the static preparer would explode zero tasks (empty child job). + assert dbt_tasks[0].manifest_path == "/opt/proj/target/dev/manifest.json" + + +def test_dbt_chain_downstream_dep_rewired_to_factory_key(): + # A non-dbt task depending on the LAST dbt op (`test`) must point at the single collapsed + # factory task (`seed`), not the vanished `test` key (which would dangle at package time). + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "from airflow_dbt.operators.dbt_operator import DbtSeedOperator, DbtRunOperator, DbtTestOperator\n" + "def pub():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " s = DbtSeedOperator(task_id='seed', dir='/opt/proj')\n" + " r = DbtRunOperator(task_id='run', dir='/opt/proj')\n" + " t = DbtTestOperator(task_id='test', dir='/opt/proj')\n" + " p2 = PythonOperator(task_id='publish', python_callable=pub)\n" + " s >> r >> t >> p2\n" + ) + tasks = _by_key(p) + factory_key = next(t.task_key for t in p.tasks if isinstance(t, DbtFactoryActivity)) + assert [d.task_key for d in tasks["publish"].depends_on] == [factory_key] + + +def test_dbt_factory_explodes_manifest_into_tasks(tmp_path): + # End-to-end: a real (synthetic) manifest must explode into per-node tasks, not an empty job. + import json + + from flowx.preparer.workflow_preparer import prepare_workflow + + manifest = { + "nodes": { + "seed.p.codes": { + "resource_type": "seed", + "name": "codes", + "fqn": ["p", "codes"], + "depends_on": {"nodes": []}, + }, + "model.p.stg": { + "resource_type": "model", + "name": "stg", + "fqn": ["p", "stg"], + "depends_on": {"nodes": ["seed.p.codes"]}, + }, + }, + "unit_tests": {}, + } + proj = tmp_path / "proj" / "target" / "dev" + proj.mkdir(parents=True) + (proj / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + dag = ( + "from airflow import DAG\n" + "from airflow_dbt.operators.dbt_operator import DbtRunOperator\n" + "with DAG(dag_id='d') as dag:\n" + f" r = DbtRunOperator(task_id='run', dir={str(tmp_path / 'proj')!r})\n" + ) + dag_file = tmp_path / "dag.py" + dag_file.write_text(dag, encoding="utf-8") + p = load_airflow_dag(dag_file) + wf = prepare_workflow(p) + inner_task_keys = {t["task_key"] for inner in wf.inner_workflows for t in inner.tasks} + assert inner_task_keys == {"seed_codes", "model_stg"} # non-empty, both manifest nodes exploded # -------------------------------------------------------------------------------------- diff --git a/tests/unit/test_dbt_factory_preparer.py b/tests/unit/test_dbt_factory_preparer.py index caaf882..ab43631 100644 --- a/tests/unit/test_dbt_factory_preparer.py +++ b/tests/unit/test_dbt_factory_preparer.py @@ -83,9 +83,11 @@ def test_static_parent_hop_keeps_upstream_dependency(): def test_pydabs_emits_hook_module_and_no_inner_job(): prepared = prepare_activity(_dbt_activity(render_mode="pydabs", manifest_path="target/manifest.json")) assert prepared.inner_workflows == [] - hook_paths = [nb.relative_path for nb in prepared.notebooks] - assert hook_paths == ["resources/dbt_transform_dbt_job.py"] - assert "load_resources" in prepared.notebooks[0].content + hook_paths = {nb.relative_path for nb in prepared.notebooks} + # The hook module plus a resources/ package marker so `python.resources` can import it. + assert hook_paths == {"resources/dbt_transform_dbt_job.py", "resources/__init__.py"} + hook = next(nb for nb in prepared.notebooks if nb.relative_path.endswith("_dbt_job.py")) + assert "load_resources" in hook.content assert "run_job_task" in prepared.task @@ -128,3 +130,40 @@ def test_survives_json_report_round_trip(): assert isinstance(dbt, DbtFactoryActivity) assert dbt.render_mode == "static" assert {n["task_key"] for n in dbt.nodes} == {"seed_codes", "model_stg", "model_fct", "test_stg"} + + +def test_pydabs_bundle_wires_python_resources_and_setup(tmp_path): + # End-to-end: PyDABs mode must register the hook under databricks.yml python.resources, write the + # hook + package marker to the bundle root (not src/), and surface the setup steps in SETUP.md. + import yaml + + from flowx.bundler.dab_writer import write_bundle + + pipeline = Pipeline( + name="orders", + tasks=[ + NotebookActivity( + name="ingest", + task_key="ingest", + notebook_path="notebooks/ingest.py", + generated_source="# Databricks notebook source\nprint('x')\n", + ), + _dbt_activity( + depends_on=[Dependency(task_key="ingest")], + render_mode="pydabs", + manifest_path="dbt/target/manifest.json", + ), + ], + ) + write_bundle(prepare_workflow(pipeline), tmp_path) + + databricks_yml = yaml.safe_load((tmp_path / "databricks.yml").read_text()) + assert databricks_yml["python"]["resources"] == ["resources.dbt_transform_dbt_job:load_resources"] + assert databricks_yml["python"]["venv_path"] == ".venv" + # Hook + package marker live at the bundle root so `resources.` imports resolve. + assert (tmp_path / "resources" / "dbt_transform_dbt_job.py").exists() + assert (tmp_path / "resources" / "__init__.py").exists() + assert not (tmp_path / "src" / "resources").exists() + setup = (tmp_path / "SETUP.md").read_text() + assert "dbt factory (PyDABs mode)" in setup + assert "databricks-dbt-factory" in setup From c18c42a4d4b6c61018ac233ef3fa4da84369d07c Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 20 Jul 2026 12:54:42 -0700 Subject: [PATCH 29/77] Document Airflow conversion coverage and follow-ups --- .../flowx-convert/sources/airflow-coverage.md | 86 +++++++++++++++++++ skills/flowx-convert/sources/airflow.md | 10 +++ skills/flowx-discover/sources/airflow.md | 7 ++ 3 files changed, 103 insertions(+) create mode 100644 skills/flowx-convert/sources/airflow-coverage.md diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md new file mode 100644 index 0000000..6aaf010 --- /dev/null +++ b/skills/flowx-convert/sources/airflow-coverage.md @@ -0,0 +1,86 @@ +# Airflow → DABs — coverage and follow-ups + +What the `--source airflow` path converts today, and what it does **not** yet handle. This is a +verified inventory against the parser (`src/flowx/sources/airflow/`), not an aspirational roadmap. +Use it to set expectations before a migration and to prioritize follow-up work. + +The Airflow parser is a **static AST walk** — it reads DAG modules with `ast.parse`, never installs +Airflow, and never executes a DAG. Anything the static walk can't see, it can't convert. + +## Supported today + +| Construct | Result | +| --- | --- | +| `PythonOperator` (classic) | Notebook task; callable `def` preserved, transitive helpers/constants/non-Airflow imports carried, `op_args`/`op_kwargs` passed as JSON widgets, return value via `dbutils.jobs.taskValues.set`. | +| `PythonVirtualenvOperator` / `ExternalPythonOperator` | Notebook task with a `%pip install` cell for `requirements`. | +| `BranchPythonOperator` / `ShortCircuitOperator` | Placeholder routed to the agentic-gap round (runtime branch selection can't be lowered statically). | +| `BashOperator` / `SSHOperator` | `%sh` notebook; a wrapped `spark-submit` is lifted to a Spark JAR/Python task. | +| `SparkSubmitOperator` | Spark JAR or Python task. | +| Databricks provider operators (`DatabricksSubmitRun*`, `DatabricksRunNow*`, `DatabricksNotebookOperator`) | Notebook / run-job tasks. | +| SQL operators (`DatabricksSql*`, `SQLExecuteQueryOperator`, `PostgresOperator`, `MySqlOperator`, `HiveOperator`, `DatabricksCopyIntoOperator`) | `sql_task` (SqlActivity); Jinja → `:name` markers + `sql_task.parameters`. | +| `TriggerDagRunOperator` | `run_job_task` referencing the target DAG by sanitized job name. | +| `EmailOperator` | Placeholder recommending job-level email notifications. | +| dbt CLI operators (`DbtRun/Test/Seed/Snapshot/Build/Deps`) and Cosmos `DbtDag` / `DbtTaskGroup` | Single `DbtFactoryActivity`, **static explosion** (default) or **PyDABs** (`--dbt-mode pydabs`); see [dbt factory](#dbt-factory-mode). | +| **TaskFlow API** (`@dag`, `@task`, `@task.virtualenv`) | Each `@task` invocation → a task; implicit XCom data flow (`transform(extract())`) → a notebook that reads upstream return values via `dbutils.jobs.taskValues.get`, calls the function, and publishes its own. `@task.branch` / `@task.short_circuit`, or a callable reading task context/XCom, route to a placeholder + gap. | +| File sensors (`S3KeySensor`, `GCSObjectExistenceSensor`, `FileSensor`, `HdfsSensor`, `WebHdfsSensor`) | Root sensor with no schedule → `file_arrival` trigger; otherwise a `dbutils.fs` polling notebook task. | +| Table/SQL sensors (`DatabricksPartitionSensor`, `DatabricksSqlSensor`, `DatabricksSQLStatementsSensor`, `SqlSensor`) | Root sensor naming a literal table with no schedule → `table_update` trigger; otherwise a `spark.sql` polling notebook task. | +| `ExternalTaskSensor` | Cross-DAG wait: a notebook polling the upstream DAG's Databricks job run state (referenced by sanitized job name). | +| `HttpSensor` / `PythonSensor` / `DateTimeSensor` | Polling notebook tasks (`requests` poll / callable poll / wait-until). A `PythonSensor` callable reading task context routes to a placeholder. | +| Time sensors (`TimeSensor`, `TimeDeltaSensor`) | Absorbed into the schedule (start-of-DAG delay); dropped with dependency rewiring. | +| `DummyOperator` / `EmptyOperator` | Dropped, downstream dependencies rewired. | +| `.expand()` on an operator | `for_each_task`. | +| Dependencies | `>>` / `<<` chains (incl. list/tuple fan-out and inline TaskFlow calls) and `set_upstream` / `set_downstream`. | +| **TaskGroups** | Static nesting → task-key namespacing (`group__subgroup__task`); group-level edges (`group_a >> group_b`, `task >> group`) expand to leaf→root edges between member tasks. | +| Schedule | Cron `schedule_interval` → Quartz (Unix DOW 0–6 → Quartz 1–7); `timedelta` → periodic. | +| `trigger_rule` | DAB `run_if` constant per edge (`ALL_DONE`, `ALL_FAILED`, `AT_LEAST_ONE_SUCCESS`, `NONE_FAILED`, …). | +| Job parameters | `params={...}` / `Param(default=...)` → job parameters with defaults; `{{ params.x }}` / `{{ var.value.x }}` / `{{ dag_run.conf['x'] }}` → `{{job.parameters.x}}`. | +| `Variable.get` / `BaseHook.get_connection` in a callable | Rewritten to `dbutils.widgets.get` / `dbutils.secrets.get`. | + +Any operator not listed becomes a `PlaceholderActivity` **and** a `gaps.json` entry carrying the +operator's raw source, for LLM-assisted translation in the agentic-gap round. That is the safe +fallback: a flagged manual task, not a silent omission. Callables that read Airflow task context +(`**context` / `ti`) or XCom, and runtime-branching decorators, take the same route rather than +emitting code that fails at runtime. + +## Not yet supported + +These are absent but fail safely — routed to a placeholder + `gaps.json`, or simply not exploded — or +are deliberate scope decisions. + +- **Dynamic TaskGroup mapping** (`.expand()` on a `@task_group` or `TaskGroup.partial().expand()`). + `.expand()` is only recognized on operator/`@task` calls; a mapped *group's* fan-out is lost. +- **Sensors beyond the mapped families** (`S3PrefixSensor`, custom sensors, etc.) → placeholder + gap. + A file sensor with a non-literal path, or a table/SQL sensor with no literal `sql` / `table_name`, + also falls back to a placeholder. +- **Shared multi-DAG bundle** *(by design, for now)*. Converting a directory of N DAGs produces **N + independent bundles**, one per DAG in its own subdirectory — not a single bundle with N jobs. + Cross-DAG job references (`TriggerDagRunOperator`, `ExternalTaskSensor`) reference the sibling job by + sanitized name; that job lives in a separate bundle, so both bundles must be deployed to the same + workspace for the reference to resolve. +- **Scaffolding for dbt static mode.** Static explosion assumes the dbt project, `manifest.json`, and + profiles already ship in the bundle. No `pyproject.toml` / `Makefile` / `profiles.yml` scaffolding is + generated (PyDABs mode surfaces the setup steps in SETUP.md instead). + +## dbt factory mode + +Two front-ends feed a single `DbtFactoryActivity`: Cosmos `DbtDag` / `DbtTaskGroup`, and a chain of +dbt CLI operators (collapsed into one factory at the first dbt task's position). Select the render +mode with `--dbt-mode {static,pydabs}` on the convert phase (default `static`). + +- **Static explosion (default).** Emits an inner job with one `notebook_task` per exploded dbt node + (dependency-wired from a pruned `manifest.json`), a shared `run_dbt_command.py` runner notebook that + shells out to the dbt CLI, and a `run_job_task` hop from the parent. The manifest is read at package + time. **Assumes** the dbt project + `manifest.json` + profiles already exist in the bundle. +- **PyDABs (`--dbt-mode pydabs`).** Emits a `resources/_dbt_job.py` hook (plus a + `resources/__init__.py` package marker) at the bundle root, registers it under `databricks.yml` + `python.resources`, and surfaces the setup steps in SETUP.md (install `databricks-dbt-factory`, + ensure `manifest.json` exists). `bundle deploy` runs the hook to build the dbt job from the live + manifest. + +## Priority for remaining follow-ups + +1. **Dynamic TaskGroup mapping** — expand a mapped `@task_group` / `TaskGroup.partial().expand()` into + a for-each over the group's tasks (today the fan-out is lost). +2. **Shared multi-DAG bundle** — a single bundle with N jobs so cross-DAG references resolve within one + deploy (currently one bundle per DAG, by design). +3. **Additional sensor families** — as demand warrants; unmapped sensors route to a placeholder today. diff --git a/skills/flowx-convert/sources/airflow.md b/skills/flowx-convert/sources/airflow.md index 079999e..78aac0b 100644 --- a/skills/flowx-convert/sources/airflow.md +++ b/skills/flowx-convert/sources/airflow.md @@ -9,6 +9,16 @@ deterministic mapping become `PlaceholderActivity` tasks *and* are recorded in ` carrying the operator's raw source so an agent can reason out the translation and replace the placeholder — the same `gaps.json` + `merge_agentic` flow the ADF source uses. +**Before converting, check [`sources/airflow-coverage.md`](airflow-coverage.md)** — the verified +support matrix (classic operators, TaskFlow, sensors, TaskGroups, dbt factory) and the constructs +still **not** handled (dynamic TaskGroup mapping, shared multi-DAG bundle). Constructs flowx can't +lower deterministically — callables reading task context (`**context` / `ti`) or XCom, and +runtime-branching decorators — are routed to a placeholder + `gaps.json` for the agentic round rather +than emitted as broken code. + +dbt workloads default to static explosion; pass `--dbt-mode pydabs` to emit a deploy-time PyDABs hook +instead (see the dbt factory section of the coverage doc). + ## Step 1 — Run the translation ```bash diff --git a/skills/flowx-discover/sources/airflow.md b/skills/flowx-discover/sources/airflow.md index d347ab8..190cb44 100644 --- a/skills/flowx-discover/sources/airflow.md +++ b/skills/flowx-discover/sources/airflow.md @@ -62,3 +62,10 @@ Current deterministic coverage: `PythonOperator` (callable body → generated Py `BashOperator` (command → `%sh` notebook). Dependencies (`>>` / `<<`) and cron `schedule_interval` → Quartz are handled. Other operators become placeholders. Confirm the output location and proceed to `flowx-convert` with the same `` and `--source airflow`. + +For the full verified support matrix — classic operators, TaskFlow (`@dag`/`@task`), sensors, +TaskGroups (incl. group-level dependencies), and dbt factory (static + PyDABs) — plus the constructs +that are **not** handled (dynamic TaskGroup mapping, shared multi-DAG bundle), see +[`../../flowx-convert/sources/airflow-coverage.md`](../../flowx-convert/sources/airflow-coverage.md). +Callables reading Airflow task context (`**context` / `ti`) or XCom, and runtime-branching +decorators, are routed to placeholders for manual/agentic translation rather than converted. From bcedf5480e06adb472ce449c98f0d4e18d73a143 Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 20 Jul 2026 12:55:04 -0700 Subject: [PATCH 30/77] Harden Airflow runtime and conversion parity --- .../flowx-convert/sources/airflow-coverage.md | 39 +- skills/flowx-convert/sources/airflow.md | 4 +- .../sources/airflow/callable_notebook.py | 133 ++++- src/flowx/sources/airflow/loader.py | 337 ++++++++---- src/flowx/sources/airflow/operators.py | 93 ++-- src/flowx/sources/airflow/templating.py | 45 +- .../integration/test_airflow_golden_bundle.py | 2 +- tests/unit/test_airflow_operators.py | 481 +++++++++++++++++- 8 files changed, 933 insertions(+), 201 deletions(-) diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md index 6aaf010..a2ce62d 100644 --- a/skills/flowx-convert/sources/airflow-coverage.md +++ b/skills/flowx-convert/sources/airflow-coverage.md @@ -17,24 +17,25 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't | `BashOperator` / `SSHOperator` | `%sh` notebook; a wrapped `spark-submit` is lifted to a Spark JAR/Python task. | | `SparkSubmitOperator` | Spark JAR or Python task. | | Databricks provider operators (`DatabricksSubmitRun*`, `DatabricksRunNow*`, `DatabricksNotebookOperator`) | Notebook / run-job tasks. | -| SQL operators (`DatabricksSql*`, `SQLExecuteQueryOperator`, `PostgresOperator`, `MySqlOperator`, `HiveOperator`, `DatabricksCopyIntoOperator`) | `sql_task` (SqlActivity); Jinja → `:name` markers + `sql_task.parameters`. | +| SQL operators (`DatabricksSql*`, `SQLExecuteQueryOperator`, `PostgresOperator`, `MySqlOperator`, `HiveOperator`, `DatabricksCopyIntoOperator`) | `sql_task` (SqlActivity); Jinja values → `:name`, identifier positions → `IDENTIFIER(:name)`, with `sql_task.parameters`. | | `TriggerDagRunOperator` | `run_job_task` referencing the target DAG by sanitized job name. | | `EmailOperator` | Placeholder recommending job-level email notifications. | | dbt CLI operators (`DbtRun/Test/Seed/Snapshot/Build/Deps`) and Cosmos `DbtDag` / `DbtTaskGroup` | Single `DbtFactoryActivity`, **static explosion** (default) or **PyDABs** (`--dbt-mode pydabs`); see [dbt factory](#dbt-factory-mode). | | **TaskFlow API** (`@dag`, `@task`, `@task.virtualenv`) | Each `@task` invocation → a task; implicit XCom data flow (`transform(extract())`) → a notebook that reads upstream return values via `dbutils.jobs.taskValues.get`, calls the function, and publishes its own. `@task.branch` / `@task.short_circuit`, or a callable reading task context/XCom, route to a placeholder + gap. | | File sensors (`S3KeySensor`, `GCSObjectExistenceSensor`, `FileSensor`, `HdfsSensor`, `WebHdfsSensor`) | Root sensor with no schedule → `file_arrival` trigger; otherwise a `dbutils.fs` polling notebook task. | | Table/SQL sensors (`DatabricksPartitionSensor`, `DatabricksSqlSensor`, `DatabricksSQLStatementsSensor`, `SqlSensor`) | Root sensor naming a literal table with no schedule → `table_update` trigger; otherwise a `spark.sql` polling notebook task. | -| `ExternalTaskSensor` | Cross-DAG wait: a notebook polling the upstream DAG's Databricks job run state (referenced by sanitized job name). | -| `HttpSensor` / `PythonSensor` / `DateTimeSensor` | Polling notebook tasks (`requests` poll / callable poll / wait-until). A `PythonSensor` callable reading task context routes to a placeholder. | -| Time sensors (`TimeSensor`, `TimeDeltaSensor`) | Absorbed into the schedule (start-of-DAG delay); dropped with dependency rewiring. | +| `ExternalTaskSensor` | Placeholder explaining logical-run-aware migration options; polling the latest Databricks job run is not equivalent to Airflow's matching logical run. | +| `HttpSensor` / `PythonSensor` / `DateTimeSensor` | Polling notebook tasks for absolute HTTP URLs, callable polls, and wait-until. Relative HTTP endpoints and Python callables reading task context route to placeholders. | +| Time sensors (`TimeSensor`, `TimeDeltaSensor`) | Placeholder; their per-run wait semantics are not silently folded into or removed from the job schedule. | | `DummyOperator` / `EmptyOperator` | Dropped, downstream dependencies rewired. | | `.expand()` on an operator | `for_each_task`. | | Dependencies | `>>` / `<<` chains (incl. list/tuple fan-out and inline TaskFlow calls) and `set_upstream` / `set_downstream`. | | **TaskGroups** | Static nesting → task-key namespacing (`group__subgroup__task`); group-level edges (`group_a >> group_b`, `task >> group`) expand to leaf→root edges between member tasks. | -| Schedule | Cron `schedule_interval` → Quartz (Unix DOW 0–6 → Quartz 1–7); `timedelta` → periodic. | +| Schedule | Cron `schedule_interval` → Quartz (Unix DOW 0–6 → Quartz 1–7); exact sub-hour `timedelta` → Quartz, longer intervals → periodic, `@continuous` → continuous mode. | | `trigger_rule` | DAB `run_if` constant per edge (`ALL_DONE`, `ALL_FAILED`, `AT_LEAST_ONE_SUCCESS`, `NONE_FAILED`, …). | | Job parameters | `params={...}` / `Param(default=...)` → job parameters with defaults; `{{ params.x }}` / `{{ var.value.x }}` / `{{ dag_run.conf['x'] }}` → `{{job.parameters.x}}`. | -| `Variable.get` / `BaseHook.get_connection` in a callable | Rewritten to `dbutils.widgets.get` / `dbutils.secrets.get`. | +| `Variable.get` in a callable | Rewritten to `dbutils.widgets.get`; a callable using an Airflow `Connection` object routes to a placeholder because one secret string cannot preserve the object API. | +| Multiple DAGs | Every DAG, including multiple declarations in one Python file, becomes a sibling job in one shared Airflow bundle so `TriggerDagRunOperator` resource references resolve. | Any operator not listed becomes a `PlaceholderActivity` **and** a `gaps.json` entry carrying the operator's raw source, for LLM-assisted translation in the agentic-gap round. That is the safe @@ -52,14 +53,9 @@ are deliberate scope decisions. - **Sensors beyond the mapped families** (`S3PrefixSensor`, custom sensors, etc.) → placeholder + gap. A file sensor with a non-literal path, or a table/SQL sensor with no literal `sql` / `table_name`, also falls back to a placeholder. -- **Shared multi-DAG bundle** *(by design, for now)*. Converting a directory of N DAGs produces **N - independent bundles**, one per DAG in its own subdirectory — not a single bundle with N jobs. - Cross-DAG job references (`TriggerDagRunOperator`, `ExternalTaskSensor`) reference the sibling job by - sanitized name; that job lives in a separate bundle, so both bundles must be deployed to the same - workspace for the reference to resolve. -- **Scaffolding for dbt static mode.** Static explosion assumes the dbt project, `manifest.json`, and - profiles already ship in the bundle. No `pyproject.toml` / `Makefile` / `profiles.yml` scaffolding is - generated (PyDABs mode surfaces the setup steps in SETUP.md instead). +- **Dynamic dbt configuration.** Project/profile paths, selectors, excludes, vars, and full-refresh + flags must be statically visible. Missing project, profile, or manifest inputs produce a failing + setup-required placeholder rather than a partially deployable dbt job. ## dbt factory mode @@ -69,18 +65,17 @@ mode with `--dbt-mode {static,pydabs}` on the convert phase (default `static`). - **Static explosion (default).** Emits an inner job with one `notebook_task` per exploded dbt node (dependency-wired from a pruned `manifest.json`), a shared `run_dbt_command.py` runner notebook that - shells out to the dbt CLI, and a `run_job_task` hop from the parent. The manifest is read at package - time. **Assumes** the dbt project + `manifest.json` + profiles already exist in the bundle. + invokes the dbt CLI with pinned task libraries, and a `run_job_task` hop from the parent. The manifest + is read at package time; the available project, profile, and manifest files are copied into `src/`. - **PyDABs (`--dbt-mode pydabs`).** Emits a `resources/_dbt_job.py` hook (plus a `resources/__init__.py` package marker) at the bundle root, registers it under `databricks.yml` - `python.resources`, and surfaces the setup steps in SETUP.md (install `databricks-dbt-factory`, - ensure `manifest.json` exists). `bundle deploy` runs the hook to build the dbt job from the live - manifest. + `python.resources`, generates a pinned uv `pyproject.toml` plus the dbt-factory-compatible runner, + and copies the project/profile/manifest inputs. `bundle deploy` runs the hook to build the dbt job. + A source `--select` restriction falls back to static explosion so the generated per-node commands + can preserve dbt selector intersection semantics. ## Priority for remaining follow-ups 1. **Dynamic TaskGroup mapping** — expand a mapped `@task_group` / `TaskGroup.partial().expand()` into a for-each over the group's tasks (today the fan-out is lost). -2. **Shared multi-DAG bundle** — a single bundle with N jobs so cross-DAG references resolve within one - deploy (currently one bundle per DAG, by design). -3. **Additional sensor families** — as demand warrants; unmapped sensors route to a placeholder today. +2. **Additional sensor families** — as demand warrants; unmapped sensors route to a placeholder today. diff --git a/skills/flowx-convert/sources/airflow.md b/skills/flowx-convert/sources/airflow.md index 78aac0b..324f60c 100644 --- a/skills/flowx-convert/sources/airflow.md +++ b/skills/flowx-convert/sources/airflow.md @@ -11,7 +11,7 @@ placeholder — the same `gaps.json` + `merge_agentic` flow the ADF source uses. **Before converting, check [`sources/airflow-coverage.md`](airflow-coverage.md)** — the verified support matrix (classic operators, TaskFlow, sensors, TaskGroups, dbt factory) and the constructs -still **not** handled (dynamic TaskGroup mapping, shared multi-DAG bundle). Constructs flowx can't +still **not** handled (including dynamic TaskGroup mapping). Constructs flowx can't lower deterministically — callables reading task context (`**context` / `ti`) or XCom, and runtime-branching decorators — are routed to a placeholder + `gaps.json` for the agentic round rather than emitted as broken code. @@ -53,7 +53,7 @@ Write one result JSON per gap into `/agentic_results/` and merge the `merge_agentic` command (see the parent `SKILL.md`): ```bash -"$PY" -m flowx.adapter convert --merge-agentic \ +"$PY" -m flowx.adapter convert --source airflow --merge-agentic \ --report /.work/translation_report.json \ --agentic-results /agentic_results ``` diff --git a/src/flowx/sources/airflow/callable_notebook.py b/src/flowx/sources/airflow/callable_notebook.py index 75c584b..764fc90 100644 --- a/src/flowx/sources/airflow/callable_notebook.py +++ b/src/flowx/sources/airflow/callable_notebook.py @@ -4,12 +4,13 @@ transitive module-level dependencies (helper functions, literal constants, non-Airflow imports) are carried, and ``op_args`` / ``op_kwargs`` are passed as JSON widgets and splatted into a call. Airflow/provider imports are dropped (they fail on Databricks); -Variable/connection access is rewritten by :func:`flowx.sources.airflow.templating.rewrite_airflow_calls`. +Airflow variables are rewritten, while Connection-object usage is routed to a placeholder. """ from __future__ import annotations import ast +import builtins from flowx.sources.airflow import templating @@ -17,24 +18,58 @@ _AIRFLOW_IMPORT_ROOTS: frozenset[str] = frozenset({"airflow", "cosmos", "airflow_dbt"}) -def _module_symbols(module: ast.Module) -> tuple[dict[str, ast.stmt], dict[str, ast.stmt]]: - """Returns ``(defs, assigns)`` -- module-level function/class defs and simple constant assigns.""" +def _enclosing_statements(module: ast.Module, func: ast.FunctionDef) -> list[ast.stmt]: + """Returns safe statements visible from the callable's enclosing function scopes.""" + scopes = [ + node + for node in ast.walk(module) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.lineno < func.lineno <= (node.end_lineno or node.lineno) + ] + visible: list[ast.stmt] = [] + for scope in sorted(scopes, key=lambda node: node.lineno): + for statement in scope.body: + if statement.lineno >= func.lineno: + continue + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom)): + visible.append(statement) + elif isinstance(statement, ast.Assign): + try: + ast.literal_eval(statement.value) + except (ValueError, SyntaxError): + continue + visible.append(statement) + elif isinstance(statement, ast.AnnAssign) and statement.value is not None: + try: + ast.literal_eval(statement.value) + except (ValueError, SyntaxError): + continue + visible.append(statement) + return visible + + +def _module_symbols( + module: ast.Module, enclosing_statements: list[ast.stmt] +) -> tuple[dict[str, ast.stmt], dict[str, ast.stmt]]: + """Returns visible function/class definitions and constant assignments.""" defs: dict[str, ast.stmt] = {} assigns: dict[str, ast.stmt] = {} - for node in module.body: + for node in [*module.body, *enclosing_statements]: if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): defs[node.name] = node elif isinstance(node, ast.Assign): for target in node.targets: if isinstance(target, ast.Name): assigns[target.id] = node + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.value is not None: + assigns[node.target.id] = node return defs, assigns -def _import_bindings(module: ast.Module) -> dict[str, tuple[ast.stmt, str]]: +def _import_bindings(module: ast.Module, enclosing_statements: list[ast.stmt]) -> dict[str, tuple[ast.stmt, str]]: """Maps each imported name -> (import stmt, root module) for non-Airflow import filtering.""" bindings: dict[str, tuple[ast.stmt, str]] = {} - for node in module.body: + for node in [*module.body, *enclosing_statements]: if isinstance(node, ast.Import): for alias in node.names: bound = (alias.asname or alias.name).split(".")[0] @@ -67,7 +102,7 @@ def _closure( seen.add(func.name) while queue: name = queue.pop(0) - node = defs.get(name) or assigns.get(name) + node = func if name == func.name else defs.get(name) or assigns.get(name) if node is None: continue used = _names_used(node) @@ -78,6 +113,7 @@ def _closure( if used_name not in seen and (used_name in defs or used_name in assigns): seen.add(used_name) queue.append(used_name) + ordered.sort(key=lambda name: (defs.get(name) or assigns[name]).lineno) return ordered, all_names @@ -86,11 +122,12 @@ def render_definitions(func: ast.FunctionDef, source: str, *, note: str) -> str: Carried: an ``import json`` line, the non-Airflow imports the callable/helpers use, the referenced module-level helpers/constants, and *func* verbatim. Variable/connection access is - rewritten. The caller appends its own invocation (a splatted call, a poll loop, ...). + rewritten after callers reject Connection-object usage. The caller appends its own invocation. """ module = ast.parse(source) - defs, assigns = _module_symbols(module) - imports = _import_bindings(module) + enclosing_statements = _enclosing_statements(module, func) + defs, assigns = _module_symbols(module, enclosing_statements) + imports = _import_bindings(module, enclosing_statements) dep_names, used_names = _closure(func, defs, assigns) @@ -173,7 +210,30 @@ def _returns_value(func: ast.FunctionDef) -> bool: # Airflow injects execution context (the templated context dict, the task instance ``ti``, XCom) # into a callable at runtime. flowx runs the callable as a plain notebook with no Airflow runtime, # so a callable that reads task context or XCom cannot be lowered deterministically. -_TASK_CONTEXT_PARAMS: frozenset[str] = frozenset({"ti", "task_instance"}) +_TASK_CONTEXT_PARAMS: frozenset[str] = frozenset( + { + "conf", + "dag", + "dag_run", + "data_interval_end", + "data_interval_start", + "ds", + "ds_nodash", + "execution_date", + "logical_date", + "macros", + "params", + "run_id", + "task", + "task_instance", + "templates_dict", + "ti", + "ts", + "ts_nodash", + "ts_nodash_with_tz", + "var", + } +) _XCOM_METHODS: frozenset[str] = frozenset({"xcom_pull", "xcom_push"}) @@ -196,3 +256,54 @@ def task_context_reason(func: ast.FunctionDef) -> str | None: if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr in _XCOM_METHODS: return f"callable calls {node.func.attr}() (XCom)" return None + + +def airflow_runtime_reason(func: ast.FunctionDef, source: str) -> str | None: + """Returns why a callable requires Airflow runtime behavior that cannot be emitted safely.""" + context_reason = task_context_reason(func) + if context_reason is not None: + return context_reason + + module = ast.parse(source) + enclosing_statements = _enclosing_statements(module, func) + definitions, assignments = _module_symbols(module, enclosing_statements) + dependency_names, _ = _closure(func, definitions, assignments) + closure_nodes = [ + func, + *(definitions.get(name) or assignments[name] for name in dependency_names), + ] + closure_source = "\n".join(filter(None, (ast.get_source_segment(source, node) for node in closure_nodes))) + connections = sorted(templating.airflow_connection_names(closure_source)) + if connections: + return f"callable reads Airflow connection '{connections[0]}' as a Connection object" + unresolved_names = _unresolved_closure_names(func, source) + if unresolved_names: + return f"captures nonliteral closure '{unresolved_names[0]}'" + return None + + +def _unresolved_closure_names(func: ast.FunctionDef, source: str) -> list[str]: + """Returns loaded names that the generated standalone definition cannot resolve.""" + module = ast.parse(source) + enclosing_statements = _enclosing_statements(module, func) + definitions, assignments = _module_symbols(module, enclosing_statements) + imports = _import_bindings(module, enclosing_statements) + + bound = {func.name, *definitions, *assignments, *imports, *dir(builtins), "dbutils", "sc", "spark"} + for node in ast.walk(func): + if isinstance(node, ast.arg): + bound.add(node.arg) + elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store): + bound.add(node.id) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + bound.add(node.name) + elif isinstance(node, ast.Import): + bound.update((alias.asname or alias.name).split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + bound.update(alias.asname or alias.name for alias in node.names) + elif isinstance(node, ast.ExceptHandler) and node.name: + bound.add(node.name) + + decorator_names = {name for decorator in func.decorator_list for name in _names_used(decorator)} + unresolved = _names_used(func) - bound - decorator_names + return sorted(unresolved) diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py index c984b03..c671d70 100644 --- a/src/flowx/sources/airflow/loader.py +++ b/src/flowx/sources/airflow/loader.py @@ -13,7 +13,7 @@ (branch/virtualenv, cosmos ``DbtTaskGroup`` -> DbtFactoryActivity, Dummy/Empty dropped + rewired), Tier 3 sensors (a root file/table sensor with no schedule -> ``file_arrival`` / ``table_update`` trigger, otherwise retained as a polling task; -time sensors -> schedule), and Tier 4 (unmapped -> PlaceholderActivity). +time sensors -> PlaceholderActivity), and Tier 4 (unmapped -> PlaceholderActivity). ``>>`` / ``<<`` dependencies and cron ``schedule_interval`` -> Quartz are handled here. """ @@ -46,7 +46,7 @@ class _TaskFlowTask: ``positional_deps`` / ``keyword_deps`` map each argument position / keyword the callable was invoked with to the upstream task var it references (TaskFlow's implicit XCom data flow), so the emitted notebook can read that upstream's return value via ``dbutils.jobs.taskValues``. Literal - args are ignored (the callable's own defaults apply). + args are preserved when literal and routed to a placeholder when they cannot be resolved safely. """ task_id: str @@ -54,6 +54,9 @@ class _TaskFlowTask: decorator: str positional_deps: dict[int, str] = field(default_factory=dict) keyword_deps: dict[str, str] = field(default_factory=dict) + positional_values: dict[int, str] = field(default_factory=dict) + keyword_values: dict[str, str] = field(default_factory=dict) + unresolved_arguments: list[str] = field(default_factory=list) def _sanitize_task_key(name: str) -> str: @@ -184,17 +187,7 @@ def _timedelta_to_periodic(node: ast.expr | None) -> dict[str, object] | None: maps to the largest exact unit; anything finer (minutes/seconds) is expressed as a cron in the caller, so this returns None for those. """ - if not isinstance(node, ast.Call): - return None - func = node.func - name = func.attr if isinstance(func, ast.Attribute) else (func.id if isinstance(func, ast.Name) else "") - if name != "timedelta": - return None - total = 0 - for kw in node.keywords: - if kw.arg in _TIMEDELTA_UNIT_SECONDS and isinstance(kw.value, ast.Constant): - if isinstance(kw.value.value, int): - total += kw.value.value * _TIMEDELTA_UNIT_SECONDS[kw.arg] + total = _timedelta_seconds(node) if total <= 0: return None for unit, unit_seconds in (("WEEKS", 604800), ("DAYS", 86400), ("HOURS", 3600)): @@ -203,6 +196,22 @@ def _timedelta_to_periodic(node: ast.expr | None) -> dict[str, object] | None: return None +def _timedelta_seconds(node: ast.expr | None) -> int: + """Returns the number of seconds in a literal timedelta call, or zero.""" + if not isinstance(node, ast.Call): + return 0 + func = node.func + name = func.attr if isinstance(func, ast.Attribute) else (func.id if isinstance(func, ast.Name) else "") + if name != "timedelta": + return 0 + total = 0 + for keyword in node.keywords: + if keyword.arg in _TIMEDELTA_UNIT_SECONDS and isinstance(keyword.value, ast.Constant): + if isinstance(keyword.value.value, int): + total += keyword.value.value * _TIMEDELTA_UNIT_SECONDS[keyword.arg] + return total + + def _schedule_from_interval( interval: str | None, *, @@ -215,6 +224,8 @@ def _schedule_from_interval( a ``timedelta(...)`` -> ``kind: periodic``. Returns None when neither applies. """ if interval: + if interval == "@continuous": + return {"kind": "continuous", "pause_status": "UNPAUSED"} quartz: str | None = _CRON_PRESETS.get(interval) or _cron_to_quartz(interval) if quartz is not None: return { @@ -223,7 +234,27 @@ def _schedule_from_interval( "timezone_id": timezone or "UTC", "pause_status": "UNPAUSED", } - return _timedelta_to_periodic(node) + periodic = _timedelta_to_periodic(node) + if periodic is not None: + return periodic + total_seconds = _timedelta_seconds(node) + if 0 < total_seconds < 60 and 60 % total_seconds == 0: + return { + "kind": "schedule", + "quartz_cron_expression": f"0/{total_seconds} * * * * ?", + "timezone_id": timezone or "UTC", + "pause_status": "UNPAUSED", + } + if total_seconds % 60 == 0: + minutes = total_seconds // 60 + if 0 < minutes < 60 and 60 % minutes == 0: + return { + "kind": "schedule", + "quartz_cron_expression": f"0 0/{minutes} * * * ?", + "timezone_id": timezone or "UTC", + "pause_status": "UNPAUSED", + } + return None class _DagVisitor(ast.NodeVisitor): @@ -314,12 +345,19 @@ def _taskflow_def_name(self, call: ast.Call) -> tuple[str | None, bool, str | No func = call.func mapped = False override_id: str | None = None - while isinstance(func, ast.Attribute): - if func.attr == "expand": - mapped = True - elif func.attr == "override": - override_id = ops.literal_str({kw.arg: kw.value for kw in call.keywords if kw.arg}.get("task_id")) - func = func.value + while True: + if isinstance(func, ast.Attribute): + if func.attr == "expand": + mapped = True + func = func.value + continue + if isinstance(func, ast.Call) and isinstance(func.func, ast.Attribute): + if func.func.attr == "override": + arguments = {keyword.arg: keyword.value for keyword in func.keywords if keyword.arg} + override_id = ops.literal_str(arguments.get("task_id")) + func = func.func.value + continue + break if isinstance(func, ast.Name) and func.id in self.taskflow_defs: return func.id, mapped, override_id return None, mapped, override_id @@ -349,13 +387,26 @@ def _register_taskflow_call(self, call: ast.Call, var: str) -> bool: if dep is not None: task.positional_deps[index] = dep self.edges.append((dep, var)) + else: + value = _literal_argument_source(arg) + if value is None: + task.unresolved_arguments.append(ast.unparse(arg)) + else: + task.positional_values[index] = value for kw in call.keywords: if kw.arg is None: + task.unresolved_arguments.append(f"**{ast.unparse(kw.value)}") continue dep = self._resolve_taskflow_arg(kw.value) if dep is not None: task.keyword_deps[kw.arg] = dep self.edges.append((dep, var)) + else: + value = _literal_argument_source(kw.value) + if value is None: + task.unresolved_arguments.append(f"{kw.arg}={ast.unparse(kw.value)}") + else: + task.keyword_values[kw.arg] = value return True def _resolve_taskflow_arg(self, arg: ast.expr) -> str | None: @@ -365,7 +416,7 @@ def _resolve_taskflow_arg(self, arg: ast.expr) -> str | None: is registered as its own synthetic task instance and its var returned, so the whole expression tree becomes a chain of task instances. """ - if isinstance(arg, ast.Name): + if isinstance(arg, ast.Name) and (arg.id in self.operators or arg.id in self.taskflow_tasks): return arg.id if isinstance(arg, ast.Call): def_name, _mapped, _override = self._taskflow_def_name(arg) @@ -445,27 +496,29 @@ def visit_Expr(self, node: ast.Expr) -> None: elif isinstance(value, ast.Call): # A bare TaskFlow call (`extract()` with no assignment) is a task instance keyed by its # def name; otherwise it may be a set_upstream/set_downstream dependency call. - func = value.func - if isinstance(func, ast.Name) and func.id in self.taskflow_defs and func.id not in self.taskflow_tasks: - self._register_taskflow_call(value, func.id) + def_name, _mapped, _override = self._taskflow_def_name(value) + if def_name is not None: + task_var = def_name + if task_var in self.taskflow_tasks: + self._taskflow_counter += 1 + task_var = f"{def_name}__tf{self._taskflow_counter}" + self._register_taskflow_call(value, task_var) else: self._collect_set_dependency(value) self.generic_visit(node) def _collect_shift_chain(self, binop: ast.BinOp) -> None: - # Each chain position is a *group* of task names (a bare name, a [list], or an inline - # TaskFlow call like `extract()`); adjacent groups are connected as a cross-product so - # `a >> [b, c]` yields a->b and a->c. - groups = [self._shift_position_names(node) for node in _flatten_shift_nodes(binop)] - groups = [g for g in groups if g] - if len(groups) < 2: - return - rightward = isinstance(binop.op, ast.RShift) - for upstream_group, downstream_group in zip(groups, groups[1:]): - up, down = (upstream_group, downstream_group) if rightward else (downstream_group, upstream_group) - for u in up: - for d in down: - self.edges.append((u, d)) + self._collect_shift_expression(binop) + + def _collect_shift_expression(self, node: ast.expr) -> list[str]: + """Collects each shift edge recursively and returns the expression's chain result.""" + if not isinstance(node, ast.BinOp) or not isinstance(node.op, (ast.RShift, ast.LShift)): + return self._shift_position_names(node) + left = self._collect_shift_expression(node.left) + right = self._collect_shift_expression(node.right) + upstream, downstream = (left, right) if isinstance(node.op, ast.RShift) else (right, left) + self.edges.extend((upstream_var, downstream_var) for upstream_var in upstream for downstream_var in downstream) + return right def _shift_position_names(self, node: ast.expr) -> list[str]: # A shift-chain position resolves to task vars. An inline TaskFlow call (`extract()`) is @@ -579,6 +632,14 @@ def _names_in(node: ast.expr) -> list[str]: return [] +def _literal_argument_source(node: ast.expr) -> str | None: + """Returns stable Python source for a literal TaskFlow call argument when available.""" + try: + return repr(ast.literal_eval(node)) + except (ValueError, SyntaxError): + return None + + def _flatten_shift_nodes(node: ast.expr) -> list[ast.expr]: """Flattens a ``>>`` / ``<<`` chain into its per-position operand nodes, left to right. @@ -673,7 +734,57 @@ def _has_decorator(func: ast.FunctionDef, names: frozenset[str]) -> bool: def load_airflow_dag(dag_path: Path, *, dbt_mode: str = "static") -> Pipeline: - """Parses an Airflow DAG file into a flowx Pipeline IR. + """Parses the first Airflow DAG in a file into a flowx Pipeline IR.""" + pipelines = load_airflow_dags(dag_path, dbt_mode=dbt_mode) + if not pipelines: + raise ValueError(f"No Airflow DAG found in {dag_path}") + return pipelines[0] + + +def load_airflow_dags(dag_path: Path, *, dbt_mode: str = "static") -> list[Pipeline]: + """Parses every independently declared Airflow DAG in a Python file.""" + source = Path(dag_path).read_text(encoding="utf-8") + module = ast.parse(source) + dag_nodes = _top_level_dag_nodes(module) + if not dag_nodes: + return [_load_airflow_module(dag_path, source, module, dbt_mode=dbt_mode)] + return [ + _load_airflow_module(dag_path, source, _module_for_dag(module, dag_node), dbt_mode=dbt_mode) + for dag_node in dag_nodes + ] + + +def _top_level_dag_nodes(module: ast.Module) -> list[ast.stmt]: + """Returns top-level context-manager and decorated-function DAG declarations.""" + declarations: list[ast.stmt] = [] + for node in module.body: + if isinstance(node, ast.FunctionDef) and _has_decorator(node, _DAG_DECORATORS): + declarations.append(node) + elif isinstance(node, ast.With) and any( + isinstance(item.context_expr, ast.Call) + and isinstance(item.context_expr.func, ast.Name) + and item.context_expr.func.id == "DAG" + for item in node.items + ): + declarations.append(node) + return declarations + + +def _module_for_dag(module: ast.Module, dag_node: ast.stmt) -> ast.Module: + """Returns a module containing shared definitions and one DAG declaration.""" + dag_nodes = set(_top_level_dag_nodes(module)) + body = [node for node in module.body if node is dag_node or node not in dag_nodes] + return ast.Module(body=body, type_ignores=list(module.type_ignores)) + + +def _load_airflow_module( + dag_path: Path, + source: str, + module: ast.Module, + *, + dbt_mode: str = "static", +) -> Pipeline: + """Parses one isolated DAG declaration into a flowx Pipeline IR. Args: dag_path: Path to a ``.py`` DAG module. @@ -685,11 +796,9 @@ def load_airflow_dag(dag_path: Path, *, dbt_mode: str = "static") -> Pipeline: node (NotebookActivity, SparkPython/JarActivity, RunJobActivity, DbtFactoryActivity, ...); Dummy/Empty are dropped with dependency rewiring; file sensors lift to a job-level file_arrival trigger; time - sensors are absorbed into the schedule; unmapped operators become a + sensors remain explicit placeholders; unmapped operators become a PlaceholderActivity. """ - source = Path(dag_path).read_text(encoding="utf-8") - module = ast.parse(source) visitor = _DagVisitor(module) visitor.visit(module) functions = visitor.functions() @@ -712,7 +821,7 @@ def _task_key(var: str, task_id: str) -> str: edges = _expand_group_edges(visitor.edges, visitor.operators, visitor.groups, visitor.group_vars) # Build the upstream adjacency in dependency terms, then drop structural nodes - # (Dummy/Empty, file/time sensors) by rewiring their downstreams to their upstreams. + # (Dummy/Empty and lifted root sensors) by rewiring their downstreams to their upstreams. upstreams: dict[str, list[str]] = {var: [] for var in var_task_ids} for upstream_var, downstream_var in edges: if downstream_var in upstreams and upstream_var in var_to_task_key: @@ -727,12 +836,8 @@ def _task_key(var: str, task_id: str) -> str: schedule = _schedule_from_interval(visitor.schedule_interval, node=visitor.schedule_node, timezone=visitor.timezone) has_schedule = schedule is not None - # Dummy/Empty and time sensors always drop (structural / absorbed into the schedule as a delay). - dropped = { - var - for var, (_, op, _) in visitor.operators.items() - if op in ops.DUMMY_OPERATORS or op in ops.TIME_SENSORS - } + # Dummy/Empty operators are structural and can be removed after dependency rewiring. + dropped = {var for var, (_, op, _) in visitor.operators.items() if op in ops.DUMMY_OPERATORS} if not has_schedule: trigger_var = _root_trigger_sensor(visitor.operators, upstreams) if trigger_var is not None: @@ -769,7 +874,9 @@ def _dep(upstream_var: str, outcome: str | None) -> str: depends_on = [Dependency(task_key=k, outcome=outcome) for k in sorted(dep_keys)] or None if operator in ops.COSMOS_CONSTRUCTS: - tasks.append(_build_dbt_factory(task_id, task_key, [kwargs], depends_on, dbt_mode)) + tasks.append( + _build_dbt_factory(task_id, task_key, [kwargs], depends_on, dbt_mode, operator_types=[operator]) + ) continue if operator in ops.DBT_CLI_OPERATORS: # Emit one factory job for the whole dbt chain, at the first dbt task's position. @@ -777,7 +884,16 @@ def _dep(upstream_var: str, outcome: str | None) -> str: continue emitted_dbt = True dbt_kwargs = [visitor.operators[v][2] for v in dbt_vars] - tasks.append(_build_dbt_factory(task_id, task_key, dbt_kwargs, depends_on, dbt_mode)) + tasks.append( + _build_dbt_factory( + task_id, + task_key, + dbt_kwargs, + depends_on, + dbt_mode, + operator_types=[visitor.operators[dbt_var][1] for dbt_var in dbt_vars], + ) + ) continue call_node = visitor.calls.get(var) @@ -802,6 +918,15 @@ def _dep(upstream_var: str, outcome: str | None) -> str: activity.min_retry_interval_millis = policy.get("min_retry_interval_millis") # Convert Airflow Jinja in the activity's parameter fields to DAB refs; collect params. referenced_params |= _convert_activity_templates(activity) + unresolved_templates = _unresolved_activity_templates(activity) + if unresolved_templates: + expressions = ", ".join(sorted(unresolved_templates)) + activity = ops.build_placeholder_with_comment( + ctx, + f"Airflow template expression(s) {expressions} have no deterministic Databricks mapping; " + "translate the value manually.", + ) + activity.depends_on = depends_on if var in visitor.mapped: # Dynamic mapping (.expand()) -> a for_each_task iterating the mapped operator. @@ -903,6 +1028,15 @@ def _build_taskflow_task( original_type=f"@{tf.decorator}", comment=f"TaskFlow @{tf.decorator} '{tf.def_name}' could not be resolved; translate manually.", ) + if tf.unresolved_arguments: + arguments = ", ".join(tf.unresolved_arguments) + return PlaceholderActivity( + name=tf.task_id, + task_key=task_key, + original_type=f"@{tf.decorator}", + comment=f"TaskFlow call uses nonliteral argument(s) {arguments}; bind them manually.", + raw_definition={"operator": f"@{tf.decorator}", "source": ast.get_source_segment(source, func) or ""}, + ) if tf.decorator in _TASKFLOW_BRANCHING: return PlaceholderActivity( name=tf.task_id, @@ -915,7 +1049,7 @@ def _build_taskflow_task( ), raw_definition={"operator": f"@{tf.decorator}", "source": ast.get_source_segment(source, func) or ""}, ) - reason = callable_notebook.task_context_reason(func) + reason = callable_notebook.airflow_runtime_reason(func, source) if reason is not None: return PlaceholderActivity( name=tf.task_id, @@ -953,26 +1087,18 @@ def _reader(dep_var: str) -> str: dep_key = var_to_task_key.get(dep_var, dep_var) return f"dbutils.jobs.taskValues.get(taskKey='{dep_key}', key='return_value', debugValue=None)" - # Positional args must stay contiguous from index 0 -- only bind a leading run of positions so a - # gap doesn't shift later args. Any remaining bound positions are passed by parameter name. - param_names = [a.arg for a in (func.args.posonlyargs + func.args.args)] - index = 0 - while index in tf.positional_deps: - var = f"_upstream_{index}" - lines.append(f"{var} = {_reader(tf.positional_deps[index])}") - call_positional.append(var) - index += 1 - for pos, dep_var in sorted(tf.positional_deps.items()): - if pos < index or pos >= len(param_names): - continue - name = param_names[pos] - var = f"_upstream_{name}" - lines.append(f"{var} = {_reader(dep_var)}") - call_keywords.append(f"{name}={var}") + for position in sorted(set(tf.positional_deps) | set(tf.positional_values)): + if position in tf.positional_deps: + variable = f"_upstream_{position}" + lines.append(f"{variable} = {_reader(tf.positional_deps[position])}") + call_positional.append(variable) + else: + call_positional.append(tf.positional_values[position]) for name, dep_var in tf.keyword_deps.items(): - var = f"_upstream_{name}" - lines.append(f"{var} = {_reader(dep_var)}") - call_keywords.append(f"{name}={var}") + variable = f"_upstream_{name}" + lines.append(f"{variable} = {_reader(dep_var)}") + call_keywords.append(f"{name}={variable}") + call_keywords.extend(f"{name}={value}" for name, value in tf.keyword_values.items()) call_args = ", ".join(call_positional + call_keywords) returns = any(isinstance(n, ast.Return) and n.value is not None for n in ast.walk(func)) @@ -1018,6 +1144,14 @@ def _convert_activity_templates(activity: Activity) -> set[str]: _JOB_PARAM_REF = re.compile(r"\{\{\s*job\.parameters\.([A-Za-z0-9_]+)\s*\}\}") +def _unresolved_activity_templates(activity: Activity) -> set[str]: + """Returns residual Airflow Jinja expressions in task parameter fields.""" + unresolved: set[str] = set() + for attribute in ("base_parameters", "job_parameters", "parameters", "sql"): + unresolved |= templating.unresolved_jinja_expressions(getattr(activity, attribute, None)) + return unresolved + + def _rewire_dropped(upstreams: dict[str, list[str]], dropped: set[str]) -> dict[str, list[str]]: """Returns upstream edges with *dropped* vars removed and their edges bridged. @@ -1052,11 +1186,7 @@ def _root_trigger_sensor( take a single trigger). A table/SQL sensor lifts only when it names a literal table; one without a ``table_name`` is an arbitrary-condition sensor kept as a polling task. """ - file_roots = [ - var - for var, (_id, op, _kw) in operators.items() - if op in ops.FILE_SENSORS and not upstreams.get(var) - ] + file_roots = [var for var, (_id, op, _kw) in operators.items() if op in ops.FILE_SENSORS and not upstreams.get(var)] if file_roots: return file_roots[0] for var, (_id, op, kw) in operators.items(): @@ -1072,13 +1202,7 @@ def _trigger_from_sensor(operator: str, kwargs: dict[str, ast.expr]) -> dict[str ``table_name`` -> ``trigger.table_update``. Returns None when the sensor can't lift. """ if operator in ops.FILE_SENSORS: - url = ( - ops.literal_str(kwargs.get("bucket_key")) - or ops.literal_str(kwargs.get("filepath")) - or ops.literal_str(kwargs.get("filepath_")) - or ops.literal_str(kwargs.get("bucket_name")) - or "" - ) + url = ops.file_sensor_path(kwargs) or "" return {"kind": "file_arrival", "url": url, "pause_status": "UNPAUSED"} if operator in ops.TABLE_SENSORS: table_name = ops.literal_str(kwargs.get("table_name")) @@ -1098,6 +1222,7 @@ def _build_dbt_factory( kwargs_list: list[dict[str, ast.expr]], depends_on: list[Dependency] | None, dbt_mode: str = "static", + operator_types: list[str] | None = None, ) -> DbtFactoryActivity: """Builds a DbtFactoryActivity from cosmos config or a set of dbt CLI operators. @@ -1109,6 +1234,10 @@ def _build_dbt_factory( profiles_dir = "dbt_profiles" target = "dev" manifest_path: str | None = None + selectors: list[str] = [] + exclude_selectors: list[str] = [] + variables: dict[str, Any] | str | None = None + full_refresh = False for kwargs in kwargs_list: # dbt CLI operators pass project_dir/target directly as kwargs. project_dir = ops.literal_str(kwargs.get("project_dir")) or ops.literal_str(kwargs.get("dir")) or project_dir @@ -1118,12 +1247,30 @@ def _build_dbt_factory( project_dir = _cosmos_project_dir(kwargs.get("project_config")) or project_dir target = _cosmos_target(kwargs.get("profile_config")) or target manifest_path = _cosmos_manifest_path(kwargs.get("project_config")) or manifest_path - # The static preparer needs a manifest to explode into tasks. Point it at the per-target - # manifest `make manifest` produces (target//manifest.json), rooted at the project dir, - # unless cosmos gave an explicit manifest_path. Without this the child job would be empty. + selectors.extend(_dbt_selector_list(ops.literal_value(kwargs.get("select") or kwargs.get("models")))) + exclude_selectors.extend(_dbt_selector_list(ops.literal_value(kwargs.get("exclude")))) + dbt_variables = ops.literal_value(kwargs.get("vars")) + if isinstance(dbt_variables, (dict, str)): + variables = dbt_variables + full_refresh = full_refresh or ops.literal_value(kwargs.get("full_refresh")) is True + # The static preparer needs the standard manifest produced under target/ unless Cosmos supplied + # an explicit manifest path. Without this the child job would be empty. if manifest_path is None: base = project_dir.rstrip("/") if project_dir not in ("", ".") else "." - manifest_path = f"{base}/target/{target}/manifest.json" if base != "." else f"target/{target}/manifest.json" + manifest_path = f"{base}/target/manifest.json" if base != "." else "target/manifest.json" + commands = { + ops.DBT_OPERATOR_COMMAND[operator] for operator in operator_types or [] if operator in ops.DBT_OPERATOR_COMMAND + } + resource_types: set[str] = set() + for command in commands: + if command == "build": + resource_types.update(("model", "seed", "snapshot", "test")) + elif command == "deps": + resource_types.add("dependency") + else: + resource_types.add({"run": "model", "seed": "seed", "snapshot": "snapshot", "test": "test"}[command]) + if not operator_types or any(operator in ops.COSMOS_CONSTRUCTS for operator in operator_types): + resource_types.update(("model", "seed", "snapshot", "test")) return DbtFactoryActivity( name=task_id, task_key=task_key, @@ -1133,9 +1280,23 @@ def _build_dbt_factory( target=target, manifest_path=manifest_path, render_mode="pydabs" if dbt_mode == "pydabs" else "static", + selectors=list(dict.fromkeys(selectors)), + exclude_selectors=list(dict.fromkeys(exclude_selectors)), + variables=variables, + full_refresh=full_refresh, + resource_types=sorted(resource_types), ) +def _dbt_selector_list(value: Any) -> list[str]: + """Returns literal dbt selectors as a normalized string list.""" + if isinstance(value, str): + return [value] + if isinstance(value, (list, tuple)): + return [selector for selector in value if isinstance(selector, str)] + return [] + + def _cosmos_project_dir(node: ast.expr | None) -> str | None: """Extracts the dbt project path from a cosmos ``ProjectConfig(...)`` call. @@ -1202,7 +1363,9 @@ def load_pipelines(source_path: Path, pipeline: str | None = None, *, dbt_mode: One :class:`~flowx.models.ir.Pipeline` per discovered DAG, filtered to *pipeline* when provided. """ - pipelines = [load_airflow_dag(dag_path, dbt_mode=dbt_mode) for dag_path in discover_dags(source_path)] + pipelines = [ + loaded for dag_path in discover_dags(source_path) for loaded in load_airflow_dags(dag_path, dbt_mode=dbt_mode) + ] if pipeline is not None: pipelines = [p for p in pipelines if p.name == pipeline] return pipelines diff --git a/src/flowx/sources/airflow/operators.py b/src/flowx/sources/airflow/operators.py index e45140a..9d8ef6c 100644 --- a/src/flowx/sources/airflow/operators.py +++ b/src/flowx/sources/airflow/operators.py @@ -4,9 +4,9 @@ subclass the flowx bundler can render (``notebook_task`` / ``spark_python_task`` / ``spark_jar_task`` / ``sql_task`` / ``run_job_task`` / ``condition_task`` / ``for_each_task``). -Structural operators (Dummy/Empty) and time sensors are classified here but dropped by the -loader (Dummy/Empty with dependency rewiring; time sensors absorbed into the schedule). A file -or table sensor at the DAG root with no schedule lifts to a job-level ``file_arrival`` / +Structural operators (Dummy/Empty) are dropped by the loader with dependency rewiring. Time +sensors remain explicit placeholders because a job schedule cannot preserve their per-run wait +semantics. A file or table sensor at the DAG root with no schedule lifts to a job-level ``file_arrival`` / ``table_update`` trigger; otherwise (mid-DAG, or under a schedule) it is retained as a polling notebook task via :func:`_build_file_sensor` / :func:`_build_table_sensor`. Operators with no deterministic mapping become a PlaceholderActivity carrying guidance. @@ -30,6 +30,7 @@ SqlActivity, ) from flowx.sources.airflow import callable_notebook +from flowx.utils import normalize_task_key # -------------------------------------------------------------------------------------- # Operator classification (handled specially by the loader, not via a task builder) @@ -44,9 +45,6 @@ {"S3KeySensor", "GCSObjectExistenceSensor", "FileSensor", "HdfsSensor", "WebHdfsSensor"} ) -# Absorbed into the job schedule (a start-of-DAG delay); dropped with a migration note. -TIME_SENSORS: frozenset[str] = frozenset({"TimeSensor", "TimeDeltaSensor"}) - # Table/SQL sensors: a root table sensor naming a literal table with no schedule lifts to a # job-level table_update trigger; otherwise it is retained as a spark.sql polling task # (_build_table_sensor). A sensor with no literal sql/table_name becomes a placeholder. @@ -58,7 +56,6 @@ # a seed>>run>>test chain into one factory job). DBT_CLI_OPERATORS: frozenset[str] = frozenset( { - "DbtOperator", "DbtRunOperator", "DbtTestOperator", "DbtSeedOperator", @@ -78,6 +75,7 @@ "DbtSeedOperator": "seed", "DbtSnapshotOperator": "snapshot", "DbtBuildOperator": "build", + "DbtDepsOperator": "deps", } @@ -157,7 +155,7 @@ def notebook_from_callable( Preserves the callable's full ``def`` (early returns stay legal), carries its transitive module-level dependencies (helpers / constants / non-Airflow imports), and invokes it with - ``op_args`` / ``op_kwargs`` read from JSON widgets. Variable/connection access is rewritten. + ``op_args`` / ``op_kwargs`` read from JSON widgets. Airflow variable access is rewritten. """ return callable_notebook.render(func, source, op_args=op_args, op_kwargs=op_kwargs) @@ -204,7 +202,7 @@ def _poll_body(operator: str, check_expr: str, description: str, poke: int, time ) -def _file_sensor_path(kwargs: dict[str, ast.expr]) -> str | None: +def file_sensor_path(kwargs: dict[str, ast.expr]) -> str | None: """Best-effort literal storage path a file sensor waits on (S3/GCS/File/HDFS).""" bucket_key = literal_str(kwargs.get("bucket_key")) bucket_name = literal_str(kwargs.get("bucket_name")) @@ -221,7 +219,7 @@ def _file_sensor_path(kwargs: dict[str, ast.expr]) -> str | None: def _build_file_sensor(ctx: OperatorContext) -> Activity: """A retained file sensor -> a notebook that polls dbutils.fs for the awaited path.""" - path = _file_sensor_path(ctx.kwargs) + path = file_sensor_path(ctx.kwargs) if path is None: return _placeholder( ctx, @@ -288,12 +286,7 @@ def _build_table_sensor(ctx: OperatorContext) -> Activity: def _build_external_task_sensor(ctx: OperatorContext) -> Activity: - """ExternalTaskSensor -> a notebook that waits for the external DAG's Databricks job to succeed. - - The external DAG becomes a sibling Databricks job (one bundle per DAG); this task polls that - job's most recent run via the Jobs API until it reaches a successful terminal state. The job is - referenced by the sanitized DAG name, matching TriggerDagRunOperator's RunJobActivity naming. - """ + """Routes ExternalTaskSensor to manual translation preserving logical-run semantics.""" external_dag = literal_str(ctx.kwargs.get("external_dag_id")) if external_dag is None: return _placeholder( @@ -301,32 +294,13 @@ def _build_external_task_sensor(ctx: OperatorContext) -> Activity: f"{ctx.operator} external_dag_id is not a string literal; implement the cross-DAG wait " "(poll the upstream job's run state) manually.", ) - poke, timeout = _poke_settings(ctx.kwargs) - job_name = _sanitize_job_name(external_dag) external_task = literal_str(ctx.kwargs.get("external_task_id")) - scope = f"task '{external_task}' in " if external_task else "" - header = _notebook_header(ctx.task_id, ctx.operator) + ( - "import time\n\n" - "from databricks.sdk import WorkspaceClient\n\n" - f"EXTERNAL_JOB_NAME = {job_name!r}\n" - "w = WorkspaceClient()\n\n" - "def _external_job_succeeded():\n" - " jobs = list(w.jobs.list(name=EXTERNAL_JOB_NAME))\n" - " if not jobs:\n" - " raise RuntimeError(f\"No Databricks job named {EXTERNAL_JOB_NAME!r}; deploy the \"\n" - " \"migrated upstream DAG's bundle first.\")\n" - " runs = list(w.jobs.list_runs(job_id=jobs[0].job_id, limit=1, completed_only=True))\n" - " if not runs:\n" - " return False\n" - " state = runs[0].state\n" - ' return state is not None and str(state.result_state) == "RunResultState.SUCCESS"\n\n' - ) - loop = _poll_body(ctx.operator, "_external_job_succeeded()", f"{scope}DAG '{external_dag}'", poke, timeout) - return NotebookActivity( - name=ctx.task_id, - task_key=ctx.task_key, - notebook_path=f"notebooks/{ctx.task_key}.py", - generated_source=header + loop, + target = f" task '{external_task}'" if external_task else "" + return _placeholder( + ctx, + f"ExternalTaskSensor waits for the matching logical run of DAG '{external_dag}'{target}. Databricks has " + "no cross-job task dependency primitive; translate this to upstream run_job_task orchestration, a table " + "update trigger, or a logical-time-aware polling implementation.", ) @@ -339,11 +313,17 @@ def _build_http_sensor(ctx: OperatorContext) -> Activity: f"{ctx.operator} endpoint is not a string literal; implement the HTTP poll manually " "(the http_conn_id base URL also needs wiring).", ) + if not endpoint.startswith(("http://", "https://")): + return _placeholder( + ctx, + f"{ctx.operator} endpoint '{endpoint}' depends on http_conn_id for its base URL; map the Airflow " + "connection to a complete URL before generating a polling task.", + ) poke, timeout = _poke_settings(ctx.kwargs) header = _notebook_header(ctx.task_id, ctx.operator) + ( "import time\n\n" "import requests\n\n" - f"ENDPOINT = {endpoint!r} # TODO: prefix with the http_conn_id base URL\n\n" + f"ENDPOINT = {endpoint!r}\n\n" "def _endpoint_ready():\n" " try:\n" " return requests.get(ENDPOINT, timeout=30).ok\n" @@ -367,7 +347,7 @@ def _build_python_sensor(ctx: OperatorContext) -> Activity: ctx, f"{ctx.operator} python_callable could not be resolved; implement the poll manually.", ) - reason = callable_notebook.task_context_reason(func) + reason = callable_notebook.airflow_runtime_reason(func, ctx.source) if reason is not None: return _placeholder( ctx, @@ -494,7 +474,7 @@ def _build_python(ctx: OperatorContext) -> Activity: # A callable that reads Airflow task context (**context / ti) or XCom can't run as a plain # notebook -- route it to the agentic-gap round instead of emitting code that fails at runtime. if func is not None: - reason = callable_notebook.task_context_reason(func) + reason = callable_notebook.airflow_runtime_reason(func, ctx.source) if reason is not None: return _placeholder( ctx, @@ -502,8 +482,16 @@ def _build_python(ctx: OperatorContext) -> Activity: "translate manually -- pass upstream data via job parameters or map XCom to " "dbutils.jobs.taskValues (set in the producer, get in the consumer).", ) - op_kwargs = literal_value(ctx.kwargs.get("op_kwargs")) - op_args = literal_value(ctx.kwargs.get("op_args")) + op_kwargs_node = ctx.kwargs.get("op_kwargs") + op_args_node = ctx.kwargs.get("op_args") + op_kwargs = literal_value(op_kwargs_node) + op_args = literal_value(op_args_node) + if op_kwargs_node is not None and not isinstance(op_kwargs, dict): + return _placeholder(ctx, "PythonOperator op_kwargs is not a static dictionary; bind its arguments manually.") + if op_args_node is not None and not isinstance(op_args, (list, tuple)): + return _placeholder(ctx, "PythonOperator op_args is not a static sequence; bind its arguments manually.") + if isinstance(op_args, tuple): + op_args = list(op_args) has_kwargs = isinstance(op_kwargs, dict) has_args = isinstance(op_args, list) generated = ( @@ -607,10 +595,13 @@ def _build_run_now(ctx: OperatorContext) -> Activity: def _build_trigger_dag_run(ctx: OperatorContext) -> Activity: target = literal_str(ctx.kwargs.get("trigger_dag_id")) or ctx.task_key conf = literal_value(ctx.kwargs.get("conf")) + # job_name becomes ${resources.jobs..id}; it must match the target DAG's job resource + # key, which write_bundle derives with normalize_task_key(dag_id). Using the same sanitizer keeps + # a cross-DAG TriggerDagRunOperator ref resolvable for hyphenated / mixed-case dag_ids. return RunJobActivity( name=ctx.task_id, task_key=ctx.task_key, - job_name=_sanitize_job_name(target), + job_name=normalize_task_key(target), job_parameters={k: str(v) for k, v in conf.items()} if isinstance(conf, dict) else None, ) @@ -722,11 +713,9 @@ def build_placeholder(ctx: OperatorContext) -> Activity: ) -def _sanitize_job_name(name: str) -> str: - import re - - key = re.sub(r"[^a-zA-Z0-9_-]", "_", name) - return re.sub(r"_+", "_", key).strip("_") or "job" +def build_placeholder_with_comment(ctx: OperatorContext, comment: str) -> Activity: + """Builds a placeholder carrying a caller-supplied migration explanation.""" + return _placeholder(ctx, comment) # -------------------------------------------------------------------------------------- diff --git a/src/flowx/sources/airflow/templating.py b/src/flowx/sources/airflow/templating.py index 2cc0e84..b4fb1d1 100644 --- a/src/flowx/sources/airflow/templating.py +++ b/src/flowx/sources/airflow/templating.py @@ -75,6 +75,10 @@ def _sub(match: re.Match[str]) -> str: "logical_date": ("logical_date", "{{job.start_time.iso_datetime}}"), "run_id": ("run_id", "{{job.run_id}}"), } +_SQL_IDENTIFIER_CONTEXT = re.compile( + r"(?:\bFROM|\bJOIN|\bINTO|\bUPDATE|\bTABLE|\bVIEW|\bSCHEMA|\bCATALOG)\s*$", + re.IGNORECASE, +) def convert_sql_template(sql: str) -> tuple[str, dict[str, str]]: @@ -89,18 +93,22 @@ def convert_sql_template(sql: str) -> tuple[str, dict[str, str]]: """ parameters: dict[str, str] = {} + def _marker(name: str, match: re.Match[str]) -> str: + marker = f":{name}" + return f"IDENTIFIER({marker})" if _SQL_IDENTIFIER_CONTEXT.search(sql[: match.start()]) else marker + def _sub(match: re.Match[str]) -> str: expr = match.group(1).strip() if expr in _SQL_MACRO_PARAM: name, ref = _SQL_MACRO_PARAM[expr] parameters[name] = ref - return f":{name}" + return _marker(name, match) for pattern in _PARAM_PATTERNS: m = pattern.match(expr) if m: name = m.group(1) parameters[name] = "{{job.parameters." + name + "}}" - return f":{name}" + return _marker(name, match) return match.group(0) return _JINJA.sub(_sub, sql), parameters @@ -132,6 +140,21 @@ def convert_params(value: Any) -> tuple[Any, set[str]]: return value, params +def unresolved_jinja_expressions(value: Any) -> set[str]: + """Returns Jinja expressions that remain after deterministic conversion.""" + if isinstance(value, str): + return { + expression + for match in _JINJA.findall(value) + if not (expression := match.strip()).startswith(("job.", "tasks.", "input.")) + } + if isinstance(value, list): + return set().union(*(unresolved_jinja_expressions(item) for item in value)) if value else set() + if isinstance(value, dict): + return set().union(*(unresolved_jinja_expressions(item) for item in value.values())) if value else set() + return set() + + # -------------------------------------------------------------------------------------- # default_args (retries / timeouts / email) # -------------------------------------------------------------------------------------- @@ -207,8 +230,8 @@ def email_on_failure(dag_default_args: dict[str, ast.expr], task_kwargs: dict[st # Map Airflow trigger_rule -> the DAB job ``run_if`` constant carried as a dependency outcome. The # preparer's reducer passes these through unchanged. Rules with no exact DAB equivalent fall back to -# the closest safe constant: none_failed_min_one_success -> AT_LEAST_ONE_SUCCESS (both require >=1 -# success and no upstream failure); none_failed_or_skipped -> NONE_FAILED (skips are non-failures). +# the closest safe constant: none_failed_min_one_success -> NONE_FAILED so an upstream failure +# never permits downstream execution; none_failed_or_skipped -> NONE_FAILED (skips are non-failures). # Airflow's default all_success maps to None (no run_if key -> Databricks default ALL_SUCCESS). _TRIGGER_RULE_TO_RUN_IF: dict[str, str | None] = { "all_success": None, @@ -217,7 +240,7 @@ def email_on_failure(dag_default_args: dict[str, ast.expr], task_kwargs: dict[st "one_failed": "AT_LEAST_ONE_FAILED", "one_success": "AT_LEAST_ONE_SUCCESS", "none_failed": "NONE_FAILED", - "none_failed_min_one_success": "AT_LEAST_ONE_SUCCESS", + "none_failed_min_one_success": "NONE_FAILED", "none_failed_or_skipped": "NONE_FAILED", "always": "ALL_DONE", } @@ -226,7 +249,12 @@ def email_on_failure(dag_default_args: dict[str, ast.expr], task_kwargs: dict[st def trigger_rule_outcome(task_kwargs: dict[str, ast.expr]) -> str | None: """Maps a task's ``trigger_rule`` kwarg to a DAB ``run_if`` constant, or None (ALL_SUCCESS).""" node = task_kwargs.get("trigger_rule") - rule = node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None + if isinstance(node, ast.Constant) and isinstance(node.value, str): + rule = node.value + elif isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id == "TriggerRule": + rule = node.attr.lower() + else: + rule = None if rule is None: return None return _TRIGGER_RULE_TO_RUN_IF.get(rule) @@ -245,6 +273,11 @@ def trigger_rule_outcome(task_kwargs: dict[str, ast.expr]) -> str | None: ) +def airflow_connection_names(source: str) -> set[str]: + """Returns literal Airflow connection identifiers referenced in Python source.""" + return set(_CONNECTION_GET.findall(source)) + + def rewrite_airflow_calls(source: str) -> tuple[str, set[str], list[str]]: """Rewrites Airflow Variable/Connection calls in notebook-body *source*. diff --git a/tests/integration/test_airflow_golden_bundle.py b/tests/integration/test_airflow_golden_bundle.py index 063d881..95c4d68 100644 --- a/tests/integration/test_airflow_golden_bundle.py +++ b/tests/integration/test_airflow_golden_bundle.py @@ -104,7 +104,7 @@ def test_dependencies_from_both_shift_and_set_upstream(job_def: dict): def test_job_parameters_carry_defaults(job_def: dict): params = {p["name"]: p["default"] for p in job_def["parameters"]} assert params["target_env"] == "prod" # from Param("prod") - assert params["threshold"] == 100 # bare literal default + assert params["threshold"] == "100" # Jobs parameter defaults are strings def test_templated_param_becomes_dab_ref(job_def: dict): diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py index a354ab4..08e58bc 100644 --- a/tests/unit/test_airflow_operators.py +++ b/tests/unit/test_airflow_operators.py @@ -74,6 +74,34 @@ def test_python_operator_notebook_is_valid_python(): assert "taskValues.set" in nb # return value captured +def test_python_callable_dependencies_are_emitted_in_definition_order(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "CONST = 7\n" + "def helper(value=CONST):\n return value\n" + "def work():\n return helper()\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='work', python_callable=work)\n" + ) + + source = _by_key(p)["work"].generated_source + assert source.index("CONST = 7") < source.index("def helper") + + +def test_python_callable_carries_annotated_module_constant(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "LIMIT: int = 7\n" + "def work():\n return LIMIT\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='work', python_callable=work)\n" + ) + + assert "LIMIT: int = 7" in _by_key(p)["work"].generated_source + + def test_python_operator_with_context_kwarg_becomes_placeholder(): # A callable taking **context can't run without the Airflow runtime; route to a gap # rather than emitting a notebook that fails at runtime. @@ -90,6 +118,18 @@ def test_python_operator_with_context_kwarg_becomes_placeholder(): assert task.raw_definition is not None # carries source for the agentic round +def test_python_operator_with_named_airflow_context_becomes_placeholder(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def work(ds):\n print(ds)\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='work', python_callable=work)\n" + ) + + assert isinstance(_by_key(p)["work"], PlaceholderActivity) + + def test_python_operator_with_ti_param_becomes_placeholder(): p = _load( "from airflow import DAG\n" @@ -102,6 +142,21 @@ def test_python_operator_with_ti_param_becomes_placeholder(): assert isinstance(task, PlaceholderActivity) +def test_python_operator_with_nonliteral_arguments_becomes_placeholder(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "ARGS = {'value': 3}\n" + "def work(value):\n return value\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='work', python_callable=work, op_kwargs=ARGS)\n" + ) + + task = _by_key(p)["work"] + assert isinstance(task, PlaceholderActivity) + assert "op_kwargs" in task.comment + + def test_python_operator_with_xcom_pull_becomes_placeholder(): p = _load( "from airflow import DAG\n" @@ -180,6 +235,23 @@ def test_sql_operator_becomes_sql_task(): task = _by_key(p)["rep"] assert isinstance(task, SqlActivity) assert task.sql == "CREATE TABLE g AS SELECT 1" + + +def test_sql_identifier_template_uses_identifier_marker(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator\n" + "with DAG(dag_id='d') as dag:\n" + " t = SQLExecuteQueryOperator(task_id='rep', " + "sql='SELECT * FROM {{ params.table }} WHERE id = {{ params.id }}')\n" + ) + + task = _by_key(p)["rep"] + assert task.sql == "SELECT * FROM IDENTIFIER(:table) WHERE id = :id" + assert task.parameters == { + "table": "{{job.parameters.table}}", + "id": "{{job.parameters.id}}", + } assert task.warehouse_ref == "${var.warehouse_id}" @@ -261,7 +333,7 @@ def test_sql_condition_sensor_without_literal_sql_stays_placeholder(): assert p.schedule is None -def test_external_task_sensor_becomes_cross_dag_wait(): +def test_external_task_sensor_becomes_manual_cross_dag_placeholder(): p = _load( "from airflow import DAG\n" "from airflow.sensors.external_task import ExternalTaskSensor\n" @@ -274,15 +346,11 @@ def test_external_task_sensor_becomes_cross_dag_wait(): " wait >> go\n" ) task = _by_key(p)["wait_up"] - assert isinstance(task, NotebookActivity) - src = task.generated_source - assert "WorkspaceClient" in src - assert "EXTERNAL_JOB_NAME = 'upstream_dag'" in src # sanitized to the sibling job name - assert "POKE_INTERVAL = 45" in src - compile(src, "", "exec") + assert isinstance(task, PlaceholderActivity) + assert "logical run" in task.comment -def test_http_sensor_becomes_polling_task(): +def test_http_sensor_with_relative_endpoint_becomes_placeholder(): p = _load( "from airflow import DAG\n" "from airflow.providers.http.sensors.http import HttpSensor\n" @@ -290,11 +358,20 @@ def test_http_sensor_becomes_polling_task(): " s = HttpSensor(task_id='h', endpoint='api/ready', poke_interval=10, timeout=120)\n" ) task = _by_key(p)["h"] + assert isinstance(task, PlaceholderActivity) + assert "http_conn_id" in task.comment + + +def test_http_sensor_with_absolute_endpoint_becomes_polling_task(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.http.sensors.http import HttpSensor\n" + "with DAG(dag_id='d') as dag:\n" + " s = HttpSensor(task_id='h', endpoint='https://example.com/api/ready')\n" + ) + task = _by_key(p)["h"] assert isinstance(task, NotebookActivity) - src = task.generated_source - assert "requests.get" in src - assert "api/ready" in src - compile(src, "", "exec") + assert "requests.get" in task.generated_source def test_python_sensor_polls_callable_without_eager_call(): @@ -340,6 +417,23 @@ def test_datetime_sensor_becomes_wait_until_task(): compile(src, "", "exec") +def test_time_delta_sensor_is_retained_as_manual_placeholder(): + p = _load( + "from datetime import timedelta\n" + "from airflow import DAG\n" + "from airflow.sensors.time_delta import TimeDeltaSensor\n" + "from airflow.operators.bash import BashOperator\n" + "with DAG(dag_id='d', schedule='0 0 * * *') as dag:\n" + " wait = TimeDeltaSensor(task_id='wait', delta=timedelta(hours=2))\n" + " work = BashOperator(task_id='work', bash_command='echo work')\n" + " wait >> work\n" + ) + + tasks = _by_key(p) + assert isinstance(tasks["wait"], PlaceholderActivity) + assert tasks["work"].depends_on[0].task_key == "wait" + + def test_databricks_run_now_becomes_run_job(): p = _load( "from airflow import DAG\n" @@ -365,6 +459,21 @@ def test_trigger_dag_run_becomes_run_job_by_name(): assert task.job_parameters == {"k": "v"} +def test_trigger_dag_run_job_name_matches_target_job_resource_key(): + # job_name becomes ${resources.jobs..id}; it must equal normalize_task_key(dag_id) + # (how write_bundle keys the target job), or the cross-DAG ref dangles for hyphenated/mixed-case ids. + from flowx.utils import normalize_task_key + + p = _load( + "from airflow import DAG\n" + "from airflow.operators.trigger_dagrun import TriggerDagRunOperator\n" + "with DAG(dag_id='d') as dag:\n" + " t = TriggerDagRunOperator(task_id='f', trigger_dag_id='Upstream-DAG')\n" + ) + task = _by_key(p)["f"] + assert task.job_name == normalize_task_key("Upstream-DAG") == "upstream_dag" + + def test_databricks_submit_run_reads_notebook_from_json(): p = _load( "from airflow import DAG\n" @@ -463,7 +572,63 @@ def test_dbt_cli_operators_collapse_to_one_factory(): assert len(dbt_tasks) == 1 # the seed>>run>>test chain collapses into one factory job assert dbt_tasks[0].project_dir == "/opt/proj" # A manifest_path must be set or the static preparer would explode zero tasks (empty child job). - assert dbt_tasks[0].manifest_path == "/opt/proj/target/dev/manifest.json" + assert dbt_tasks[0].manifest_path == "/opt/proj/target/manifest.json" + + +def test_single_dbt_run_operator_limits_factory_to_models(): + p = _load( + "from airflow import DAG\n" + "from airflow_dbt.operators.dbt_operator import DbtRunOperator\n" + "with DAG(dag_id='d') as dag:\n" + " run = DbtRunOperator(task_id='run', dir='/opt/proj')\n" + ) + + task = _by_key(p)["run"] + assert isinstance(task, DbtFactoryActivity) + assert task.resource_types == ["model"] + + +def test_dbt_operator_preserves_command_options_and_standard_manifest_path(): + p = _load( + "from airflow import DAG\n" + "from airflow_dbt.operators.dbt_operator import DbtRunOperator\n" + "with DAG(dag_id='d') as dag:\n" + " run = DbtRunOperator(task_id='run', dir='/opt/proj', select=['tag:daily'], " + "exclude=['tag:slow'], vars={'region': 'west'}, full_refresh=True)\n" + ) + + task = _by_key(p)["run"] + assert isinstance(task, DbtFactoryActivity) + assert task.manifest_path == "/opt/proj/target/manifest.json" + assert task.selectors == ["tag:daily"] + assert task.exclude_selectors == ["tag:slow"] + assert task.variables == {"region": "west"} + assert task.full_refresh is True + + +def test_dbt_deps_operator_runs_only_dependency_installation(tmp_path): + from flowx.preparer.workflow_preparer import prepare_workflow + + project = tmp_path / "project" + profiles = tmp_path / "profiles" + project.mkdir() + profiles.mkdir() + (project / "dbt_project.yml").write_text("name: demo\nprofile: demo\n") + (profiles / "profiles.yml").write_text("demo:\n target: dev\n outputs: {}\n") + p = _load( + "from airflow import DAG\n" + "from airflow_dbt.operators.dbt_operator import DbtDepsOperator\n" + "with DAG(dag_id='d') as dag:\n" + f" deps = DbtDepsOperator(task_id='deps', dir={str(project)!r}, profiles_dir={str(profiles)!r})\n" + ) + + task = _by_key(p)["deps"] + assert isinstance(task, DbtFactoryActivity) + assert task.resource_types == ["dependency"] + + prepared = prepare_workflow(p) + assert prepared.tasks[0]["run_job_task"] + assert prepared.inner_workflows[0].tasks[0]["notebook_task"]["base_parameters"]["dbt_command"] == "deps" def test_dbt_chain_downstream_dep_rewired_to_factory_key(): @@ -509,21 +674,28 @@ def test_dbt_factory_explodes_manifest_into_tasks(tmp_path): }, "unit_tests": {}, } - proj = tmp_path / "proj" / "target" / "dev" + project_dir = tmp_path / "proj" + (project_dir / "dbt_project.yml").parent.mkdir(parents=True) + (project_dir / "dbt_project.yml").write_text("name: p\nprofile: p\n", encoding="utf-8") + profiles_dir = tmp_path / "profiles" + profiles_dir.mkdir() + (profiles_dir / "profiles.yml").write_text("p:\n target: dev\n outputs: {}\n", encoding="utf-8") + proj = project_dir / "target" proj.mkdir(parents=True) (proj / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") dag = ( "from airflow import DAG\n" "from airflow_dbt.operators.dbt_operator import DbtRunOperator\n" "with DAG(dag_id='d') as dag:\n" - f" r = DbtRunOperator(task_id='run', dir={str(tmp_path / 'proj')!r})\n" + f" r = DbtRunOperator(task_id='run', dir={str(tmp_path / 'proj')!r}, " + f"profiles_dir={str(profiles_dir)!r})\n" ) dag_file = tmp_path / "dag.py" dag_file.write_text(dag, encoding="utf-8") p = load_airflow_dag(dag_file) wf = prepare_workflow(p) inner_task_keys = {t["task_key"] for inner in wf.inner_workflows for t in inner.tasks} - assert inner_task_keys == {"seed_codes", "model_stg"} # non-empty, both manifest nodes exploded + assert inner_task_keys == {"model_stg"} # -------------------------------------------------------------------------------------- @@ -546,6 +718,21 @@ def test_file_sensor_lifts_to_file_arrival_trigger(): assert p.schedule == {"kind": "file_arrival", "url": "s3://landing/in/", "pause_status": "UNPAUSED"} +def test_s3_sensor_trigger_combines_bucket_and_relative_key(): + p = _load( + "from airflow import DAG\n" + "from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor\n" + "with DAG(dag_id='d') as dag:\n" + " wait = S3KeySensor(task_id='wait', bucket_name='landing', bucket_key='incoming/')\n" + ) + + assert p.schedule == { + "kind": "file_arrival", + "url": "s3://landing/incoming/", + "pause_status": "UNPAUSED", + } + + def test_cron_and_sensor_keeps_both_schedule_and_polling_task(): # cron AND-THEN wait: the cron becomes the schedule and the sensor is retained as a polling # task (never silently dropped), because schedule and file_arrival triggers are mutually exclusive. @@ -662,6 +849,48 @@ def test_convert_emits_gaps_json_for_unmapped_operators(): assert gaps[0]["raw_definition"]["source"] +def test_convert_merges_agentic_results_without_source_dir(tmp_path): + import json + + from flowx.sources.airflow.convert import main + + report = tmp_path / "translation_report.json" + report.write_text( + json.dumps( + { + "name": "example", + "tasks": [ + { + "type": "PlaceholderActivity", + "name": "pod", + "task_key": "pod", + "original_type": "KubernetesPodOperator", + } + ], + } + ) + ) + results = tmp_path / "results" + results.mkdir() + (results / "pod.json").write_text( + json.dumps( + { + "activity_name": "pod", + "task": { + "type": "NotebookActivity", + "name": "pod", + "task_key": "pod", + "notebook_path": "notebooks/pod.py", + }, + } + ) + ) + + assert main(["--merge-agentic", "--report", str(report), "--agentic-results", str(results)]) == 0 + merged = json.loads(report.read_text()) + assert merged["tasks"][0]["type"] == "NotebookActivity" + + # -------------------------------------------------------------------------------------- # Cross-cutting: Jinja templating, default_args, trigger_rule # -------------------------------------------------------------------------------------- @@ -687,6 +916,20 @@ def test_jinja_macros_convert_to_dab_refs_and_collect_params(): assert p.parameters == [{"name": "env", "default": ""}] +def test_unsupported_airflow_macro_becomes_placeholder(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def w(value=None):\n return value\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='t', python_callable=w, op_kwargs={'value': '{{ ds_nodash }}'})\n" + ) + + task = _by_key(p)["t"] + assert isinstance(task, PlaceholderActivity) + assert "ds_nodash" in task.comment + + def test_default_args_apply_retries_timeout_retry_delay(): p = _load( "from datetime import timedelta\n" @@ -725,11 +968,14 @@ def test_trigger_rule_maps_to_run_if_constant(): " only_fail = PythonOperator(task_id='only_fail', python_callable=w, trigger_rule='all_failed')\n" " any_ok = PythonOperator(task_id='any_ok', python_callable=w, trigger_rule='one_success')\n" " no_fail = PythonOperator(task_id='no_fail', python_callable=w, trigger_rule='none_failed')\n" + " no_fail_with_success = PythonOperator(task_id='no_fail_with_success', python_callable=w,\n" + " trigger_rule='none_failed_min_one_success')\n" " a >> cleanup\n" " a >> fail_only\n" " a >> only_fail\n" " a >> any_ok\n" " a >> no_fail\n" + " a >> no_fail_with_success\n" ) tasks = _by_key(p) # trigger_rule maps straight to the DAB run_if constant, carried as the dependency outcome. @@ -738,9 +984,25 @@ def test_trigger_rule_maps_to_run_if_constant(): assert tasks["only_fail"].depends_on[0].outcome == "ALL_FAILED" assert tasks["any_ok"].depends_on[0].outcome == "AT_LEAST_ONE_SUCCESS" assert tasks["no_fail"].depends_on[0].outcome == "NONE_FAILED" + assert tasks["no_fail_with_success"].depends_on[0].outcome == "NONE_FAILED" assert tasks["a"].depends_on is None # default all_success -> no outcome +def test_trigger_rule_enum_member_maps_to_run_if_constant(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "from airflow.utils.trigger_rule import TriggerRule\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " a = PythonOperator(task_id='a', python_callable=w)\n" + " cleanup = PythonOperator(task_id='cleanup', python_callable=w, trigger_rule=TriggerRule.ALL_DONE)\n" + " a >> cleanup\n" + ) + + assert _by_key(p)["cleanup"].depends_on[0].outcome == "ALL_DONE" + + # -------------------------------------------------------------------------------------- # Dynamic mapping (.expand), TaskGroup prefixing, timezone/timedelta schedules # -------------------------------------------------------------------------------------- @@ -771,6 +1033,23 @@ def test_expand_direct_call_form(): assert isinstance(_by_key(p)["run"], ForEachActivity) +def test_mixed_shift_directions_preserve_each_operator_direction(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.bash import BashOperator\n" + "with DAG(dag_id='d') as dag:\n" + " a = BashOperator(task_id='a', bash_command='a')\n" + " b = BashOperator(task_id='b', bash_command='b')\n" + " c = BashOperator(task_id='c', bash_command='c')\n" + " a >> b << c\n" + ) + + tasks = _by_key(p) + assert tasks["a"].depends_on is None + assert {dependency.task_key for dependency in tasks["b"].depends_on} == {"a", "c"} + assert tasks["c"].depends_on is None + + def test_task_group_prefixes_member_keys(): p = _load( "from airflow import DAG\n" @@ -823,6 +1102,26 @@ def test_timedelta_schedule_becomes_periodic(): assert p.schedule == {"kind": "periodic", "interval": 2, "unit": "DAYS", "pause_status": "UNPAUSED"} +def test_subhour_timedelta_schedule_becomes_quartz_cron(): + p = _load( + "from datetime import timedelta\n" + "from airflow import DAG\n" + "with DAG(dag_id='d', schedule=timedelta(minutes=30)) as dag:\n" + " pass\n" + ) + assert p.schedule == { + "kind": "schedule", + "quartz_cron_expression": "0 0/30 * * * ?", + "timezone_id": "UTC", + "pause_status": "UNPAUSED", + } + + +def test_continuous_schedule_becomes_continuous_job_mode(): + p = _load("from airflow import DAG\nwith DAG(dag_id='d', schedule='@continuous') as dag:\n pass\n") + assert p.schedule == {"kind": "continuous", "pause_status": "UNPAUSED"} + + def test_dag_timezone_extracted_into_cron_schedule(): p = _load( "from datetime import datetime\n" @@ -857,7 +1156,7 @@ def test_variable_get_rewritten_to_widget_and_declared_as_param(): assert {"name": "target_env", "default": ""} in (p.parameters or []) -def test_connection_get_rewritten_to_secrets(): +def test_connection_get_becomes_placeholder_for_connection_object_mapping(): p = _load( "from airflow import DAG\n" "from airflow.operators.python import PythonOperator\n" @@ -869,9 +1168,27 @@ def test_connection_get_rewritten_to_secrets(): " t = PythonOperator(task_id='ingest', python_callable=ingest)\n" ) task = _by_key(p)["ingest"] - assert "dbutils.secrets.get(" in task.generated_source - assert "snowflake_default_scope" in task.generated_source - assert "BaseHook.get_connection" not in task.generated_source + assert isinstance(task, PlaceholderActivity) + assert "snowflake_default" in task.comment + + +def test_connection_get_in_carried_helper_becomes_placeholder(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "from airflow.hooks.base import BaseHook\n" + "def connection_host():\n" + " conn = BaseHook.get_connection('warehouse')\n" + " return conn.host\n" + "def ingest():\n" + " print(connection_host())\n" + "with DAG(dag_id='d') as dag:\n" + " t = PythonOperator(task_id='ingest', python_callable=ingest)\n" + ) + + task = _by_key(p)["ingest"] + assert isinstance(task, PlaceholderActivity) + assert "warehouse" in task.comment def test_airflow_host_detection_from_dag_source(): @@ -946,6 +1263,108 @@ def test_taskflow_data_flow_reads_upstream_taskvalue(): assert "dbutils.jobs.taskValues.set(key='return_value', value=result)" in src # publishes +def test_taskflow_literal_arguments_are_preserved(): + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def add(x, y):\n return x + y\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " add(1, y=2)\n" + "pipeline()\n" + ) + + source = p.tasks[0].generated_source + assert "result = add(1, y=2)" in source + + +def test_taskflow_nonliteral_argument_becomes_placeholder(): + p = _load( + "from airflow.decorators import dag, task\n" + "VALUE = 3\n" + "@task\n" + "def work(value):\n return value\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " work(VALUE)\n" + "pipeline()\n" + ) + + assert isinstance(p.tasks[0], PlaceholderActivity) + assert "VALUE" in p.tasks[0].comment + + +def test_taskflow_override_call_is_preserved(): + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def work(value):\n return value\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " work.override(task_id='renamed')(2)\n" + "pipeline()\n" + ) + + assert len(p.tasks) == 1 + assert p.tasks[0].task_key == "renamed" + assert "result = work(2)" in p.tasks[0].generated_source + + +def test_nested_taskflow_callable_carries_module_imports(): + p = _load( + "from datetime import datetime\n" + "from airflow.decorators import dag, task\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " @task\n" + " def now():\n" + " return datetime.now().isoformat()\n" + " now()\n" + "pipeline()\n" + ) + + source = p.tasks[0].generated_source + assert "from datetime import datetime" in source + compile(source, "", "exec") + + +def test_nested_taskflow_callable_carries_literal_closure_bindings(): + p = _load( + "from airflow.decorators import dag, task\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " factor = 3\n" + " @task\n" + " def scale(value):\n" + " return value * factor\n" + " scale(2)\n" + "pipeline()\n" + ) + + source = p.tasks[0].generated_source + assert "factor = 3" in source + assert "result = scale(2)" in source + + +def test_nested_taskflow_callable_with_dynamic_closure_becomes_placeholder(): + p = _load( + "from airflow.decorators import dag, task\n" + "def get_factor():\n" + " return 3\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " factor = get_factor()\n" + " @task\n" + " def scale(value):\n" + " return value * factor\n" + " scale(2)\n" + "pipeline()\n" + ) + + assert isinstance(p.tasks[0], PlaceholderActivity) + assert "factor" in p.tasks[0].comment + + def test_taskflow_branch_decorator_becomes_placeholder(): p = _load( "from airflow.decorators import dag, task\n" @@ -994,3 +1413,25 @@ def test_taskflow_mixed_with_classic_operator(): assert "prep" in tasks finalize = next(t for t in p.tasks if t.task_key.startswith("finalize")) assert finalize.depends_on[0].task_key == "prep" + + +def test_multiple_dags_in_one_file_are_loaded_as_separate_pipelines(tmp_path): + from flowx.sources.airflow.loader import load_pipelines + + source = tmp_path / "multi.py" + source.write_text( + "from airflow import DAG\n" + "from airflow.operators.bash import BashOperator\n" + "with DAG(dag_id='one') as dag_one:\n" + " a = BashOperator(task_id='a', bash_command='echo a')\n" + "with DAG(dag_id='two') as dag_two:\n" + " b = BashOperator(task_id='b', bash_command='echo b')\n", + encoding="utf-8", + ) + + pipelines = load_pipelines(source) + + assert [(pipeline.name, [task.task_key for task in pipeline.tasks]) for pipeline in pipelines] == [ + ("one", ["a"]), + ("two", ["b"]), + ] From bda15affd916ec85f4490f80df89168549c52215 Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 20 Jul 2026 12:55:12 -0700 Subject: [PATCH 31/77] Harden dbt and multi-DAG bundle packaging --- src/flowx/bundler/dab_writer.py | 205 ++++++++++-- src/flowx/bundler/prereqs_writer.py | 15 +- src/flowx/dbt/manifest.py | 24 +- src/flowx/ir_serde.py | 8 + src/flowx/mcp/server.py | 11 +- src/flowx/models/ir.py | 8 + .../activity_preparers/dbt_factory.py | 291 ++++++++++++++++-- src/flowx/preparer/workflow_preparer.py | 2 + src/flowx/sources/airflow/convert.py | 29 +- src/flowx/validate/bundle_invariants.py | 56 +++- tests/unit/test_bundle_invariants.py | 48 ++- tests/unit/test_bundler.py | 96 ++++++ tests/unit/test_dbt_factory_preparer.py | 159 +++++++++- tests/unit/test_dbt_manifest.py | 28 ++ tests/unit/test_mcp_source_routing.py | 12 + tests/unit/test_package_invariants.py | 125 ++++++++ 16 files changed, 1044 insertions(+), 73 deletions(-) diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 3fbe618..7c098ec 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import copy import json import re import sys @@ -78,6 +79,7 @@ class _BundleYamlDumper(yaml.SafeDumper): _neutralized_conditions: list[dict[str, str]] = [] _WIDGET_REFERENCE = re.compile(r"""dbutils\.widgets\.get\(\s*["']([^"']+)["']\s*\)""") +_JOB_RESOURCE_ID_REFERENCE = re.compile(r"\$\{resources\.jobs\.([^.}]+)\.id\}") def write_bundle( @@ -105,12 +107,16 @@ def write_bundle( _cross_bundle_variables.clear() _neutralized_conditions.clear() + workflow = copy.deepcopy(workflow) + output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) created_files: list[Path] = [] resource_key = normalize_task_key(workflow.name) effective_name = bundle_name or resource_key + known_bundle_jobs = _known_bundle_job_keys(workflow, resource_key) + _rewrite_cross_bundle_job_references(workflow, known_bundle_jobs) # Bind clusters across the parent and inner workflows up front to decide whether databricks.yml needs # cluster tunables at all. Binding is idempotent, so _build_job_resource re-checking these is harmless. @@ -217,12 +223,16 @@ def write_bundle( src_dir = output_dir / "src" def _write_generated(notebooks: list[DabNotebook]) -> None: - hooks = [nb for nb in notebooks if nb.relative_path.startswith("resources/")] - rest = [nb for nb in notebooks if not nb.relative_path.startswith("resources/")] + root_artifacts = [ + notebook + for notebook in notebooks + if notebook.relative_path.startswith("resources/") or notebook.relative_path == "pyproject.toml" + ] + rest = [notebook for notebook in notebooks if notebook not in root_artifacts] if rest: created_files.extend(write_notebooks(rest, src_dir)) - if hooks: - created_files.extend(write_notebooks(hooks, output_dir)) + if root_artifacts: + created_files.extend(write_notebooks(root_artifacts, output_dir)) if workflow.notebooks: _write_generated(workflow.notebooks) @@ -262,7 +272,7 @@ def _write_generated(notebooks: list[DabNotebook]) -> None: parameter_approximations = list(workflow.parameter_approximations) for inner in workflow.inner_workflows: parameter_approximations.extend(inner.parameter_approximations) - known_bundle_jobs = {resource_key} | {normalize_task_key(inner.name) for inner in workflow.inner_workflows} + known_bundle_jobs = _known_bundle_job_keys(workflow, resource_key) # manual_parameters was collected above (before YAML emission) so broken values are stripped on disk too. # VAREX3-003: manual_variable_rollup SetupTasks from workflow_preparer surface in SETUP.md so the user # knows where to add a roll-up notebook. @@ -447,32 +457,44 @@ def main(argv: list[str] | None = None) -> int: print("No translated pipelines found in the report.", file=sys.stderr) return 1 + shared_airflow_bundle = len(workflows) > 1 and all(workflow.source == "airflow" for workflow in workflows) all_created: list[Path] = [] - for index, workflow in enumerate(workflows): - if len(workflows) > 1: - workflow_dir = args.output_dir / normalize_task_key(workflow.name) - else: - workflow_dir = args.output_dir - - effective_bundle_name = args.bundle_name if len(workflows) == 1 else None - created = write_bundle( - workflow=workflow, - output_dir=workflow_dir, - catalog=args.catalog, - schema=args.schema, - bundle_name=effective_bundle_name, + if shared_airflow_bundle: + combined = _combine_airflow_workflows(workflows) + all_created.extend( + write_bundle( + workflow=combined, + output_dir=args.output_dir, + catalog=args.catalog, + schema=args.schema, + bundle_name=args.bundle_name or normalize_task_key(args.output_dir.name), + ) ) - all_created.extend(created) - print(f" [{index + 1}/{len(workflows)}] {workflow.name}: {len(created)} files") + print(f" [1/1] {len(workflows)} Airflow DAG jobs: {len(all_created)} files") + else: + for index, workflow in enumerate(workflows): + workflow_dir = ( + args.output_dir / normalize_task_key(workflow.name) if len(workflows) > 1 else args.output_dir + ) + effective_bundle_name = args.bundle_name if len(workflows) == 1 else None + created = write_bundle( + workflow=workflow, + output_dir=workflow_dir, + catalog=args.catalog, + schema=args.schema, + bundle_name=effective_bundle_name, + ) + all_created.extend(created) + print(f" [{index + 1}/{len(workflows)}] {workflow.name}: {len(created)} files") # Tier-0 structural check over the emitted bundle(s): duplicate task keys / job params, # dangling depends_on, undeclared {{job.parameters.X}}, leaked YAML anchors. Source-agnostic. from flowx.validate.bundle_invariants import check_bundle_dir, format_result bundle_dirs = ( - [args.output_dir / normalize_task_key(workflow.name) for workflow in workflows] - if len(workflows) > 1 - else [args.output_dir] + [args.output_dir] + if shared_airflow_bundle or len(workflows) == 1 + else [args.output_dir / normalize_task_key(workflow.name) for workflow in workflows] ) invariant_violations = 0 for bundle_dir in bundle_dirs: @@ -504,6 +526,127 @@ def main(argv: list[str] | None = None) -> int: return 1 if invariant_violations else 0 +def _combine_airflow_workflows(workflows: list[PreparedWorkflow]) -> PreparedWorkflow: + """Combines Airflow DAG workflows into one bundle containing one job per DAG.""" + namespaced = [_namespace_workflow_assets(workflow) for workflow in workflows] + primary = namespaced[0] + inner_workflows = list(primary.inner_workflows) + for workflow in namespaced[1:]: + nested = list(workflow.inner_workflows) + workflow.inner_workflows = [] + inner_workflows.append(workflow) + inner_workflows.extend(nested) + primary.inner_workflows = inner_workflows + return primary + + +def _rewrite_cross_bundle_job_references(workflow: PreparedWorkflow, known_bundle_jobs: set[str]) -> None: + """Uses bundle variables for ``run_job_task`` targets defined outside this bundle.""" + workflows = [workflow, *workflow.inner_workflows] + for current in workflows: + for task in _iter_tasks_recursively(current.tasks): + run_job = task.get("run_job_task") + if not isinstance(run_job, dict): + continue + job_id = run_job.get("job_id") + match = _JOB_RESOURCE_ID_REFERENCE.fullmatch(job_id) if isinstance(job_id, str) else None + if match is None: + continue + target_job = match.group(1) + if target_job in known_bundle_jobs: + continue + variable_name = f"{normalize_task_key(target_job)}_job_id" + suffix = 2 + while variable_name in _cross_bundle_variables and _cross_bundle_variables[variable_name] != target_job: + variable_name = f"{normalize_task_key(target_job)}_job_id_{suffix}" + suffix += 1 + _cross_bundle_variables[variable_name] = target_job + run_job["job_id"] = f"${{var.{variable_name}}}" + + +def _known_bundle_job_keys(workflow: PreparedWorkflow, resource_key: str) -> set[str]: + """Returns static and Python-generated job resource keys owned by this bundle.""" + keys = {resource_key} | {normalize_task_key(inner.name) for inner in workflow.inner_workflows} + for current in [workflow, *workflow.inner_workflows]: + keys.update( + str(setup_task.config["job_key"]) + for setup_task in current.setup_tasks + if setup_task.type == "pydabs_dbt_factory" and setup_task.config.get("job_key") + ) + return keys + + +def _namespace_workflow_assets(workflow: PreparedWorkflow) -> PreparedWorkflow: + """Namespaces generated source files by DAG while preserving workspace paths.""" + cloned = copy.deepcopy(workflow) + prefix = normalize_task_key(cloned.name) + replacements: dict[str, str] = {} + pydabs_hooks: dict[str, tuple[str, str, str]] = {} + + nested_workflows = [cloned, *cloned.inner_workflows] + for nested in nested_workflows: + for setup_task in nested.setup_tasks: + if setup_task.type != "pydabs_dbt_factory": + continue + module = str(setup_task.config["hook_module"]) + module_name = module.removeprefix("resources.") + namespaced_module_name = normalize_task_key(f"{prefix}__{module_name}") + namespaced_module = f"resources.{namespaced_module_name}" + original_job_key = str(setup_task.config["job_key"]) + namespaced_job_key = normalize_task_key(f"{prefix}__{original_job_key}") + original_hook_path = f"resources/{module_name}.py" + namespaced_hook_path = f"resources/{namespaced_module_name}.py" + pydabs_hooks[original_hook_path] = (namespaced_hook_path, original_job_key, namespaced_job_key) + setup_task.config["hook_module"] = namespaced_module + setup_task.config["job_key"] = namespaced_job_key + setup_task.config["manifest_path"] = f"src/{prefix}/dbt_project/target/manifest.json" + replacements[f"${{resources.jobs.{original_job_key}.id}}"] = f"${{resources.jobs.{namespaced_job_key}.id}}" + + for notebook in nested.notebooks: + original_path = notebook.relative_path + if original_path in pydabs_hooks: + namespaced_path, original_job_key, namespaced_job_key = pydabs_hooks[original_path] + notebook.relative_path = namespaced_path + notebook.content = notebook.content.replace(original_job_key, namespaced_job_key) + notebook.content = notebook.content.replace("src/notebooks/", f"src/{prefix}/notebooks/") + notebook.content = notebook.content.replace("src/dbt_project", f"src/{prefix}/dbt_project") + notebook.content = notebook.content.replace("src/dbt_profiles", f"src/{prefix}/dbt_profiles") + continue + if original_path.startswith("resources/") or original_path == "pyproject.toml": + continue + notebook.relative_path = f"{prefix}/{original_path}" + replacements[f"../src/{original_path}"] = f"../src/{notebook.relative_path}" + replacements[f"src/{original_path}"] = f"src/{notebook.relative_path}" + + for inner in cloned.inner_workflows: + original_key = normalize_task_key(inner.name) + inner.name = f"{prefix}__{inner.name}" + replacements[f"${{resources.jobs.{original_key}.id}}"] = ( + f"${{resources.jobs.{normalize_task_key(inner.name)}.id}}" + ) + + _replace_strings(cloned.tasks, replacements) + for inner in cloned.inner_workflows: + _replace_strings(inner.tasks, replacements) + return cloned + + +def _replace_strings(value: Any, replacements: dict[str, str]) -> Any: + """Replaces generated path and resource references recursively in place.""" + if isinstance(value, dict): + for key, item in value.items(): + value[key] = _replace_strings(item, replacements) + return value + if isinstance(value, list): + for index, item in enumerate(value): + value[index] = _replace_strings(item, replacements) + return value + if isinstance(value, str): + for original, replacement in replacements.items(): + value = value.replace(original, replacement) + return value + + def _warn(task_key: str, message: str) -> None: """Record a translation warning for the current bundle.""" _bundle_warnings.append(f"- **{task_key}**: {message}") @@ -1168,6 +1311,9 @@ def _apply_schedule_to_job(job_def: dict[str, Any], spec: dict[str, Any]) -> Non # malformed schedule. SETUP.md picks it up downstream. job_def["schedule_setup_note"] = spec return + if kind == "continuous": + job_def["continuous"] = {"pause_status": spec.get("pause_status", "UNPAUSED")} + return if kind == "periodic": # SCHED3-002: Day/Week/Month with interval > 1 maps to trigger.periodic. trigger_block: dict[str, Any] = { @@ -1388,10 +1534,7 @@ def _build_job_resource( seen_param_names.add(name) entry: dict[str, Any] = {"name": name} default = parameter.get("default") - if default is not None: - # Databricks job-parameter defaults are strings; JSON-encode - # Array / Object defaults so the YAML carries valid JSON. - entry["default"] = json.dumps(default) if isinstance(default, (list, dict)) else default + entry["default"] = default if isinstance(default, str) else json.dumps(default) normalized_parameters.append(entry) job_def["parameters"] = normalized_parameters @@ -1405,7 +1548,8 @@ def _build_job_resource( if overrides and job_def.get("parameters"): for entry in job_def["parameters"]: if entry.get("name") in overrides: - entry["default"] = overrides[entry["name"]] + override = overrides[entry["name"]] + entry["default"] = override if isinstance(override, str) else json.dumps(override) return { "resources": { @@ -1567,6 +1711,7 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d parameters=parameters or None, translation_configuration=_reconstruct_configuration(pipeline_dict.get("translation_configuration")), schedule=pipeline_dict.get("schedule"), + tags=dict(pipeline_dict.get("tags") or {}), ) return pipeline, parameters @@ -1703,6 +1848,10 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity: manifest_path=task_ir.get("manifest_path"), render_mode=task_ir.get("render_mode", "static"), selectors=list(task_ir.get("selectors") or []), + exclude_selectors=list(task_ir.get("exclude_selectors") or []), + variables=task_ir.get("variables"), + full_refresh=bool(task_ir.get("full_refresh", False)), + resource_types=list(task_ir.get("resource_types") or []), nodes=list(task_ir.get("nodes") or []), ) if task_type == "SqlActivity": diff --git a/src/flowx/bundler/prereqs_writer.py b/src/flowx/bundler/prereqs_writer.py index c4ee335..a0bc593 100644 --- a/src/flowx/bundler/prereqs_writer.py +++ b/src/flowx/bundler/prereqs_writer.py @@ -517,11 +517,11 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append("") lines.append( "Each row below describes a `run_job_task` that invokes a job **not** " - "defined in this bundle. flowx emitted a bundle variable for each " - "one (`${var.}`) so `databricks bundle validate` passes. " - "Before running, populate the variable with the numeric job ID the " - "target pipeline was deployed under — either set a `default:` in " - '`databricks.yml` or pass `--var "="` at deploy time.' + "defined in this bundle. flowx replaced each external resource reference " + "with a declared bundle variable (`${var.}`). Before validating, " + "deploying, or running the bundle, populate the variable with the numeric " + "job ID the target pipeline was deployed under — either set a `default:` " + 'in `databricks.yml` or pass `--var "="` to the bundle command.' ) lines.append("") lines.append("| Variable | Target pipeline |") @@ -775,10 +775,11 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: "`databricks bundle deploy` can run them:" ) lines.append("") - lines.append("1. Install the generator and PyDABs into the bundle's venv (the `python.venv_path`):") + lines.append("1. Synchronize the generated Python project into the bundle's `python.venv_path`:") + lines.append(" The generated `pyproject.toml` pins `databricks-dbt-factory`, PyDABs, and dbt.") lines.append("") lines.append("```bash") - lines.append("uv venv .venv && uv pip install databricks-dbt-factory databricks-bundles") + lines.append("uv sync") lines.append("```") lines.append("") lines.append("2. Ensure each dbt project's `manifest.json` exists (run `dbt parse`/`dbt compile`).") diff --git a/src/flowx/dbt/manifest.py b/src/flowx/dbt/manifest.py index 35b43a1..a48e850 100644 --- a/src/flowx/dbt/manifest.py +++ b/src/flowx/dbt/manifest.py @@ -79,7 +79,7 @@ def _fqn_selector(fqn: list[str]) -> str: return "fqn:" + ".".join(fqn) -def load_dbt_nodes(manifest_path: Path) -> list[DbtNode]: +def load_dbt_nodes(manifest_path: Path, *, resource_types: set[str] | None = None) -> list[DbtNode]: """Reads a dbt manifest and returns its runnable nodes as task specs. Args: @@ -96,21 +96,22 @@ def load_dbt_nodes(manifest_path: Path) -> list[DbtNode]: them), or when a node's fqn contains unsafe characters. """ manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) - return explode_manifest(manifest) + return explode_manifest(manifest, resource_types=resource_types) -def explode_manifest(manifest: dict) -> list[DbtNode]: +def explode_manifest(manifest: dict, *, resource_types: set[str] | None = None) -> list[DbtNode]: """Explodes an in-memory dbt manifest dict into runnable task specs. Split out from :func:`load_dbt_nodes` so tests can pass a synthetic manifest dict without touching the filesystem. """ nodes: dict[str, dict] = manifest.get("nodes", {}) + enabled_types = _RUNNABLE_RESOURCE_TYPES if resource_types is None else _RUNNABLE_RESOURCE_TYPES & resource_types runnable: dict[str, DbtNode] = {} for unique_id, node in nodes.items(): resource_type = node.get("resource_type", "") - if resource_type not in _RUNNABLE_RESOURCE_TYPES: + if resource_type not in enabled_types: continue fqn = node.get("fqn") or [node.get("name", unique_id)] runnable[unique_id] = DbtNode( @@ -133,9 +134,22 @@ def explode_manifest(manifest: dict) -> list[DbtNode]: # Prune dependency edges to the exploded set. dbt nodes depend on sources, macros, and each other; # only edges between two runnable nodes become task dependencies. task_key_by_uid = {uid: dbt_node.task_key for uid, dbt_node in runnable.items()} + tests_by_tested_uid: dict[str, list[str]] = {} + for test_uid, test_node in runnable.items(): + if test_node.resource_type != "test": + continue + for tested_uid in nodes[test_uid].get("depends_on", {}).get("nodes") or []: + tests_by_tested_uid.setdefault(tested_uid, []).append(test_node.task_key) for uid, dbt_node in runnable.items(): upstream_uids = nodes[uid].get("depends_on", {}).get("nodes") or [] - dbt_node.depends_on = [task_key_by_uid[up] for up in upstream_uids if up in task_key_by_uid] + dependencies = [ + task_key_by_uid[upstream_uid] for upstream_uid in upstream_uids if upstream_uid in task_key_by_uid + ] + if dbt_node.resource_type != "test": + dependencies.extend( + test_key for upstream_uid in upstream_uids for test_key in tests_by_tested_uid.get(upstream_uid, []) + ) + dbt_node.depends_on = list(dict.fromkeys(dependencies)) _assert_unique_task_keys(list(runnable.values())) # Deterministic order: manifest iteration order is stable, but sort by task_key so the emitted diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py index 2d86861..6844d47 100644 --- a/src/flowx/ir_serde.py +++ b/src/flowx/ir_serde.py @@ -167,6 +167,14 @@ def activity_extra_fields(activity: Activity) -> dict[str, Any]: extra["render_mode"] = activity.render_mode if activity.selectors: extra["selectors"] = list(activity.selectors) + if activity.exclude_selectors: + extra["exclude_selectors"] = list(activity.exclude_selectors) + if activity.variables is not None: + extra["variables"] = activity.variables + if activity.full_refresh: + extra["full_refresh"] = True + if activity.resource_types: + extra["resource_types"] = list(activity.resource_types) if activity.nodes: extra["nodes"] = list(activity.nodes) case CopyActivity(): diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py index 95513ff..90cc397 100644 --- a/src/flowx/mcp/server.py +++ b/src/flowx/mcp/server.py @@ -180,7 +180,16 @@ def _cmd_convert(p: dict[str, Any]) -> dict[str, Any]: def _cmd_merge_agentic(p: dict[str, Any]) -> dict[str, Any]: - args = ["convert", "--merge-agentic", "--report", p["report_path"], "--agentic-results", p["agentic_results_dir"]] + args = [ + "convert", + "--source", + _source_name(p), + "--merge-agentic", + "--report", + p["report_path"], + "--agentic-results", + p["agentic_results_dir"], + ] if p.get("output_path"): args += ["--output", p["output_path"]] result = runner.run_adapter(args) diff --git a/src/flowx/models/ir.py b/src/flowx/models/ir.py index 242bd91..931f2d8 100644 --- a/src/flowx/models/ir.py +++ b/src/flowx/models/ir.py @@ -509,6 +509,10 @@ class DbtFactoryActivity(Activity): render_mode: ``"static"`` or ``"pydabs"``. selectors: dbt ``--select`` selectors the source restricted the run to, if any (empty means the whole project). + exclude_selectors: dbt ``--exclude`` selectors from the source task. + variables: Literal dbt ``--vars`` value from the source task. + full_refresh: Whether the source requested ``--full-refresh``. + resource_types: dbt manifest resource types enabled by the source command. nodes: Pre-exploded node specs (list of dicts with ``task_key``, ``command``, ``selector``, ``depends_on``) when the front-end already read the manifest; empty when the preparer should read @@ -521,6 +525,10 @@ class DbtFactoryActivity(Activity): manifest_path: str | None = None render_mode: str = "static" selectors: list[str] = field(default_factory=list) + exclude_selectors: list[str] = field(default_factory=list) + variables: dict[str, Any] | str | None = None + full_refresh: bool = False + resource_types: list[str] = field(default_factory=list) nodes: list[dict[str, Any]] = field(default_factory=list) diff --git a/src/flowx/preparer/activity_preparers/dbt_factory.py b/src/flowx/preparer/activity_preparers/dbt_factory.py index a330a03..abbe934 100644 --- a/src/flowx/preparer/activity_preparers/dbt_factory.py +++ b/src/flowx/preparer/activity_preparers/dbt_factory.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +import shlex from pathlib import Path from typing import TYPE_CHECKING, Any @@ -29,6 +30,80 @@ from flowx.models.ir import DbtFactoryActivity _RUNNER_RELATIVE_PATH = "notebooks/run_dbt_command.py" +_DBT_PROJECT_RELATIVE_PATH = "dbt_project" +_DBT_PROFILES_RELATIVE_PATH = "dbt_profiles" +_EXCLUDED_DBT_PATH_PARTS = {".git", ".venv", "__pycache__", "logs", "target"} + + +def _tree_artifacts(source_root: Path, destination_root: str) -> list[DabNotebook]: + """Returns bundle artifacts for files beneath a local source directory.""" + if not source_root.is_dir(): + return [] + return [ + DabNotebook( + relative_path=(Path(destination_root) / source.relative_to(source_root)).as_posix(), + binary_content=source.read_bytes(), + ) + for source in sorted(source_root.rglob("*")) + if source.is_file() + and not source.is_symlink() + and not (_EXCLUDED_DBT_PATH_PARTS & set(source.relative_to(source_root).parts)) + ] + + +def _dbt_source_artifacts(activity: DbtFactoryActivity) -> list[DabNotebook]: + """Returns deployable project, profile, and manifest files available on the local filesystem.""" + project_dir = Path(activity.project_dir).expanduser() + artifacts = ( + _tree_artifacts(project_dir, _DBT_PROJECT_RELATIVE_PATH) if (project_dir / "dbt_project.yml").is_file() else [] + ) + + profiles_dir = Path(activity.profiles_dir).expanduser() + if (profiles_dir / "profiles.yml").is_file(): + artifacts.extend(_tree_artifacts(profiles_dir, _DBT_PROFILES_RELATIVE_PATH)) + + manifest_path = Path(activity.manifest_path).expanduser() if activity.manifest_path else None + if manifest_path and manifest_path.is_file(): + artifacts.append( + DabNotebook( + relative_path=f"{_DBT_PROJECT_RELATIVE_PATH}/target/manifest.json", + binary_content=manifest_path.read_bytes(), + ) + ) + return artifacts + + +def _pydabs_pyproject_source() -> str: + """Returns the uv project required by generated PyDABs hooks.""" + return ( + "[project]\n" + 'name = "flowx-dbt-bundle"\n' + 'version = "0.1.0"\n' + 'requires-python = ">=3.10"\n' + "dependencies = [\n" + ' "databricks-bundles>=1.0.0,<2.0.0",\n' + ' "databricks-dbt-factory==0.2.1",\n' + ' "dbt-databricks==1.12.2",\n' + ' "dbt-core==1.11.12",\n' + "]\n" + ) + + +def _pydabs_options_by_resource_type(activity: DbtFactoryActivity) -> dict[str, str]: + """Returns shell-safe dbt options for each generated task-factory type.""" + common = ["--target", activity.target] + for selector in activity.exclude_selectors: + common.extend(("--exclude", selector)) + if activity.variables is not None: + variables = json.dumps(activity.variables) if isinstance(activity.variables, dict) else activity.variables + common.extend(("--vars", variables)) + options: dict[str, str] = {} + for resource_type in activity.resource_types or ["model", "seed", "snapshot", "test"]: + tokens = [*common] + if activity.full_refresh and resource_type in {"model", "seed"}: + tokens.append("--full-refresh") + options[resource_type] = shlex.join(tokens) + return options def _nodes_from_activity(activity: DbtFactoryActivity) -> list[dict[str, Any]]: @@ -39,7 +114,20 @@ def _nodes_from_activity(activity: DbtFactoryActivity) -> list[dict[str, Any]]: ``selector``, and ``depends_on`` (task keys). """ if activity.nodes: - return activity.nodes + if not activity.resource_types: + return activity.nodes + command_types = {"run": "model", "seed": "seed", "snapshot": "snapshot", "test": "test"} + selected = [] + for node in activity.nodes: + command = node.get("command") + resource_type = command_types.get(command) if isinstance(command, str) else None + if resource_type is not None and resource_type in activity.resource_types: + selected.append(node) + selected_keys = {node["task_key"] for node in selected} + return [ + {**node, "depends_on": [key for key in node.get("depends_on") or [] if key in selected_keys]} + for node in selected + ] if activity.manifest_path: manifest = json.loads(Path(activity.manifest_path).read_text(encoding="utf-8")) return [ @@ -49,7 +137,7 @@ def _nodes_from_activity(activity: DbtFactoryActivity) -> list[dict[str, Any]]: "selector": node.selector, "depends_on": node.depends_on, } - for node in explode_manifest(manifest) + for node in explode_manifest(manifest, resource_types=set(activity.resource_types) or None) ] return [] @@ -65,24 +153,47 @@ def _runner_notebook_source() -> str: "# Databricks notebook source\n" "# Owned dbt-command runner for flowx dbt-factory (static) mode.\n" "# One task per dbt node passes its command + fqn: selector as widgets.\n\n" + "import json\n" + "import os\n" "import subprocess\n\n" "dbutils.widgets.text('dbt_command', 'run')\n" "dbutils.widgets.text('dbt_select', '')\n" + "dbutils.widgets.text('dbt_selectors', '[]')\n" + "dbutils.widgets.text('dbt_exclude', '[]')\n" + "dbutils.widgets.text('dbt_vars', '')\n" + "dbutils.widgets.text('dbt_full_refresh', 'false')\n" "dbutils.widgets.text('dbt_target', 'dev')\n" "dbutils.widgets.text('dbt_project_dir', '.')\n" "dbutils.widgets.text('dbt_profiles_dir', 'dbt_profiles')\n\n" "command = dbutils.widgets.get('dbt_command')\n" "select = dbutils.widgets.get('dbt_select')\n" + "selectors = json.loads(dbutils.widgets.get('dbt_selectors'))\n" + "exclude = json.loads(dbutils.widgets.get('dbt_exclude'))\n" + "variables = dbutils.widgets.get('dbt_vars')\n" + "full_refresh = dbutils.widgets.get('dbt_full_refresh').lower() == 'true'\n" "target = dbutils.widgets.get('dbt_target')\n" "project_dir = dbutils.widgets.get('dbt_project_dir')\n" "profiles_dir = dbutils.widgets.get('dbt_profiles_dir')\n\n" + "context = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n" + "notebook_dir = os.path.dirname('/Workspace' + context.notebookPath().get())\n" + "if not os.path.isabs(project_dir):\n" + " project_dir = os.path.normpath(os.path.join(notebook_dir, project_dir))\n" + "if not os.path.isabs(profiles_dir):\n" + " profiles_dir = os.path.normpath(os.path.join(notebook_dir, profiles_dir))\n\n" "argv = ['dbt', command, '--target', target, '--project-dir', project_dir,\n" " '--profiles-dir', profiles_dir]\n" "if select:\n" - " argv += ['--select', select]\n" + " selected_nodes = [f'{select},{selector}' for selector in selectors] if selectors else [select]\n" + " argv += ['--select', *selected_nodes]\n" " if command == 'test':\n" " # Pin test selection to the node itself; don't pull in indirectly-selected tests.\n" " argv += ['--indirect-selection', 'empty']\n\n" + "if exclude:\n" + " argv += ['--exclude', *exclude]\n" + "if variables:\n" + " argv += ['--vars', variables]\n" + "if full_refresh and command in {'run', 'seed'}:\n" + " argv.append('--full-refresh')\n\n" "print('running:', ' '.join(argv))\n" "result = subprocess.run(argv, check=False)\n" "if result.returncode != 0:\n" @@ -90,18 +201,68 @@ def _runner_notebook_source() -> str: ) +def _pydabs_runner_notebook_source() -> str: + """Returns the notebook contract expected by databricks-dbt-factory notebook tasks.""" + return ( + "# Databricks notebook source\n\n" + "import json\n" + "import os\n" + "import shlex\n\n" + "from dbt.cli.main import dbtRunner\n\n" + "dbutils.widgets.text('dbt_commands', '')\n" + "dbutils.widgets.text('project_directory', '')\n" + "dbutils.widgets.text('profiles_directory', '')\n\n" + "commands = json.loads(dbutils.widgets.get('dbt_commands'))\n" + "project_directory = dbutils.widgets.get('project_directory')\n" + "profiles_directory = dbutils.widgets.get('profiles_directory')\n" + "context = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n" + "os.environ['DBT_ACCESS_TOKEN'] = context.apiToken().get()\n" + "os.environ['DBT_HOST'] = context.apiUrl().get()\n\n" + "if project_directory:\n" + " notebook_dir = os.path.dirname('/Workspace' + context.notebookPath().get())\n" + " project_path = (\n" + " project_directory\n" + " if os.path.isabs(project_directory)\n" + " else os.path.normpath(os.path.join(notebook_dir, project_directory))\n" + " )\n" + " os.chdir(project_path)\n\n" + "runner = dbtRunner()\n" + "for command in commands:\n" + " command = command.strip()\n" + " if command.startswith('dbt '):\n" + " command = command[4:]\n" + " arguments = shlex.split(command)\n" + " if profiles_directory:\n" + " arguments.extend(['--profiles-dir', profiles_directory])\n" + " result = runner.invoke(arguments)\n" + " if not result.success:\n" + " detail = result.exception or result.result or '(no further details)'\n" + " raise RuntimeError(f\"dbt command failed: dbt {' '.join(arguments)}\\n{detail}\")\n" + ) + + def _node_task(node: dict[str, Any], activity: DbtFactoryActivity) -> dict[str, Any]: """Builds one inner-job notebook task for a dbt node.""" task: dict[str, Any] = { "task_key": node["task_key"], + "libraries": [ + {"pypi": {"package": "dbt-databricks==1.12.2"}}, + {"pypi": {"package": "dbt-core==1.11.12"}}, + ], "notebook_task": { "notebook_path": f"../src/{_RUNNER_RELATIVE_PATH}", "base_parameters": { "dbt_command": node["command"], "dbt_select": node["selector"], + "dbt_selectors": json.dumps(activity.selectors), + "dbt_exclude": json.dumps(activity.exclude_selectors), + "dbt_vars": ( + json.dumps(activity.variables) if isinstance(activity.variables, dict) else activity.variables or "" + ), + "dbt_full_refresh": str(activity.full_refresh).lower(), "dbt_target": activity.target, - "dbt_project_dir": activity.project_dir, - "dbt_profiles_dir": activity.profiles_dir, + "dbt_project_dir": f"../{_DBT_PROJECT_RELATIVE_PATH}", + "dbt_profiles_dir": f"../{_DBT_PROFILES_RELATIVE_PATH}", }, }, } @@ -117,12 +278,15 @@ def _prepare_static(activity: DbtFactoryActivity, nodes: list[dict[str, Any]]) - inner_job_name = f"{activity.task_key}_dbt" inner_tasks = [_node_task(node, activity) for node in nodes] - runner_notebook = DabNotebook(relative_path=_RUNNER_RELATIVE_PATH, content=_runner_notebook_source()) + notebooks = [ + DabNotebook(relative_path=_RUNNER_RELATIVE_PATH, content=_runner_notebook_source()), + *_dbt_source_artifacts(activity), + ] inner_workflow = PreparedWorkflow( name=inner_job_name, tasks=inner_tasks, - notebooks=[runner_notebook], + notebooks=notebooks, secrets=[], setup_tasks=[], ) @@ -133,21 +297,93 @@ def _prepare_static(activity: DbtFactoryActivity, nodes: list[dict[str, Any]]) - return PreparedActivity(task=parent_task, inner_workflows=[inner_workflow]) +def _prepare_missing_inputs(activity: DbtFactoryActivity, missing_inputs: list[str]) -> PreparedActivity: + """Returns a failing placeholder task when required local dbt inputs are unavailable.""" + relative_path = f"notebooks/{activity.task_key}_dbt_setup_required.py" + missing = ", ".join(missing_inputs) + content = ( + "# Databricks notebook source\n" + f"# dbt project inputs required for migrated task {activity.task_key}.\n\n" + f"raise RuntimeError({f'Missing dbt project input(s): {missing}. Add them and re-run flowx package.'!r})\n" + ) + task = build_common_task_fields(activity) + task["notebook_task"] = {"notebook_path": f"../src/{relative_path}"} + return PreparedActivity(task=task, notebooks=[DabNotebook(relative_path=relative_path, content=content)]) + + +def _missing_dbt_inputs(activity: DbtFactoryActivity, *, require_manifest: bool = True) -> list[str]: + """Returns required dbt inputs that are not available to copy into the bundle.""" + project_dir = Path(activity.project_dir).expanduser() + profiles_dir = Path(activity.profiles_dir).expanduser() + manifest_path = Path(activity.manifest_path).expanduser() if activity.manifest_path else None + missing: list[str] = [] + if not (project_dir / "dbt_project.yml").is_file(): + missing.append(f"dbt project at {project_dir}") + if not (profiles_dir / "profiles.yml").is_file(): + missing.append(f"profiles.yml under {profiles_dir}") + if require_manifest and (manifest_path is None or not manifest_path.is_file()): + missing.append(f"manifest at {manifest_path or ''}") + return missing + + def _pydabs_hook_source(activity: DbtFactoryActivity) -> str: """Returns the PyDABs hook module body for deploy-time dbt-factory generation.""" + resource_types = activity.resource_types or ["model", "seed", "snapshot", "test"] + dbt_options = _pydabs_options_by_resource_type(activity) return ( '"""PyDABs hook: build the dbt job from the live manifest at deploy time."""\n\n' "from databricks.bundles.core import Bundle, Resources\n" + "from databricks.bundles.jobs import Job\n" "from databricks_dbt_factory.DbtFactory import DbtFactory\n" - "from databricks_dbt_factory.Utils import read_dbt_manifest\n\n" - f"MANIFEST_PATH = {activity.manifest_path or 'target/manifest.json'!r}\n" - f"PROJECT_DIR = {activity.project_dir!r}\n" - f"PROFILES_DIR = {activity.profiles_dir!r}\n\n" + "from databricks_dbt_factory.DbtTask import DbtTaskOptions, TaskType\n" + "from databricks_dbt_factory.SpecsHandler import SpecsHandler\n" + "from databricks_dbt_factory.TaskFactory import (\n" + " DbtDependencyResolver,\n" + " ModelTaskFactory,\n" + " SeedTaskFactory,\n" + " SnapshotTaskFactory,\n" + " TestTaskFactory,\n" + ")\n\n" + f"MANIFEST_PATH = {'src/dbt_project/target/manifest.json'!r}\n" + f"PROJECT_DIR = {'../dbt_project'!r}\n" + f"PROFILES_DIR = {'../dbt_profiles'!r}\n" + f"RESOURCE_TYPES = {resource_types!r}\n\n" + f"DBT_OPTIONS = {dbt_options!r}\n\n" + "def _task_factories():\n" + " resolver = DbtDependencyResolver()\n" + " options = DbtTaskOptions(\n" + " task_type=TaskType.NOTEBOOK,\n" + " environment_key='Default',\n" + " notebook_path='src/notebooks/run_dbt_command.py',\n" + " project_directory=PROJECT_DIR,\n" + " profiles_directory=PROFILES_DIR,\n" + " )\n" + " factory_classes = {\n" + " 'model': ModelTaskFactory,\n" + " 'seed': SeedTaskFactory,\n" + " 'snapshot': SnapshotTaskFactory,\n" + " 'test': TestTaskFactory,\n" + " }\n" + " return {\n" + " name: factory_classes[name](resolver, options, DBT_OPTIONS[name])\n" + " for name in RESOURCE_TYPES\n" + " }\n\n" "def load_resources(bundle: Bundle) -> Resources:\n" + " manifest = SpecsHandler.read_dbt_manifest(MANIFEST_PATH)\n" + " task_factories = _task_factories()\n" " resources = Resources()\n" - " factory = DbtFactory()\n" - " tasks = factory.create_tasks(read_dbt_manifest(MANIFEST_PATH))\n" - f" resources.add_job({normalize_task_key(activity.task_key + '_dbt')!r}, {{'tasks': tasks}})\n" + " factory = DbtFactory(SpecsHandler(), task_factories, bundle_tests=False)\n" + " tasks = factory.create_tasks(manifest)\n" + " environment = {\n" + " 'environment_key': 'Default',\n" + " 'spec': {\n" + " 'environment_version': '4',\n" + " 'dependencies': ['dbt-databricks==1.12.2', 'dbt-core==1.11.12'],\n" + " },\n" + " }\n" + f" resources.add_job({normalize_task_key(activity.task_key + '_dbt')!r}, Job(\n" + f" name={normalize_task_key(activity.task_key + '_dbt')!r}, tasks=tasks, environments=[environment]\n" + " ))\n" " return resources\n" ) @@ -165,26 +401,45 @@ def _prepare_pydabs(activity: DbtFactoryActivity) -> PreparedActivity: hook_relative_path = f"resources/{activity.task_key}_dbt_job.py" hook_notebook = DabNotebook(relative_path=hook_relative_path, content=_pydabs_hook_source(activity)) + runner_notebook = DabNotebook(relative_path=_RUNNER_RELATIVE_PATH, content=_pydabs_runner_notebook_source()) + pyproject = DabNotebook(relative_path="pyproject.toml", content=_pydabs_pyproject_source()) # `resources` must be an importable package for `python.resources: resources.` to resolve. package_marker = DabNotebook(relative_path="resources/__init__.py", content="") + source_artifacts = _dbt_source_artifacts(activity) setup_task = SetupTask( type="pydabs_dbt_factory", config={ "hook_module": f"resources.{activity.task_key}_dbt_job", "job_key": inner_job_key, - "manifest_path": activity.manifest_path or "target/manifest.json", + "manifest_path": "src/dbt_project/target/manifest.json", "note": ( - "dbt-factory PyDABs mode: add a `python.resources` entry to databricks.yml pointing at " - f"`resources.{activity.task_key}_dbt_job:load_resources`, and `pip install databricks-dbt-factory`." + "dbt-factory PyDABs mode: databricks.yml registers " + f"`resources.{activity.task_key}_dbt_job:load_resources`; run `uv sync` before bundle commands." ), }, ) - return PreparedActivity(task=parent_task, notebooks=[hook_notebook, package_marker], setup_tasks=[setup_task]) + return PreparedActivity( + task=parent_task, + notebooks=[hook_notebook, package_marker, runner_notebook, pyproject, *source_artifacts], + setup_tasks=[setup_task], + ) def prepare(activity: DbtFactoryActivity, *, scope: str = "") -> PreparedActivity: """Converts a DbtFactoryActivity into DAB tasks per its render mode.""" + if activity.resource_types == ["dependency"]: + missing_inputs = _missing_dbt_inputs(activity, require_manifest=False) + if missing_inputs: + return _prepare_missing_inputs(activity, missing_inputs) + dependency_node = {"task_key": "dbt_deps", "command": "deps", "selector": "", "depends_on": []} + return _prepare_static(activity, [dependency_node]) + if activity.render_mode == "pydabs" or not activity.nodes: + missing_inputs = _missing_dbt_inputs(activity) + if missing_inputs: + return _prepare_missing_inputs(activity, missing_inputs) if activity.render_mode == "pydabs": + if activity.selectors: + return _prepare_static(activity, _nodes_from_activity(activity)) return _prepare_pydabs(activity) nodes = _nodes_from_activity(activity) return _prepare_static(activity, nodes) diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py index 72e782a..aebc581 100644 --- a/src/flowx/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -68,6 +68,7 @@ class PreparedWorkflow: # C-10 (SCHED-001): serialised schedule / trigger spec the bundler # renders as ``schedule:`` / ``trigger:`` on the emitted DAB job. schedule: dict[str, Any] | None = None + source: str | None = None # The DAB job ``run_if`` vocabulary. Airflow maps ``trigger_rule`` straight to one of these @@ -416,6 +417,7 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: pipeline_resources=list(artifacts.pipeline_resources), parameter_approximations=list(artifacts.parameter_approximations), schedule=pipeline.schedule, + source=str(pipeline.tags.get("source")) if pipeline.tags.get("source") else None, ) diff --git a/src/flowx/sources/airflow/convert.py b/src/flowx/sources/airflow/convert.py index 0d9c443..b3d57fb 100644 --- a/src/flowx/sources/airflow/convert.py +++ b/src/flowx/sources/airflow/convert.py @@ -14,6 +14,7 @@ import logging from pathlib import Path +from flowx import ir_serde from flowx.ir_serde import pipeline_to_dict from flowx.models.ir import PlaceholderActivity from flowx.sources.airflow.loader import load_pipelines @@ -24,7 +25,7 @@ def main(argv: list[str] | None = None) -> int: """Convert-phase entry point for the Airflow source.""" parser = argparse.ArgumentParser(description="Translate Airflow DAGs into flowx Pipeline IR.") - parser.add_argument("--source-dir", required=True, type=Path, help="A DAG .py file or directory of DAGs.") + parser.add_argument("--source-dir", required=False, type=Path, help="A DAG .py file or directory of DAGs.") parser.add_argument("--output-dir", type=Path, default=Path("./flowx_output"), help="Shared migration output dir.") parser.add_argument("--pipeline", type=str, default=None, help="Translate only the named DAG (default: all).") parser.add_argument( @@ -34,10 +35,36 @@ def main(argv: list[str] | None = None) -> int: help="dbt-factory render mode: 'static' (inner job of per-node tasks, default) or 'pydabs' " "(a deploy-time PyDABs hook that builds the dbt job from the live manifest).", ) + parser.add_argument( + "--merge-agentic", + action="store_true", + help="Merge agent-produced results from --agentic-results into --report instead of translating.", + ) + parser.add_argument("--report", type=Path, default=None, help="Translation report to merge agentic results into.") + parser.add_argument( + "--agentic-results", + type=Path, + default=None, + help="Directory of per-activity agentic result JSON files.", + ) + parser.add_argument("--output", type=Path, default=None, help="Merged report destination; defaults to --report.") args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + if args.merge_agentic: + if not args.report or not args.agentic_results: + parser.error("--merge-agentic requires --report and --agentic-results") + merged_count, unmatched_count = ir_serde.merge_agentic_results(args.report, args.agentic_results, args.output) + print("\nAgentic Merge Summary") + print("=====================") + print(f"Merged: {merged_count}") + print(f"Unmatched: {unmatched_count}") + return 0 if unmatched_count == 0 else 1 + + if not args.source_dir: + parser.error("--source-dir is required (unless using --merge-agentic)") + pipelines = load_pipelines(args.source_dir, pipeline=args.pipeline, dbt_mode=args.dbt_mode) if not pipelines: logger.error("No Airflow DAGs found under %s (or none matched --pipeline).", args.source_dir) diff --git a/src/flowx/validate/bundle_invariants.py b/src/flowx/validate/bundle_invariants.py index c457a16..4043469 100644 --- a/src/flowx/validate/bundle_invariants.py +++ b/src/flowx/validate/bundle_invariants.py @@ -24,6 +24,7 @@ # appears more than once in the tree. flowx never intends to emit these. _ANCHOR_RE = re.compile(r"[&*]id\d+\b") _JOB_PARAM_REF_RE = re.compile(r"\{\{\s*job\.parameters\.([A-Za-z0-9_]+)\s*\}\}") +_JOB_RESOURCE_ID_RE = re.compile(r"\$\{resources\.jobs\.([^.}]+)\.id\}") @dataclass(slots=True, kw_only=True) @@ -76,6 +77,17 @@ def _collect_task_keys(tasks: list[dict[str, Any]]) -> list[str]: return keys +def _iter_tasks(tasks: list[dict[str, Any]]): + """Yields top-level tasks and nested ``for_each_task.task`` bodies.""" + for task in tasks: + if not isinstance(task, dict): + continue + yield task + nested = (task.get("for_each_task") or {}).get("task") + if isinstance(nested, dict): + yield from _iter_tasks([nested]) + + def _dump(obj: Any) -> str: """Serialise a structure to a string for reference scanning.""" return yaml.safe_dump(obj, default_flow_style=False) @@ -212,15 +224,53 @@ def check_bundle_dir(bundle_dir: Path) -> BundleInvariantResult: databricks_yml = bundle_dir / "databricks.yml" if databricks_yml.exists(): yaml_files.append(databricks_yml) + + documents: list[tuple[Path, dict[str, Any]]] = [] for path in yaml_files: - findings.extend(check_resource_text(path.read_text(encoding="utf-8"), filename=path.name)) + text = path.read_text(encoding="utf-8") + findings.extend(check_resource_text(text, filename=path.name)) + document = yaml.safe_load(text) or {} + if isinstance(document, dict): + documents.append((path, document)) + + known_jobs: set[str] = set() + for _path, document in documents: + jobs = (document.get("resources") or {}).get("jobs") or {} + if isinstance(jobs, dict): + known_jobs.update(str(job_key) for job_key in jobs) + + for path, document in documents: + jobs = (document.get("resources") or {}).get("jobs") or {} + if not isinstance(jobs, dict): + continue + for job_key, job in jobs.items(): + if not isinstance(job, dict): + continue + for task in _iter_tasks(job.get("tasks") or []): + run_job = task.get("run_job_task") or {} + job_id = run_job.get("job_id") if isinstance(run_job, dict) else None + match = _JOB_RESOURCE_ID_RE.fullmatch(job_id) if isinstance(job_id, str) else None + if match is None or match.group(1) in known_jobs: + continue + findings.append( + BundleFinding( + code="dangling_run_job_reference", + severity="warning", + location=f"{path.name}, job '{job_key}', task '{task.get('task_key', '')}'", + message=( + f"run_job_task references bundle job '{match.group(1)}', which is not declared " + "in static resource YAML. Confirm it is supplied by a Python resource or replace " + "the reference with a declared bundle variable containing the external job ID." + ), + ) + ) return BundleInvariantResult(findings=findings) def format_result(result: BundleInvariantResult) -> str: """Render a result as a compact human-readable report.""" - if result.ok: + if not result.findings: return "Bundle invariants: OK" - lines = ["Bundle invariants: FAILED"] + lines = ["Bundle invariants: FAILED" if result.violations else "Bundle invariants: WARNINGS"] lines.extend(f" - [{finding.code}] {finding.location}: {finding.message}" for finding in result.findings) return "\n".join(lines) diff --git a/tests/unit/test_bundle_invariants.py b/tests/unit/test_bundle_invariants.py index d61a12d..8c2f8a1 100644 --- a/tests/unit/test_bundle_invariants.py +++ b/tests/unit/test_bundle_invariants.py @@ -2,7 +2,7 @@ from __future__ import annotations -from flowx.validate.bundle_invariants import check_job, check_resource_text +from flowx.validate.bundle_invariants import check_bundle_dir, check_job, check_resource_text, format_result def _codes(findings) -> set[str]: @@ -80,3 +80,49 @@ def test_acyclic_chain_has_no_cycle_finding(): ] } assert "dependency_cycle" not in _codes(check_job("p", job)) + + +def test_bundle_job_reference_can_target_job_in_another_resource_file(tmp_path): + resources = tmp_path / "resources" + resources.mkdir() + (resources / "parent.yml").write_text( + "resources:\n" + " jobs:\n" + " parent:\n" + " tasks:\n" + " - task_key: call_child\n" + " run_job_task:\n" + " job_id: ${resources.jobs.child.id}\n", + encoding="utf-8", + ) + (resources / "child.yml").write_text( + "resources:\n jobs:\n child:\n tasks: []\n", + encoding="utf-8", + ) + + result = check_bundle_dir(tmp_path) + + assert "dangling_run_job_reference" not in _codes(result.findings) + + +def test_bundle_job_reference_to_unknown_resource_is_flagged(tmp_path): + resources = tmp_path / "resources" + resources.mkdir() + (resources / "parent.yml").write_text( + "resources:\n" + " jobs:\n" + " parent:\n" + " tasks:\n" + " - task_key: call_missing\n" + " run_job_task:\n" + " job_id: ${resources.jobs.missing.id}\n", + encoding="utf-8", + ) + + result = check_bundle_dir(tmp_path) + finding = next(finding for finding in result.findings if finding.code == "dangling_run_job_reference") + + assert finding.severity == "warning" + assert "parent.yml" in finding.location + assert "call_missing" in finding.location + assert "dangling_run_job_reference" in format_result(result) diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index 90ecb02..371eb9c 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -212,6 +212,67 @@ def test_module_state_does_not_leak_across_calls(self, tmp_path): assert not (first_dir / "WARNINGS.md").exists() assert not (second_dir / "WARNINGS.md").exists() + def test_cross_bundle_job_reference_uses_declared_job_id_variable(self, tmp_path): + workflow = PreparedWorkflow( + name="parent", + tasks=[ + { + "task_key": "call_child", + "run_job_task": {"job_id": "${resources.jobs.child.id}"}, + } + ], + notebooks=[], + secrets=[], + setup_tasks=[], + ) + + write_bundle(workflow, tmp_path) + + config = yaml.safe_load((tmp_path / "databricks.yml").read_text()) + resource = yaml.safe_load((tmp_path / "resources" / "parent.yml").read_text()) + task = resource["resources"]["jobs"]["parent"]["tasks"][0] + setup = (tmp_path / "SETUP.md").read_text() + assert "child_job_id" in config["variables"] + assert task["run_job_task"]["job_id"] == "${var.child_job_id}" + assert "`child_job_id`" in setup + assert "`child`" in setup + + second_output = tmp_path / "second" + write_bundle(workflow, second_output) + second_config = yaml.safe_load((second_output / "databricks.yml").read_text()) + assert "child_job_id" in second_config["variables"] + + def test_python_resource_job_reference_remains_a_resource_substitution(self, tmp_path): + workflow = PreparedWorkflow( + name="parent", + tasks=[ + { + "task_key": "call_dbt", + "run_job_task": {"job_id": "${resources.jobs.generated_dbt.id}"}, + } + ], + notebooks=[], + secrets=[], + setup_tasks=[ + SetupTask( + type="pydabs_dbt_factory", + config={ + "hook_module": "resources.generated_dbt_job", + "job_key": "generated_dbt", + }, + ) + ], + ) + + write_bundle(workflow, tmp_path) + + config = yaml.safe_load((tmp_path / "databricks.yml").read_text()) + resource = yaml.safe_load((tmp_path / "resources" / "parent.yml").read_text()) + task = resource["resources"]["jobs"]["parent"]["tasks"][0] + assert task["run_job_task"]["job_id"] == "${resources.jobs.generated_dbt.id}" + assert "generated_dbt_job_id" not in config["variables"] + assert "## Cross-bundle job references" not in (tmp_path / "SETUP.md").read_text() + def test_load_report_handles_aggregated_translations_format(self, tmp_path): """``_load_report`` accepts the multi-pipeline aggregated report. @@ -314,6 +375,19 @@ def test_periodic_trigger_emitted(self, tmp_path): # The cron-style schedule block must NOT appear for periodic specs. assert "schedule" not in job + def test_continuous_mode_emitted(self, tmp_path): + pipeline = Pipeline( + name="continuous_job", + tasks=[WaitActivity(name="Pause", task_key="pause", wait_time_seconds=10)], + schedule={"kind": "continuous", "pause_status": "UNPAUSED"}, + ) + write_bundle(prepare_workflow(pipeline), tmp_path) + resource_file = next((tmp_path / "resources").glob("*.yml")) + job = next(iter(yaml.safe_load(resource_file.read_text())["resources"]["jobs"].values())) + assert job["continuous"] == {"pause_status": "UNPAUSED"} + assert "schedule" not in job + assert "trigger" not in job + def test_trigger_parameter_overrides_mutate_job_parameter_defaults(self, tmp_path): """SCHED3-003: schedule.parameter_overrides mutates matching job.parameters entries' default values.""" @@ -350,6 +424,28 @@ def test_trigger_parameter_overrides_mutate_job_parameter_defaults(self, tmp_pat assert params["negocio"] == "GLP" assert params["applicationName"] == "app0001" + def test_job_parameter_defaults_are_strings(self, tmp_path): + pipeline = Pipeline( + name="typed_parameters", + tasks=[WaitActivity(name="Pause", task_key="pause", wait_time_seconds=10)], + parameters=[ + {"name": "threshold", "default": 10}, + {"name": "enabled", "default": True}, + {"name": "settings", "default": {"mode": "fast"}}, + ], + ) + + write_bundle(prepare_workflow(pipeline), tmp_path) + + resource_file = next((tmp_path / "resources").glob("*.yml")) + job = next(iter(yaml.safe_load(resource_file.read_text())["resources"]["jobs"].values())) + defaults = {parameter["name"]: parameter["default"] for parameter in job["parameters"]} + assert defaults == { + "threshold": "10", + "enabled": "true", + "settings": '{"mode": "fast"}', + } + def test_file_arrival_trigger_emitted(self, tmp_path): pipeline = Pipeline( name="blob_triggered_job", diff --git a/tests/unit/test_dbt_factory_preparer.py b/tests/unit/test_dbt_factory_preparer.py index ab43631..b6e1fbd 100644 --- a/tests/unit/test_dbt_factory_preparer.py +++ b/tests/unit/test_dbt_factory_preparer.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json + from flowx.models.ir import DbtFactoryActivity, Dependency, NotebookActivity, Pipeline from flowx.preparer.workflow_preparer import prepare_activity, prepare_workflow @@ -32,6 +34,24 @@ def _dbt_activity(**overrides): return DbtFactoryActivity(**kwargs) +def _pydabs_activity(tmp_path, **overrides): + project = tmp_path / "dbt-source" + profiles = tmp_path / "dbt-profiles" + (project / "target").mkdir(parents=True) + profiles.mkdir(parents=True) + (project / "dbt_project.yml").write_text("name: demo\nprofile: demo\n") + (project / "target" / "manifest.json").write_text(json.dumps({"nodes": {}})) + (profiles / "profiles.yml").write_text("demo:\n target: dev\n outputs: {}\n") + return _dbt_activity( + nodes=[], + render_mode="pydabs", + project_dir=str(project), + profiles_dir=str(profiles), + manifest_path=str(project / "target" / "manifest.json"), + **overrides, + ) + + def test_static_parent_task_is_run_job_hop(): prepared = prepare_activity(_dbt_activity()) assert "run_job_task" in prepared.task @@ -46,6 +66,11 @@ def test_static_emits_inner_job_with_one_task_per_node(): assert task_keys == {"seed_codes", "model_stg", "model_fct", "test_stg"} +def test_static_filters_preloaded_nodes_to_command_resource_types(): + prepared = prepare_activity(_dbt_activity(resource_types=["model"])) + assert {task["task_key"] for task in prepared.inner_workflows[0].tasks} == {"model_stg", "model_fct"} + + def test_static_preserves_node_dependencies(): prepared = prepare_activity(_dbt_activity()) inner = prepared.inner_workflows[0] @@ -62,6 +87,32 @@ def test_static_node_task_carries_command_and_selector(): assert params["dbt_command"] == "test" assert params["dbt_select"] == "fqn:p.staging.t" assert params["dbt_target"] == "dev" + packages = {library["pypi"]["package"] for library in test_task["libraries"]} + assert packages == {"dbt-databricks==1.12.2", "dbt-core==1.11.12"} + + +def test_static_node_task_carries_source_dbt_options(): + prepared = prepare_activity( + _dbt_activity( + selectors=["tag:daily"], + exclude_selectors=["tag:slow"], + variables={"region": "west"}, + full_refresh=True, + ) + ) + model_task = next(t for t in prepared.inner_workflows[0].tasks if t["task_key"] == "model_stg") + params = model_task["notebook_task"]["base_parameters"] + + assert params["dbt_selectors"] == '["tag:daily"]' + assert params["dbt_exclude"] == '["tag:slow"]' + assert params["dbt_vars"] == '{"region": "west"}' + assert params["dbt_full_refresh"] == "true" + runner = next( + notebook for notebook in prepared.inner_workflows[0].notebooks if "run_dbt_command" in notebook.relative_path + ) + assert "--exclude" in runner.content + assert "--vars" in runner.content + assert "--full-refresh" in runner.content def test_static_emits_single_shared_runner_notebook(): @@ -80,19 +131,59 @@ def test_static_parent_hop_keeps_upstream_dependency(): assert prepared.task["depends_on"] == [{"task_key": "ingest"}] -def test_pydabs_emits_hook_module_and_no_inner_job(): - prepared = prepare_activity(_dbt_activity(render_mode="pydabs", manifest_path="target/manifest.json")) +def test_static_missing_manifest_emits_manual_placeholder_instead_of_crashing(tmp_path): + prepared = prepare_activity( + _dbt_activity(nodes=[], manifest_path=str(tmp_path / "missing" / "manifest.json"), resource_types=["model"]) + ) + + assert prepared.inner_workflows == [] + assert "notebook_task" in prepared.task + assert "manifest" in prepared.notebooks[0].content + + +def test_pydabs_missing_local_inputs_emits_setup_placeholder(tmp_path): + prepared = prepare_activity( + _dbt_activity( + nodes=[], + render_mode="pydabs", + project_dir=str(tmp_path / "missing-project"), + profiles_dir=str(tmp_path / "missing-profiles"), + manifest_path=str(tmp_path / "missing-manifest.json"), + ) + ) + + assert prepared.inner_workflows == [] + assert "notebook_task" in prepared.task + assert not any(notebook.relative_path.startswith("resources/") for notebook in prepared.notebooks) + assert "dbt project" in prepared.notebooks[0].content + + +def test_pydabs_emits_hook_module_and_no_inner_job(tmp_path): + prepared = prepare_activity(_pydabs_activity(tmp_path)) assert prepared.inner_workflows == [] hook_paths = {nb.relative_path for nb in prepared.notebooks} # The hook module plus a resources/ package marker so `python.resources` can import it. - assert hook_paths == {"resources/dbt_transform_dbt_job.py", "resources/__init__.py"} + assert { + "resources/dbt_transform_dbt_job.py", + "resources/__init__.py", + "notebooks/run_dbt_command.py", + "pyproject.toml", + } <= hook_paths hook = next(nb for nb in prepared.notebooks if nb.relative_path.endswith("_dbt_job.py")) assert "load_resources" in hook.content + assert "from databricks_dbt_factory.SpecsHandler import SpecsHandler" in hook.content + assert "DbtFactory(SpecsHandler(), task_factories" in hook.content + import_block = hook.content.split("from databricks_dbt_factory", maxsplit=1)[1].split("\n\n", maxsplit=1)[0] + assert "read_dbt_manifest" not in import_block + runner = next(nb for nb in prepared.notebooks if nb.relative_path == "notebooks/run_dbt_command.py") + assert "dbt_commands" in runner.content + assert "project_directory" in runner.content + assert "profiles_directory" in runner.content assert "run_job_task" in prepared.task -def test_pydabs_records_setup_task(): - prepared = prepare_activity(_dbt_activity(render_mode="pydabs")) +def test_pydabs_records_setup_task(tmp_path): + prepared = prepare_activity(_pydabs_activity(tmp_path)) setup_types = {t.type for t in prepared.setup_tasks} assert "pydabs_dbt_factory" in setup_types @@ -124,12 +215,28 @@ def test_survives_json_report_round_trip(): from flowx.ir_serde import pipeline_to_dict from flowx.models.ir import DbtFactoryActivity - pipeline = Pipeline(name="orders", tasks=[_dbt_activity()]) + pipeline = Pipeline( + name="orders", + tasks=[ + _dbt_activity( + resource_types=["model"], + selectors=["tag:daily"], + exclude_selectors=["tag:slow"], + variables={"region": "west"}, + full_refresh=True, + ) + ], + ) rehydrated, _ = pipeline_dict_to_ir(pipeline_to_dict(pipeline)) dbt = rehydrated.tasks[0] assert isinstance(dbt, DbtFactoryActivity) assert dbt.render_mode == "static" assert {n["task_key"] for n in dbt.nodes} == {"seed_codes", "model_stg", "model_fct", "test_stg"} + assert dbt.resource_types == ["model"] + assert dbt.selectors == ["tag:daily"] + assert dbt.exclude_selectors == ["tag:slow"] + assert dbt.variables == {"region": "west"} + assert dbt.full_refresh is True def test_pydabs_bundle_wires_python_resources_and_setup(tmp_path): @@ -148,10 +255,9 @@ def test_pydabs_bundle_wires_python_resources_and_setup(tmp_path): notebook_path="notebooks/ingest.py", generated_source="# Databricks notebook source\nprint('x')\n", ), - _dbt_activity( + _pydabs_activity( + tmp_path, depends_on=[Dependency(task_key="ingest")], - render_mode="pydabs", - manifest_path="dbt/target/manifest.json", ), ], ) @@ -164,6 +270,41 @@ def test_pydabs_bundle_wires_python_resources_and_setup(tmp_path): assert (tmp_path / "resources" / "dbt_transform_dbt_job.py").exists() assert (tmp_path / "resources" / "__init__.py").exists() assert not (tmp_path / "src" / "resources").exists() + pyproject = (tmp_path / "pyproject.toml").read_text() + assert "databricks-dbt-factory==0.2.1" in pyproject + assert "dbt-databricks==1.12.2" in pyproject setup = (tmp_path / "SETUP.md").read_text() assert "dbt factory (PyDABs mode)" in setup assert "databricks-dbt-factory" in setup + assert "uv sync" in setup + + +def test_pydabs_copies_available_dbt_project_into_bundle(tmp_path): + from flowx.bundler.dab_writer import write_bundle + + project = tmp_path / "project" + (project / "models").mkdir(parents=True) + (project / "dbt_project.yml").write_text("name: demo\nprofile: demo\n") + (project / "models" / "orders.sql").write_text("select 1\n") + (project / "target").mkdir() + (project / "target" / "manifest.json").write_text(json.dumps({"nodes": {}})) + profiles = tmp_path / "profiles" + profiles.mkdir() + (profiles / "profiles.yml").write_text("demo:\n target: dev\n outputs: {}\n") + output = tmp_path / "bundle" + pipeline = Pipeline( + name="orders", + tasks=[ + _dbt_activity( + render_mode="pydabs", + project_dir=str(project), + profiles_dir=str(profiles), + manifest_path=str(project / "target" / "manifest.json"), + ) + ], + ) + + write_bundle(prepare_workflow(pipeline), output) + + assert (output / "src" / "dbt_project" / "dbt_project.yml").exists() + assert (output / "src" / "dbt_project" / "models" / "orders.sql").exists() diff --git a/tests/unit/test_dbt_manifest.py b/tests/unit/test_dbt_manifest.py index 25a2af4..b3b217e 100644 --- a/tests/unit/test_dbt_manifest.py +++ b/tests/unit/test_dbt_manifest.py @@ -46,6 +46,20 @@ def test_explodes_each_runnable_resource_type(): assert by_key["test_t"].command == "test" +def test_explosion_can_limit_resource_types_to_airflow_command_scope(): + manifest = _manifest( + { + "model.p.stg": _model("stg", ["p", "staging", "stg"]), + "seed.p.codes": _seed("codes", ["p", "codes"]), + "test.p.t": _test("t", ["p", "staging", "t"], deps=["model.p.stg"]), + } + ) + + nodes = explode_manifest(manifest, resource_types={"model"}) + + assert [(node.resource_type, node.name) for node in nodes] == [("model", "stg")] + + def test_fqn_selector_built_from_components(): manifest = _manifest({"model.p.stg": _model("stg", ["p", "staging", "stg"])}) (node,) = explode_manifest(manifest) @@ -65,6 +79,20 @@ def test_dependency_edges_pruned_to_exploded_set(): assert by_key["model_fct"].depends_on == ["model_stg"] # source edge dropped, model kept +def test_downstream_model_waits_for_tests_on_its_upstream_model(): + manifest = _manifest( + { + "model.p.stg": _model("stg", ["p", "stg"]), + "test.p.stg_not_null": _test("stg_not_null", ["p", "stg_not_null"], deps=["model.p.stg"]), + "model.p.fct": _model("fct", ["p", "fct"], deps=["model.p.stg"]), + } + ) + + by_key = {node.task_key: node for node in explode_manifest(manifest)} + + assert by_key["model_fct"].depends_on == ["model_stg", "test_stg_not_null"] + + def test_non_runnable_resource_types_skipped(): manifest = _manifest( { diff --git a/tests/unit/test_mcp_source_routing.py b/tests/unit/test_mcp_source_routing.py index 5ecdfd5..5b5a81a 100644 --- a/tests/unit/test_mcp_source_routing.py +++ b/tests/unit/test_mcp_source_routing.py @@ -64,6 +64,18 @@ def test_convert_threads_source(captured, tmp_path: Path): assert argv[argv.index("--source") + 1] == "airflow" +def test_merge_agentic_threads_source(captured): + server._cmd_merge_agentic( + { + "source": "adf", + "report_path": "/tmp/report.json", + "agentic_results_dir": "/tmp/results", + } + ) + argv = _argv(captured, "convert") + assert argv[argv.index("--source") + 1] == "adf" + + def test_inputs_threads_source(captured): server._cmd_inputs({"phase": "discover", "source": "airflow"}) argv = _argv(captured, "inputs") diff --git a/tests/unit/test_package_invariants.py b/tests/unit/test_package_invariants.py index 6a0e8ab..a5154c8 100644 --- a/tests/unit/test_package_invariants.py +++ b/tests/unit/test_package_invariants.py @@ -6,6 +6,8 @@ import tempfile from pathlib import Path +import yaml + from flowx.bundler.dab_writer import main as package_main @@ -59,3 +61,126 @@ def test_package_loads_multi_pipeline_report(): workflows = _load_report(report_path) assert [w.name for w in workflows] == ["first", "second"] assert package_main(["--output-dir", str(out)]) == 0 + + +def test_package_writes_airflow_dags_as_jobs_in_one_shared_bundle(): + report = { + "pipelines": [ + { + "name": "parent", + "tags": {"source": "airflow"}, + "tasks": [ + _notebook_task("extract", "extract"), + { + "name": "trigger_child", + "task_key": "trigger_child", + "type": "RunJobActivity", + "job_name": "child", + }, + ], + }, + { + "name": "child", + "tags": {"source": "airflow"}, + "tasks": [_notebook_task("extract", "extract")], + }, + ] + } + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + work = out / ".work" + work.mkdir(parents=True) + (work / "translation_report.json").write_text(json.dumps(report), encoding="utf-8") + + assert package_main(["--output-dir", str(out), "--bundle-name", "airflow-suite"]) == 0 + assert (out / "databricks.yml").exists() + assert {path.name for path in (out / "resources").glob("*.yml")} == {"parent.yml", "child.yml"} + assert not (out / "parent" / "databricks.yml").exists() + assert not (out / "child" / "databricks.yml").exists() + assert (out / "src" / "parent" / "notebooks" / "extract.py").exists() + assert (out / "src" / "child" / "notebooks" / "extract.py").exists() + + parent_resource = (out / "resources" / "parent.yml").read_text(encoding="utf-8") + assert "${resources.jobs.child.id}" in parent_resource + + +def test_shared_bundle_cross_dag_ref_resolves_for_hyphenated_dag_id(): + # A TriggerDagRunOperator targeting a hyphenated/mixed-case dag_id must reference the target + # job by its normalized resource key, not a differently-sanitized name, or the ref dangles. + report = { + "pipelines": [ + { + "name": "downstream", + "tags": {"source": "airflow"}, + "tasks": [ + { + "name": "trig", + "task_key": "trig", + "type": "RunJobActivity", + "job_name": "upstream_dag", # normalize_task_key("Upstream-DAG") + }, + ], + }, + { + "name": "Upstream-DAG", + "tags": {"source": "airflow"}, + "tasks": [_notebook_task("a", "a")], + }, + ] + } + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + work = out / ".work" + work.mkdir(parents=True) + (work / "translation_report.json").write_text(json.dumps(report), encoding="utf-8") + + assert package_main(["--output-dir", str(out), "--bundle-name", "airflow-suite"]) == 0 + job_files = {path.name for path in (out / "resources").glob("*.yml")} + assert "upstream_dag.yml" in job_files + upstream = yaml.safe_load((out / "resources" / "upstream_dag.yml").read_text()) + assert "upstream_dag" in upstream["resources"]["jobs"] + downstream = (out / "resources" / "downstream.yml").read_text(encoding="utf-8") + # The ref matches the emitted job resource key (would be ${...Upstream-DAG.id} before the fix). + assert "${resources.jobs.upstream_dag.id}" in downstream + + +def test_shared_airflow_bundle_namespaces_pydabs_hooks_and_jobs(): + dbt_task = { + "name": "dbt", + "task_key": "dbt", + "type": "DbtFactoryActivity", + "project_dir": ".", + "manifest_path": "target/manifest.json", + "render_mode": "pydabs", + "resource_types": ["model"], + } + report = { + "pipelines": [ + {"name": "first", "tags": {"source": "airflow"}, "tasks": [dbt_task]}, + {"name": "second", "tags": {"source": "airflow"}, "tasks": [dbt_task]}, + ] + } + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + project = out / "dbt-project" + profiles = out / "dbt-profiles" + (project / "target").mkdir(parents=True) + profiles.mkdir() + (project / "dbt_project.yml").write_text("name: demo\nprofile: demo\n") + (project / "target" / "manifest.json").write_text(json.dumps({"nodes": {}})) + (profiles / "profiles.yml").write_text("demo:\n target: dev\n outputs: {}\n") + dbt_task["project_dir"] = str(project) + dbt_task["profiles_dir"] = str(profiles) + dbt_task["manifest_path"] = str(project / "target" / "manifest.json") + work = out / ".work" + work.mkdir(parents=True) + (work / "translation_report.json").write_text(json.dumps(report), encoding="utf-8") + + assert package_main(["--output-dir", str(out)]) == 0 + databricks_yml = (out / "databricks.yml").read_text(encoding="utf-8") + assert "resources.first_dbt_dbt_job:load_resources" in databricks_yml + assert "resources.second_dbt_dbt_job:load_resources" in databricks_yml + assert (out / "resources" / "first_dbt_dbt_job.py").exists() + assert (out / "resources" / "second_dbt_dbt_job.py").exists() + assert "${resources.jobs.first_dbt_dbt.id}" in (out / "resources" / "first.yml").read_text() + assert "${resources.jobs.second_dbt_dbt.id}" in (out / "resources" / "second.yml").read_text() From 16e7dc39084e8c5db2507d4007600ddbb843199d Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 20 Jul 2026 17:42:06 -0700 Subject: [PATCH 32/77] comment cleanup --- src/flowx/sources/airflow/operators.py | 3 +-- tests/integration/test_airflow_golden_bundle.py | 4 ++-- tests/resources/airflow/golden_pipeline_dag.py | 2 +- tests/unit/test_airflow_operators.py | 2 +- tests/unit/test_mcp_source_routing.py | 3 +-- tests/unit/test_package_invariants.py | 4 ++-- 6 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/flowx/sources/airflow/operators.py b/src/flowx/sources/airflow/operators.py index 9d8ef6c..9e713b8 100644 --- a/src/flowx/sources/airflow/operators.py +++ b/src/flowx/sources/airflow/operators.py @@ -653,8 +653,7 @@ def _build_branch(ctx: OperatorContext) -> Activity: # Airflow Branch/ShortCircuit gate *sibling* tasks on a Python callable's return, which flowx # can't statically lower to a condition_task's left/op/right plus per-branch true/false outcome # wiring. Emitting it as an ordinary notebook would silently let every downstream branch run, so - # route it to the agentic-gap round with the callable source instead (a real translation, not a - # wrong-but-quiet one). Full Branch->condition_task remains a scoped follow-up. + # route it to the agentic-gap round with the callable source instead. return _placeholder( ctx, f"Airflow {ctx.operator} selects downstream tasks at runtime. Translate to a Databricks " diff --git a/tests/integration/test_airflow_golden_bundle.py b/tests/integration/test_airflow_golden_bundle.py index 95c4d68..7e47bbe 100644 --- a/tests/integration/test_airflow_golden_bundle.py +++ b/tests/integration/test_airflow_golden_bundle.py @@ -1,8 +1,8 @@ """Golden-bundle test for the Airflow source. Converts a single representative DAG (tests/resources/airflow/golden_pipeline_dag.py) all the -way to a DAB bundle on disk and pins the emitted job YAML + notebooks. It guards the Phase-2 -conversion behaviours together, end-to-end, so a regression in any one of them fails here: +way to a DAB bundle on disk and pins the emitted job YAML + notebooks. It guards these +conversion behaviours together, end-to-end: - cron schedule + a root file sensor -> schedule kept AND sensor retained as a polling task (schedule / file_arrival triggers are mutually exclusive on a Databricks job) diff --git a/tests/resources/airflow/golden_pipeline_dag.py b/tests/resources/airflow/golden_pipeline_dag.py index a77ec84..6b73eb8 100644 --- a/tests/resources/airflow/golden_pipeline_dag.py +++ b/tests/resources/airflow/golden_pipeline_dag.py @@ -1,6 +1,6 @@ """Golden-bundle fixture DAG for the airflow source. -Exercises the Phase-2 conversion behaviours in one representative DAG so the golden test +Exercises several conversion behaviours in one representative DAG so the golden test pins their end-to-end bundle output: - cron schedule AND a root file sensor -> schedule kept + sensor retained as a polling task - a mid-DAG table sensor -> polling task (not a trigger) diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py index 08e58bc..36d9a50 100644 --- a/tests/unit/test_airflow_operators.py +++ b/tests/unit/test_airflow_operators.py @@ -49,7 +49,7 @@ def test_python_operator_becomes_generated_notebook(): def test_python_operator_notebook_is_valid_python(): # A callable with an early return, a helper, a constant, and a non-Airflow import must - # produce a notebook that compiles (the review's P1 blind spot: top-level return / undefined names). + # produce a notebook that compiles (no top-level return, no undefined names). p = _load( "from datetime import datetime\n" "from airflow import DAG\n" diff --git a/tests/unit/test_mcp_source_routing.py b/tests/unit/test_mcp_source_routing.py index 5b5a81a..ff3f03a 100644 --- a/tests/unit/test_mcp_source_routing.py +++ b/tests/unit/test_mcp_source_routing.py @@ -1,7 +1,6 @@ """Tests that the MCP dispatcher threads --source to the adapter for both sources. -Guards the P0 regression where the adapter began requiring --source but the MCP commands -never passed it (breaking ADF and never supporting Airflow). +The adapter requires --source for discover/convert, so every MCP command must pass it. """ from __future__ import annotations diff --git a/tests/unit/test_package_invariants.py b/tests/unit/test_package_invariants.py index a5154c8..c1a0416 100644 --- a/tests/unit/test_package_invariants.py +++ b/tests/unit/test_package_invariants.py @@ -43,7 +43,7 @@ def test_package_fails_on_duplicate_task_key(): def test_package_loads_multi_pipeline_report(): # A {"pipelines": [...]} report (emitted for multi-DAG conversion) must package all pipelines, - # not silently produce "no pipelines found". Guards the P0 multi-DAG load crash. + # not silently produce "no pipelines found". from flowx.bundler.dab_writer import _load_report report = { @@ -140,7 +140,7 @@ def test_shared_bundle_cross_dag_ref_resolves_for_hyphenated_dag_id(): upstream = yaml.safe_load((out / "resources" / "upstream_dag.yml").read_text()) assert "upstream_dag" in upstream["resources"]["jobs"] downstream = (out / "resources" / "downstream.yml").read_text(encoding="utf-8") - # The ref matches the emitted job resource key (would be ${...Upstream-DAG.id} before the fix). + # The ref must match the emitted job resource key, which is normalize_task_key(dag_id). assert "${resources.jobs.upstream_dag.id}" in downstream From d5a4f74ac6894c848b892e69e381570e1180d86a Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:07:41 -0400 Subject: [PATCH 33/77] Fix actions for deploying documentation (#13) ## Changes This PR fixes GitHub actions for deploying flowx documentation to GitHub pages. ### Linked issues N/A ### Tests - [x] manually tested - [ ] added unit tests - [ ] added integration tests --- .github/workflows/docs-release.yml | 4 +++- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml index 4c06abf..c46e51c 100644 --- a/.github/workflows/docs-release.yml +++ b/.github/workflows/docs-release.yml @@ -53,7 +53,9 @@ jobs: deploy: name: Deploy to GitHub Pages needs: build - runs-on: ubuntu-latest + runs-on: + group: databricks-solutions-protected-runner-group + labels: linux-ubuntu-latest permissions: pages: write diff --git a/pyproject.toml b/pyproject.toml index 2792f4a..8d3e242 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ yq = [ [build-system] requires = [ - "hatchling>=1.90,<2.0" + "hatchling>=1.27,<2.0" ] build-backend = "hatchling.build" From 294f65802ca21bb199fe4af27f29d34f16ee9ac9 Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Tue, 21 Jul 2026 09:20:15 -0700 Subject: [PATCH 34/77] Require --source everywhere; drop the adf default Remove the implicit adf fallback from the remaining entry points so source selection is uniformly explicit: MCP _source_name, the workspace-paths and inputs CLIs, and MigrationInputSession no longer default to adf. discover/ convert/migrate now fail clearly when source is absent; package stays source-independent. Tests pass --source explicitly. --- AGENTS.md | 2 +- app/README.md | 2 +- docs/content/docs/installation.mdx | 2 +- scripts/bootstrap.sh | 4 +- skills/flowx-migrate/SKILL.md | 24 +++--- skills/flowx-package/SKILL.md | 5 +- skills/flowx-setup/SKILL.md | 2 +- src/flowx/adapter/__main__.py | 40 ++++++---- src/flowx/adapter/session.py | 42 ++++++---- src/flowx/mcp/server.py | 81 +++++++++++++------- tests/unit/test_adapter.py | 37 ++++++--- tests/unit/test_airflow_adapter_reporting.py | 9 ++- tests/unit/test_mcp_migrate.py | 12 ++- tests/unit/test_mcp_source_routing.py | 37 ++++++++- 14 files changed, 207 insertions(+), 92 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3bfe895..10deb4f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ bootstrap a self-contained virtual environment with pip via the `setup` skill or bash scripts/bootstrap.sh # creates the venv, pip-installs requirements.txt, writes .migration-venv # then run plugin code with src/ on PYTHONPATH, using the interpreter from the marker file: PY="$(cat .migration-venv)" -PYTHONPATH=src "$PY" -m flowx.adapter inputs discover +PYTHONPATH=src "$PY" -m flowx.adapter inputs discover --source adf # or --source airflow ``` `bootstrap.sh` creates the venv at `/Workspace/Users//.migration-skills` when running diff --git a/app/README.md b/app/README.md index c60cbf7..3faa3d7 100644 --- a/app/README.md +++ b/app/README.md @@ -30,7 +30,7 @@ operation; `parameters` is its keyword-argument dict. | `record_results` | `adapter record-results` | Write coverage to a UC table | | `install_dashboard` | `adapter install-dashboard` | Publish the coverage dashboard | -Example: `flowx(command="discover", parameters={"adf_source_path": "/Volumes/main/default/adf_export", "output_dir": "./out"})`. +Example: `flowx(command="discover", parameters={"source": "adf", "adf_source_path": "/Volumes/main/default/adf_export", "output_dir": "./out"})`. Each command is a thin bridge over `python -m flowx.adapter` (the same entry point the agent skills use), then reads back the JSON/CSV artifacts each phase writes — so the MCP surface stays in diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index cb9f5d9..9c984e9 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -180,7 +180,7 @@ If you hit a `ModuleNotFoundError` while running a phase, the venv is missing or ```bash export PYTHONPATH="/src" PY="$(cat /.migration-venv)" -"$PY" -m flowx.adapter inputs discover +"$PY" -m flowx.adapter inputs discover --source adf # or --source airflow ``` diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 50cb58e..93d89e3 100644 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -17,7 +17,7 @@ # After bootstrapping, run the plugin's Python code with the venv interpreter and # src/ on PYTHONPATH, e.g.: # -# PYTHONPATH="/src" "/bin/python" -m flowx.adapter inputs discover +# PYTHONPATH="/src" "/bin/python" -m flowx.adapter inputs discover --source adf # # The resolved interpreter path is also written to /.migration-venv # so the skills can discover it without re-deriving the location. @@ -201,5 +201,5 @@ flowx Python environment is ready. Marker file : $PLUGIN_ROOT/.migration-venv Run the plugin's Python code with src/ on PYTHONPATH, for example: - PYTHONPATH="$PLUGIN_ROOT/src" "$VENV_PYTHON" -m flowx.adapter inputs discover + PYTHONPATH="$PLUGIN_ROOT/src" "$VENV_PYTHON" -m flowx.adapter inputs discover --source adf EOF diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index f707de5..81cca14 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -58,6 +58,7 @@ phase: ``` flowx(command="migrate", parameters={ + "source": "adf", "adf_definitions": {"pipeline/Foo.json": {...}, "linkedService/Bar.json": {...}, ...}, "output_dir": ..., "catalog": ..., "schema": ..., "pipeline": ""}) ``` @@ -94,13 +95,15 @@ To accept all defaults and skip the prompts, pass `"interactive": false`. (Re-ca > `mcp-flowx` app's service principal read on the source and write on the output target. For step-by-step control, run the commands in order (the app reuses `output_dir` across calls, so -only `discover` needs `adf_definitions`): +only `discover` needs the source input). `source` ("adf" | "airflow") is required for +discover/convert/merge_agentic and for `inputs discover`/`inputs convert`; for Airflow, swap +`adf_definitions` for `airflow_source_path`. `package` and `inputs package` are source-independent: ``` -flowx(command="inputs", parameters={"phase": "discover" | "convert" | "package"}) # learn each phase's inputs -flowx(command="discover", parameters={"adf_definitions": {...}, "output_dir": ..., "pipeline": ...}) -flowx(command="convert", parameters={"output_dir": ..., "pipeline": ...}) -flowx(command="merge_agentic", parameters={"report_path": ..., "agentic_results_dir": ..., "output_path": ...}) # if agentic results +flowx(command="inputs", parameters={"phase": "discover", "source": "adf"}) # source req for discover/convert +flowx(command="discover", parameters={"source": "adf", "adf_definitions": {...}, "output_dir": ..., "pipeline": ...}) +flowx(command="convert", parameters={"source": "adf", "output_dir": ..., "pipeline": ...}) +flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...}) # if agentic results flowx(command="inspect", parameters={"report_path": ...}) flowx(command="apply_answers", parameters={"report_path": ..., "answers": [...], "output_dir": ...}) flowx(command="package", parameters={"output_dir": ..., "catalog": ..., "schema": ...}) @@ -128,7 +131,7 @@ interpreter (from the marker file `/.migration-venv`) and `src/` on ```bash export PYTHONPATH="/src" PY="$(cat /.migration-venv)" -"$PY" -m flowx.adapter inputs discover +"$PY" -m flowx.adapter inputs discover --source adf # or --source airflow ``` If Python or pip is missing, `bootstrap.sh` prints a warning telling the user what to install — relay @@ -144,9 +147,9 @@ Before invoking discover, run the adapter inputs subcommand once per phase so the agent surfaces the matching free-text prompts: ```bash -"$PY" -m flowx.adapter inputs discover -"$PY" -m flowx.adapter inputs convert -"$PY" -m flowx.adapter inputs package +"$PY" -m flowx.adapter inputs discover --source adf # or --source airflow +"$PY" -m flowx.adapter inputs convert --source adf # or --source airflow +"$PY" -m flowx.adapter inputs package # source-independent ``` Each response carries the options for that phase plus their descriptions and @@ -311,7 +314,8 @@ the bundle would need to download: ```bash "$PY" -m flowx.adapter workspace-paths \ /.work/translation_report.stamped.json \ - --source-dir + --source adf \ + --source-dir ``` When the response carries `needs_auth: true`: diff --git a/skills/flowx-package/SKILL.md b/skills/flowx-package/SKILL.md index f22769e..fda6dc7 100644 --- a/skills/flowx-package/SKILL.md +++ b/skills/flowx-package/SKILL.md @@ -41,7 +41,7 @@ This phase runs one of two ways; run the **`setup`** skill first if you haven't. "download_workspace_files": true, "output_volume_path": "", "output_workspace_path": ""}) - flowx(command="workspace_paths", parameters={"report_path": "...", "source_dir": ""}) + flowx(command="workspace_paths", parameters={"source": "adf", "report_path": "...", "source_dir": ""}) flowx(command="record_results", parameters={"output_dir": "", "results_table": "catalog.schema.table", "warehouse_id": ""}) flowx(command="install_dashboard", parameters={"results_table": "catalog.schema.table", "warehouse_id": ""}) ``` @@ -118,7 +118,8 @@ download to be self-contained: ```bash "$PY" -m flowx.adapter workspace-paths \ /.work/translation_report.stamped.json \ - --source-dir + --source adf \ + --source-dir ``` The command emits: diff --git a/skills/flowx-setup/SKILL.md b/skills/flowx-setup/SKILL.md index cd0ecf7..597a099 100644 --- a/skills/flowx-setup/SKILL.md +++ b/skills/flowx-setup/SKILL.md @@ -128,7 +128,7 @@ skills run Python with that interpreter and `src/` on `PYTHONPATH`. Resolve it f ```bash export PYTHONPATH="/src" PY="$(cat /.migration-venv)" -"$PY" -m flowx.adapter inputs discover +"$PY" -m flowx.adapter inputs discover --source adf # or --source airflow ``` `$PY` resolves to `/.venv/bin/python` (on Windows, `\.venv\Scripts\python.exe`). diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py index 6fcf69a..2449a9d 100644 --- a/src/flowx/adapter/__main__.py +++ b/src/flowx/adapter/__main__.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Any -from flowx.adapter.constants import MOTIF_CONSOLIDATE_OPTION_PREFIX +from flowx.adapter.constants import MOTIF_CONSOLIDATE_OPTION_PREFIX, PHASE_PACKAGE from flowx.adapter.models import ( DEFAULT_CONFIGURATION, CopyActivityParadigm, @@ -39,6 +39,8 @@ provision_notification_destinations, validate_answer, ) +from flowx.adapter.session import MigrationInputSession +from flowx.sources import available_sources, get_source # bundler.dab_writer + translator.engine (sqlglot) are imported lazily inside inspect/modify only, so # the cheap commands (inputs, phase pass-throughs, materialize-lookup, workspace-paths) skip ~0.15s of @@ -150,7 +152,7 @@ def _run_workspace_paths(args: argparse.Namespace) -> int: paths = collect_workspace_artifact_paths(args.report) suggested_hosts: list[str] = [] if args.source_dir: - if getattr(args, "source", "adf") == "airflow": + if args.source == "airflow": from flowx.sources.airflow.loader import detect_hosts suggested_hosts = detect_hosts(args.source_dir) @@ -169,15 +171,24 @@ def _run_inputs(args: argparse.Namespace) -> int: """Implements the ``inputs`` subcommand. Args: - args: Parsed CLI namespace carrying ``phase`` and ``out``. + args: Parsed CLI namespace carrying ``phase``, ``source``, and ``out``. Returns: - ``0`` on success. The CLI never raises here because the phase - argument is constrained by argparse. + ``0`` on success, or ``2`` when the discover/convert phase is missing + the required ``--source``. """ - from flowx.adapter.session import MigrationInputSession - - session = MigrationInputSession(phase=args.phase, source=getattr(args, "source", "adf")) + source = getattr(args, "source", None) + # discover/convert prompts are source-specific and need a known source; package is + # source-independent. Validate here so a missing or unknown source is a clean usage error + # rather than an uncaught ValueError from session.pending(). + if args.phase != PHASE_PACKAGE and source not in available_sources(): + problem = "is required" if source is None else f"{source!r} is not recognized" + print( + f"--source {problem} for the {args.phase} phase; choose one of: {', '.join(available_sources())}", + file=sys.stderr, + ) + return 2 + session = MigrationInputSession(phase=args.phase, source=source) pending = session.pending() payload = { "phase": pending.phase, @@ -293,7 +304,7 @@ def _build_parser() -> argparse.ArgumentParser: workspace_paths.add_argument("report", type=Path, help="Path to the translation report or pipeline IR JSON.") workspace_paths.add_argument( "--source", - default="adf", + required=True, help="Migration source (adf | airflow); selects how workspace hosts are detected.", ) workspace_paths.add_argument( @@ -324,8 +335,8 @@ def _build_parser() -> argparse.ArgumentParser: ) inputs.add_argument( "--source", - default="adf", - help="Migration source (adf | airflow); words the source-path prompt for discover/convert.", + default=None, + help="Migration source (adf | airflow); required for discover/convert, unused for package.", ) inputs.add_argument( "--out", @@ -405,8 +416,9 @@ def _build_parser() -> argparse.ArgumentParser: ) # Unified phase runners: `adapter --source -- ` routes discover/convert - # to the named source's phase module (default: adf, for back-compat). package is source-independent. - # --source-path (and each source's own alias, e.g. --adf-source-path) normalise to --source-dir. + # to the named source's phase module. --source is required for those phases (no default); + # package is source-independent. --source-path (and each source's own alias, e.g. + # --adf-source-path) normalise to --source-dir. for _phase in ("discover", "convert", "package"): _runner = subparsers.add_parser( _phase, @@ -474,8 +486,6 @@ def _run_phase(phase: str, forward: list[str]) -> int: """ import importlib - from flowx.sources import available_sources, get_source - source_name, remaining = _split_source(forward) if phase == "package": diff --git a/src/flowx/adapter/session.py b/src/flowx/adapter/session.py index 3a60374..f69ab17 100644 --- a/src/flowx/adapter/session.py +++ b/src/flowx/adapter/session.py @@ -271,7 +271,7 @@ def _collect_motif_consolidations(self) -> dict[str, MotifConsolidate]: def _discover_options(source: str) -> tuple[MigrationInputOption, ...]: """Discover-phase input prompts for *source* (source-path prompt varies by source).""" - spec = _SOURCE_PATH_OPTION.get(source, _SOURCE_PATH_OPTION["adf"]) + spec = _SOURCE_PATH_OPTION[source] options = [ MigrationInputOption( option_id=spec["option_id"], prompt=spec["prompt"], description=spec["description"], required=True @@ -296,7 +296,7 @@ def _discover_options(source: str) -> tuple[MigrationInputOption, ...]: def _convert_options(source: str) -> tuple[MigrationInputOption, ...]: """Convert-phase input prompts for *source*.""" - spec = _SOURCE_PATH_OPTION.get(source, _SOURCE_PATH_OPTION["adf"]) + spec = _SOURCE_PATH_OPTION[source] return ( MigrationInputOption( option_id=INPUT_INVENTORY_PATH, @@ -404,13 +404,15 @@ def _convert_options(source: str) -> tuple[MigrationInputOption, ...]: _SUPPORTED_PHASES: frozenset[str] = frozenset({PHASE_DISCOVER, PHASE_CONVERT, PHASE_PACKAGE}) -def _options_for(phase: str, source: str) -> tuple[MigrationInputOption, ...]: - """Returns the input options for *phase*, source-worded for discover/convert.""" - if phase == PHASE_DISCOVER: - return _discover_options(source) - if phase == PHASE_CONVERT: - return _convert_options(source) - return _PACKAGE_OPTIONS +def _options_for(phase: str, source: str | None) -> tuple[MigrationInputOption, ...]: + """Returns the input options for *phase*; discover/convert need a known source (else ``ValueError``).""" + if phase == PHASE_PACKAGE: + return _PACKAGE_OPTIONS + if source not in _SOURCE_PATH_OPTION: + raise ValueError( + f"--source is required for the {phase} phase; choose one of: {', '.join(sorted(_SOURCE_PATH_OPTION))}" + ) + return _discover_options(source) if phase == PHASE_DISCOVER else _convert_options(source) class UnknownMigrationPhaseError(ValueError): @@ -431,11 +433,12 @@ class MigrationInputSession: Attributes: phase: One of ``"discover"``, ``"convert"``, ``"package"``. source: Migration source (``"adf"`` / ``"airflow"``); words the - source-path prompt for the discover/convert phases. + source-path prompt for the discover/convert phases. Required for + those phases (there is no default source); unused for ``package``. """ phase: str - source: str = "adf" + source: str | None = None _answers: dict[str, str] = field(default_factory=dict) def __post_init__(self) -> None: @@ -455,6 +458,10 @@ def pending(self) -> PendingMigrationInputs: Returns: A :class:`PendingMigrationInputs` with the unanswered options for ``self.phase`` in registration order. + + Raises: + ValueError: When the discover/convert phase has a missing or + unrecognised ``source`` (there is no default source). """ options = [option for option in _options_for(self.phase, self.source) if option.option_id not in self._answers] return PendingMigrationInputs(phase=self.phase, options=options) @@ -468,7 +475,8 @@ def answer(self, option_id: str, value: str) -> None: Raises: ValueError: When *option_id* is not a known input for the - session's phase. + session's phase, or when the discover/convert phase has a + missing or unrecognised ``source`` (there is no default source). """ if not any(option.option_id == option_id for option in _options_for(self.phase, self.source)): raise ValueError(f"Unknown input option {option_id!r} for phase {self.phase!r}") @@ -481,8 +489,10 @@ def answer_many(self, answers: dict[str, str]) -> None: answers: Mapping of option_id to the caller-supplied value. Raises: - ValueError: When any pair references an unknown option. - No answers are recorded when the call raises. + ValueError: When any pair references an unknown option, or when + the discover/convert phase has a missing or unrecognised + ``source`` (there is no default source). No answers are + recorded when the call raises. """ known_ids = {option.option_id for option in _options_for(self.phase, self.source)} unknown = set(answers) - known_ids @@ -499,6 +509,10 @@ def collected(self) -> dict[str, str]: the option's ``default`` value (which may be the empty string) is used. Required options whose answers are missing are omitted so the caller can detect them. + + Raises: + ValueError: When the discover/convert phase has a missing or + unrecognised ``source`` (there is no default source). """ collected: dict[str, str] = {} for option in _options_for(self.phase, self.source): diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py index 90cc397..8c238e5 100644 --- a/src/flowx/mcp/server.py +++ b/src/flowx/mcp/server.py @@ -38,21 +38,26 @@ def _transport_security() -> TransportSecuritySettings: _INSTRUCTIONS = """\ -flowx translates Azure Data Factory (ADF) pipelines into Databricks Lakeflow Jobs packaged as -Declarative Automation Bundles (DABs). Everything is driven through the single `flowx` tool: -`flowx(command="", parameters={...})`. - -Typical flow: - flowx("inputs", {"phase": "discover"}) # learn a phase's inputs - flowx("discover", {"adf_source_path": "...", "output_dir": "..."}) - flowx("convert", {"output_dir": "..."}) +flowx translates a source orchestrator's pipelines (Azure Data Factory or Apache Airflow) into +Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). Everything is driven +through the single `flowx` tool: `flowx(command="", parameters={...})`. + +Every discover/convert/migrate call requires `source` ("adf" | "airflow") — there is no default. +ADF reads adf_volume_path | adf_workspace_path | adf_definitions | adf_source_path; Airflow reads +airflow_source_path (a DAG .py file or directory). + +Typical flow (ADF shown; swap source + source-path for Airflow): + flowx("inputs", {"phase": "discover", "source": "adf"}) # learn a phase's inputs + flowx("discover", {"source": "adf", "adf_source_path": "...", "output_dir": "..."}) + flowx("convert", {"source": "adf", "output_dir": "..."}) flowx("inspect", {"report_path": "/.work/translation_report.json"}) flowx("apply_answers", {"report_path": "...", "answers": ["id=value"], "output_dir": "..."}) flowx("package", {"output_dir": "...", "catalog": "main", "schema": "default"}) Or run it all at once: - flowx("migrate", {"adf_source_path": "...", "output_dir": "...", "catalog": "...", "schema": "..."}) + flowx("migrate", {"source": "airflow", "airflow_source_path": "...", "output_dir": "...", + "catalog": "...", "schema": "..."}) -All phases share one output_dir. Provide ADF source paths and output_dir as locations the server can +All phases share one output_dir. Provide source paths and output_dir as locations the server can read/write (a local path, or a Unity Catalog Volume path when the host has volume access). """ @@ -65,8 +70,8 @@ def _phase_result(result: runner.AdapterResult, output_dir: Path, **extra: Any) def _source_name(p: dict[str, Any]) -> str: - """The migration source for a command (``adf`` default; ``airflow`` when requested).""" - return str(p.get("source", "adf")) + """The migration source for a command; required (no default) -- ``KeyError`` when absent.""" + return str(p["source"]) def _resolve_source(p: dict[str, Any], path_key: str | None = None) -> tuple[str | None, Callable[[], None]]: @@ -79,7 +84,9 @@ def _resolve_source(p: dict[str, Any], path_key: str | None = None) -> tuple[str 2. ``adf_workspace_path`` — a ``/Workspace`` directory (e.g. an ADF Git folder); downloaded via the SDK Workspace API. Both (1) and (2) scale to large factories — the bytes bypass the agent. 3. ``adf_definitions`` — an inline ARM-JSON payload (small jobs); materialized to a temp dir. - 4. ``_source_path`` / explicit ``path_key`` — a path the server itself can read. + 4. ``_source_path`` (e.g. ``airflow_source_path``) or the explicit ``path_key`` — a path + the server itself can read. ``path_key`` is an *additional* key to try (e.g. ``source_dir``), + not a replacement, so the source's natural key still resolves. For ``source="airflow"`` the volume/workspace/inline modes are ADF-specific and skipped; the DAG path is read from ``airflow_source_path`` (or the explicit ``path_key``). @@ -96,8 +103,11 @@ def _resolve_source(p: dict[str, Any], path_key: str | None = None) -> tuple[str if definitions: src = runner.materialize_adf_definitions(definitions) return src, lambda: runner.cleanup_materialized(src) - default_key = path_key or f"{source}_source_path" - return p.get(default_key), (lambda: None) + candidate_keys = [f"{source}_source_path"] + if path_key: + candidate_keys.append(path_key) + resolved = next((p[key] for key in candidate_keys if p.get(key)), None) + return resolved, (lambda: None) def _bundle_output(p: dict[str, Any], out: Path) -> dict[str, Any]: @@ -138,7 +148,12 @@ def _pending_options(inspect_result: dict[str, Any]) -> list[dict[str, Any]]: def _cmd_inputs(p: dict[str, Any]) -> dict[str, Any]: - result = runner.run_adapter(["inputs", p["phase"], "--source", _source_name(p)]) + phase = p["phase"] + args = ["inputs", phase] + # package is source-independent; discover/convert prompts are source-specific (source required). + if phase != "package": + args += ["--source", _source_name(p)] + result = runner.run_adapter(args) return {"ok": result.ok, "inputs": runner.parse_stdout_json(result), "process": result.as_dict()} @@ -423,35 +438,43 @@ def build_server() -> FastMCP: # declare one); the dict is still returned as JSON text. See "MCP server design notes" in AGENTS.md. @mcp.tool(structured_output=False) def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, Any]: - """Run an flowx ADF→Databricks migration command. + """Run an flowx source→Databricks migration command (source: Azure Data Factory or Apache Airflow). Call as ``flowx(command="", parameters={...})``. Commands and their - ``parameters`` keys (req = required; phases share ``output_dir``, default "./flowx_output"): - - - "inputs": phase(req: "discover"|"convert"|"package") — list a phase's input prompts. - - "discover": one of adf_volume_path | adf_workspace_path | adf_definitions | adf_source_path - (req), output_dir, pipeline — parse ADF JSON, classify activities. - - "convert": output_dir, (adf_volume_path | adf_workspace_path | adf_definitions | - adf_source_path), pipeline. - - "merge_agentic": report_path(req), agentic_results_dir(req), output_path — merge agent results. + ``parameters`` keys (req = required; phases share ``output_dir``, default "./flowx_output"). + ``source`` ("adf" | "airflow") is **required** for discover/convert/migrate/inputs (and + workspace_paths); there is no default. It selects both the parser and which source-path key + applies: ADF reads adf_volume_path | adf_workspace_path | adf_definitions | adf_source_path, + Airflow reads ``airflow_source_path`` (a DAG .py file or directory). ``package`` is + source-independent (it consumes the translation report). + + - "inputs": phase(req: "discover"|"convert"|"package"), source(req for discover/convert) — + list a phase's input prompts. + - "discover": source(req), one ADF source key | airflow_source_path (req), output_dir, + pipeline — parse the source's definitions, classify activities. + - "convert": source(req), (one ADF source key | airflow_source_path), output_dir, pipeline. + - "merge_agentic": source(req), report_path(req), agentic_results_dir(req), output_path — + merge agent results. - "inspect": report_path(req) — return the full translation-option schema (every option with a `show_when` condition) for the agent to walk locally. See "Collecting options" below. - "apply_answers": report_path(req), answers(req, list of "ID=VALUE"), output_dir, lookup_csv. - "materialize_lookup": source(req: CSV path or literal CSV), out(req: destination JSON path). - - "workspace_paths": report_path(req), (adf_volume_path | adf_workspace_path | adf_definitions + - "workspace_paths": source(req), report_path(req), (one ADF source key | airflow_source_path | source_dir). - "package": output_dir, output_volume_path, output_workspace_path, report_path, catalog(default "main"), schema(default "default"), bundle_name, profile, download_workspace_files(bool), keep_intermediates(bool). - - "migrate": one of adf_volume_path | adf_workspace_path | adf_definitions | adf_source_path - (req), output_dir, output_volume_path, output_workspace_path, catalog, schema, pipeline, + - "migrate": source(req), one ADF source key | airflow_source_path (req), output_dir, + output_volume_path, output_workspace_path, catalog, schema, pipeline, answers(list of "ID=VALUE"), interactive(bool, default true), lookup_csv — runs discover→convert→package, returning the full option schema once (status "needs_input") when configuration is available; re-call once with the complete answers to apply (see below). - "record_results": output_dir(req), results_table(req: catalog.schema.table), warehouse_id. - "install_dashboard": results_table(req), warehouse_id, dashboard_name, parent_path. - Providing the ADF source (a hosted app can't read the user's workspace/volume files directly): + Providing the source (a hosted app can't read the user's workspace/volume files directly). For + ``source="airflow"`` pass ``airflow_source_path`` (a DAG .py file or directory the server can + read). For ``source="adf"``, in priority order: - ``adf_volume_path``: a UC Volume directory the server reads via the SDK Files API. **Preferred for large factories** — the bytes never pass through the agent. Requires the app's service principal to have read on the volume. diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 2b4f734..1ea7ef5 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -527,14 +527,14 @@ class TestMigrationInputSession: def test_discover_session_lists_expected_options(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="discover") + session = MigrationInputSession(phase="discover", source="adf") ids = [q.option_id for q in session.pending().options] assert ids == ["adf_source_path", "adf_resource_url", "output_dir"] def test_convert_session_lists_expected_options(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="convert") + session = MigrationInputSession(phase="convert", source="adf") ids = [q.option_id for q in session.pending().options] assert "inventory_path" in ids assert "adf_source_path" in ids @@ -555,7 +555,7 @@ def test_unknown_phase_raises(self): def test_answer_records_value_and_drops_from_pending(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="discover") + session = MigrationInputSession(phase="discover", source="adf") session.answer("adf_source_path", "/Volumes/main/default/adf") ids = [q.option_id for q in session.pending().options] assert "adf_source_path" not in ids @@ -563,7 +563,7 @@ def test_answer_records_value_and_drops_from_pending(self): def test_answer_rejects_unknown_option(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="discover") + session = MigrationInputSession(phase="discover", source="adf") with pytest.raises(ValueError, match="Unknown input option"): session.answer("not_a_field", "x") @@ -580,7 +580,7 @@ def test_collected_merges_answers_with_defaults(self): def test_collected_omits_required_when_missing(self): from flowx.adapter import MigrationInputSession - session = MigrationInputSession(phase="discover") + session = MigrationInputSession(phase="discover", source="adf") collected = session.collected() assert "adf_source_path" not in collected assert collected["output_dir"] == "./flowx_output" @@ -600,7 +600,7 @@ def test_workspace_paths_detects_notebook_paths(self, tmp_path: Path): report_path = tmp_path / "report.json" report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) out = tmp_path / "ws.json" - exit_code = adapter_cli_main(["workspace-paths", str(report_path), "--out", str(out)]) + exit_code = adapter_cli_main(["workspace-paths", str(report_path), "--source", "adf", "--out", str(out)]) assert exit_code == 0 payload = json.loads(out.read_text()) assert payload["paths"] == ["/Shared/team/a", "/Shared/team/b"] @@ -614,7 +614,7 @@ def test_workspace_paths_reports_no_auth_when_paths_empty(self, tmp_path: Path): report_path = tmp_path / "report.json" report_path.write_text(json.dumps(pipeline_to_dict(pipeline))) out = tmp_path / "ws.json" - adapter_cli_main(["workspace-paths", str(report_path), "--out", str(out)]) + adapter_cli_main(["workspace-paths", str(report_path), "--source", "adf", "--out", str(out)]) payload = json.loads(out.read_text()) assert payload["paths"] == [] assert payload["needs_auth"] is False @@ -642,20 +642,39 @@ def test_workspace_paths_suggests_host_from_databricks_linked_service(self, tmp_ json.dumps({"name": "LS_Other", "properties": {"type": "AzureSqlDatabase"}}) ) out = tmp_path / "ws.json" - adapter_cli_main(["workspace-paths", str(report_path), "--source-dir", str(source_dir), "--out", str(out)]) + adapter_cli_main( + ["workspace-paths", str(report_path), "--source", "adf", "--source-dir", str(source_dir), "--out", str(out)] + ) payload = json.loads(out.read_text()) assert payload["suggested_hosts"] == ["https://adb-1234.5.azuredatabricks.net"] class TestInputsCli: def test_inputs_emits_discover_options(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): - exit_code = adapter_cli_main(["inputs", "discover"]) + exit_code = adapter_cli_main(["inputs", "discover", "--source", "adf"]) assert exit_code == 0 payload = json.loads(capsys.readouterr().out) assert payload["phase"] == "discover" ids = [q["option_id"] for q in payload["options"]] assert ids == ["adf_source_path", "adf_resource_url", "output_dir"] + def test_inputs_requires_source_for_discover(self, capsys: pytest.CaptureFixture[str]): + # discover prompts are source-specific; no default source -> clear error, exit 2. + exit_code = adapter_cli_main(["inputs", "discover"]) + assert exit_code == 2 + assert "source" in capsys.readouterr().err.lower() + + def test_inputs_rejects_unknown_source(self, capsys: pytest.CaptureFixture[str]): + # An unrecognised source is a clean usage error (exit 2), not an uncaught ValueError traceback. + exit_code = adapter_cli_main(["inputs", "discover", "--source", "typo"]) + assert exit_code == 2 + assert "not recognized" in capsys.readouterr().err.lower() + + def test_inputs_package_ignores_missing_source(self, capsys: pytest.CaptureFixture[str]): + # package is source-independent: no --source needed, and it succeeds. + exit_code = adapter_cli_main(["inputs", "package"]) + assert exit_code == 0 + def test_inputs_writes_to_file(self, tmp_path: Path): out = tmp_path / "options.json" exit_code = adapter_cli_main(["inputs", "package", "--out", str(out)]) diff --git a/tests/unit/test_airflow_adapter_reporting.py b/tests/unit/test_airflow_adapter_reporting.py index f101b80..44b5e27 100644 --- a/tests/unit/test_airflow_adapter_reporting.py +++ b/tests/unit/test_airflow_adapter_reporting.py @@ -5,6 +5,8 @@ import tempfile from pathlib import Path +import pytest + from flowx.adapter.session import MigrationInputSession from flowx.reporting.coverage import COVERAGE_METRIC_COLUMNS, build_coverage_rows from flowx.sources.airflow.discover import main as discover_main @@ -31,10 +33,11 @@ def test_inputs_convert_airflow_uses_airflow_source_path(): assert "adf_source_path" not in ids -def test_inputs_default_source_is_adf(): - # Back-compat: no source arg -> ADF prompts. +def test_inputs_discover_requires_source(): + # No default source: discover prompts can't be worded without one, so pending() raises. session = MigrationInputSession(phase="discover") - assert any(o.option_id == "adf_source_path" for o in session.pending().options) + with pytest.raises(ValueError, match="source is required"): + session.pending() def test_airflow_profile_csv_has_all_coverage_columns(): diff --git a/tests/unit/test_mcp_migrate.py b/tests/unit/test_mcp_migrate.py index cf38589..bc005bb 100644 --- a/tests/unit/test_mcp_migrate.py +++ b/tests/unit/test_mcp_migrate.py @@ -79,7 +79,9 @@ def fake_run_adapter(args): def test_first_call_returns_full_schema_without_packaging(stub_adapter, tmp_path: Path): - result = server._cmd_migrate({"adf_source_path": str(tmp_path / "adf"), "output_dir": str(tmp_path / "out")}) + result = server._cmd_migrate( + {"source": "adf", "adf_source_path": str(tmp_path / "adf"), "output_dir": str(tmp_path / "out")} + ) assert result["status"] == "needs_input" # The whole tree (including the conditional slack follow-up) is returned up front. option_ids = {o["option_id"] for pipe in result["pending_options"] for o in pipe["options"]} @@ -98,6 +100,7 @@ def test_resume_with_answers_applies_and_packages_once(stub_adapter, tmp_path: P result = server._cmd_migrate( { + "source": "adf", "adf_source_path": str(tmp_path / "adf"), "output_dir": str(out), "answers": ["notify_destination=slack", "notify_slack_url=https://hooks.slack.com/x"], @@ -111,7 +114,12 @@ def test_resume_with_answers_applies_and_packages_once(stub_adapter, tmp_path: P def test_interactive_false_skips_prompt_and_packages(stub_adapter, tmp_path: Path): result = server._cmd_migrate( - {"adf_source_path": str(tmp_path / "adf"), "output_dir": str(tmp_path / "out"), "interactive": False} + { + "source": "adf", + "adf_source_path": str(tmp_path / "adf"), + "output_dir": str(tmp_path / "out"), + "interactive": False, + } ) assert result["status"] == "completed" assert stub_adapter == ["discover", "convert", "package"] # no inspect, no pause diff --git a/tests/unit/test_mcp_source_routing.py b/tests/unit/test_mcp_source_routing.py index ff3f03a..ce8fa3e 100644 --- a/tests/unit/test_mcp_source_routing.py +++ b/tests/unit/test_mcp_source_routing.py @@ -43,13 +43,30 @@ def _argv(calls: list[list[str]], subcommand: str) -> list[str]: return next(argv for argv in calls if argv and argv[0] == subcommand) -def test_discover_defaults_to_adf_source(captured, tmp_path: Path): - server._cmd_discover({"adf_source_path": str(tmp_path), "output_dir": str(tmp_path / "o")}) +def test_discover_threads_explicit_adf_source(captured, tmp_path: Path): + server._cmd_discover({"source": "adf", "adf_source_path": str(tmp_path), "output_dir": str(tmp_path / "o")}) argv = _argv(captured, "discover") assert "--source" in argv and argv[argv.index("--source") + 1] == "adf" assert "--source-path" in argv +def test_discover_requires_source(tmp_path: Path): + # No default source: a command with no 'source' raises KeyError, which the dispatcher (below) + # converts into a clear "Missing required parameter 'source'" error instead of assuming adf. + with pytest.raises(KeyError): + server._cmd_discover({"adf_source_path": str(tmp_path), "output_dir": str(tmp_path / "o")}) + + +def test_dispatcher_reports_missing_source_clearly(tmp_path: Path): + handler = server._COMMANDS["discover"] + try: + result = handler({"adf_source_path": str(tmp_path), "output_dir": str(tmp_path / "o")}) + except KeyError as missing: + result = {"ok": False, "error": f"Missing required parameter {missing} for command 'discover'."} + assert result["ok"] is False + assert "source" in result["error"].lower() + + def test_discover_routes_airflow_source(captured, tmp_path: Path): server._cmd_discover({"source": "airflow", "airflow_source_path": str(tmp_path), "output_dir": str(tmp_path / "o")}) argv = _argv(captured, "discover") @@ -81,6 +98,22 @@ def test_inputs_threads_source(captured): assert argv[argv.index("--source") + 1] == "airflow" +def test_inputs_package_is_source_independent(captured): + # package prompts don't vary by source, so `inputs package` must not require (or pass) --source. + server._cmd_inputs({"phase": "package"}) + argv = _argv(captured, "inputs") + assert "--source" not in argv + + +def test_workspace_paths_forwards_airflow_source_path(captured, tmp_path: Path): + server._cmd_workspace_paths( + {"source": "airflow", "report_path": "/tmp/report.json", "airflow_source_path": str(tmp_path)} + ) + argv = _argv(captured, "workspace-paths") + assert argv[argv.index("--source") + 1] == "airflow" + assert argv[argv.index("--source-dir") + 1] == str(tmp_path) + + def test_discover_missing_source_path_errors_clearly(captured, tmp_path: Path): result = server._cmd_discover({"source": "airflow", "output_dir": str(tmp_path)}) assert result["ok"] is False From bb657cb6dea214392ae65961551c5e1eea3e01ab Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Tue, 21 Jul 2026 11:12:48 -0700 Subject: [PATCH 35/77] Translate TaskFlow .expand()/@task_group and tighten source validation - @task.expand(param=[literal list]) lowers to a for_each_task (the inner callable notebook reads each element from the per-iteration item widget); a non-literal iterable, .partial(...).expand(...), or .expand_kwargs([...]) routes to a placeholder + gap for the agentic round instead of a silent single-run notebook or a silent drop - a mapped iterable from an upstream task (process.expand(x=vals)) keeps its vals->process dependency edge whether it lowers to a for_each or a placeholder - @task_group invocations (mapped or plain) route to a placeholder + gap with dependency edges preserved, instead of silently dropping the whole group - workspace-paths rejects an unknown --source (exit 2); MCP _source_name rejects a non-string source instead of coercing it - update the coverage doc's .expand/TaskGroup rows and follow-ups --- .../flowx-convert/sources/airflow-coverage.md | 15 +- src/flowx/adapter/__main__.py | 13 +- src/flowx/mcp/server.py | 11 +- src/flowx/preparer/workflow_preparer.py | 18 +- src/flowx/sources/airflow/loader.py | 237 +++++++++++++++++- tests/unit/test_adapter.py | 9 + tests/unit/test_airflow_operators.py | 183 ++++++++++++++ tests/unit/test_mcp_source_routing.py | 6 + 8 files changed, 460 insertions(+), 32 deletions(-) diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md index a2ce62d..acee272 100644 --- a/skills/flowx-convert/sources/airflow-coverage.md +++ b/skills/flowx-convert/sources/airflow-coverage.md @@ -28,9 +28,10 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't | `HttpSensor` / `PythonSensor` / `DateTimeSensor` | Polling notebook tasks for absolute HTTP URLs, callable polls, and wait-until. Relative HTTP endpoints and Python callables reading task context route to placeholders. | | Time sensors (`TimeSensor`, `TimeDeltaSensor`) | Placeholder; their per-run wait semantics are not silently folded into or removed from the job schedule. | | `DummyOperator` / `EmptyOperator` | Dropped, downstream dependencies rewired. | -| `.expand()` on an operator | `for_each_task`. | +| `.expand()` on an operator or `@task` | `for_each_task` when the mapped iterable is a literal list; a non-literal iterable (e.g. an upstream task's output) routes to a placeholder + gap. | | Dependencies | `>>` / `<<` chains (incl. list/tuple fan-out and inline TaskFlow calls) and `set_upstream` / `set_downstream`. | -| **TaskGroups** | Static nesting → task-key namespacing (`group__subgroup__task`); group-level edges (`group_a >> group_b`, `task >> group`) expand to leaf→root edges between member tasks. | +| **TaskGroups** (context-manager `with TaskGroup(...)`) | Static nesting → task-key namespacing (`group__subgroup__task`); group-level edges (`group_a >> group_b`, `task >> group`) expand to leaf→root edges between member tasks. | +| **`@task_group`** (decorator form) | Placeholder + gap (with dependency edges preserved); a decorator group is a sub-pipeline flowx doesn't lower — the agentic round expands it into its member tasks / a for_each when mapped. | | Schedule | Cron `schedule_interval` → Quartz (Unix DOW 0–6 → Quartz 1–7); exact sub-hour `timedelta` → Quartz, longer intervals → periodic, `@continuous` → continuous mode. | | `trigger_rule` | DAB `run_if` constant per edge (`ALL_DONE`, `ALL_FAILED`, `AT_LEAST_ONE_SUCCESS`, `NONE_FAILED`, …). | | Job parameters | `params={...}` / `Param(default=...)` → job parameters with defaults; `{{ params.x }}` / `{{ var.value.x }}` / `{{ dag_run.conf['x'] }}` → `{{job.parameters.x}}`. | @@ -48,8 +49,10 @@ emitting code that fails at runtime. These are absent but fail safely — routed to a placeholder + `gaps.json`, or simply not exploded — or are deliberate scope decisions. -- **Dynamic TaskGroup mapping** (`.expand()` on a `@task_group` or `TaskGroup.partial().expand()`). - `.expand()` is only recognized on operator/`@task` calls; a mapped *group's* fan-out is lost. +- **Full TaskGroup expansion** — a `@task_group` invocation (mapped `pair.expand(...)` or plain + `pair(...)`) and `TaskGroup.partial().expand()` aren't lowered into their member tasks. They route + to a placeholder + gap with dependency edges preserved (never a silent drop); the agentic round + expands the group. `.expand()` on an operator/`@task` *call* is supported (see the table). - **Sensors beyond the mapped families** (`S3PrefixSensor`, custom sensors, etc.) → placeholder + gap. A file sensor with a non-literal path, or a table/SQL sensor with no literal `sql` / `table_name`, also falls back to a placeholder. @@ -76,6 +79,6 @@ mode with `--dbt-mode {static,pydabs}` on the convert phase (default `static`). ## Priority for remaining follow-ups -1. **Dynamic TaskGroup mapping** — expand a mapped `@task_group` / `TaskGroup.partial().expand()` into - a for-each over the group's tasks (today the fan-out is lost). +1. **Full TaskGroup expansion** — lower a `@task_group` / `TaskGroup.partial().expand()` into its + member tasks (a for-each over the group when mapped) instead of a placeholder. 2. **Additional sensor families** — as demand warrants; unmapped sensors route to a placeholder today. diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py index 2449a9d..e9957e5 100644 --- a/src/flowx/adapter/__main__.py +++ b/src/flowx/adapter/__main__.py @@ -145,10 +145,17 @@ def _run_workspace_paths(args: argparse.Namespace) -> int: and ``out``. Returns: - ``0`` on success. The command always succeeds when the report - can be read; missing or unreadable inputs simply produce empty - path / host lists so the skill can detect the no-op case. + ``0`` on success, or ``2`` when ``--source`` names an unknown source. + Otherwise the command succeeds when the report can be read; missing or + unreadable inputs simply produce empty path / host lists so the skill + can detect the no-op case. """ + if args.source not in available_sources(): + print( + f"--source {args.source!r} is not recognized; choose one of: {', '.join(available_sources())}", + file=sys.stderr, + ) + return 2 paths = collect_workspace_artifact_paths(args.report) suggested_hosts: list[str] = [] if args.source_dir: diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py index 8c238e5..450881d 100644 --- a/src/flowx/mcp/server.py +++ b/src/flowx/mcp/server.py @@ -70,8 +70,15 @@ def _phase_result(result: runner.AdapterResult, output_dir: Path, **extra: Any) def _source_name(p: dict[str, Any]) -> str: - """The migration source for a command; required (no default) -- ``KeyError`` when absent.""" - return str(p["source"]) + """The migration source for a command; required as a string (no default). + + Raises ``KeyError`` when absent and ``ValueError`` when non-string; the dispatcher + surfaces both as a clear error rather than coercing e.g. ``123`` to ``"123"``. + """ + source = p["source"] + if not isinstance(source, str): + raise ValueError(f"'source' must be a string, got {type(source).__name__}") + return source def _resolve_source(p: dict[str, Any], path_key: str | None = None) -> tuple[str | None, Callable[[], None]]: diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py index aebc581..e94adad 100644 --- a/src/flowx/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -249,19 +249,19 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: notebook_name = f"{activity.task_key}.py" notebook_path = f"notebooks/{notebook_name}" - # When the activity is an agentic gap (e.g. Until), embed its full ADF/ARM - # JSON so the agentic handler can translate it directly from source. - arm_block = "" + # When the activity is an agentic gap, embed its raw source definition (ADF/ARM JSON for the ADF + # source, the operator source for Airflow) so the agentic handler can translate it directly. + source_block = "" if raw_definition is not None: import json as _json - arm_lines = _json.dumps(raw_definition, indent=2).splitlines() - arm_block = ( + source_lines = _json.dumps(raw_definition, indent=2).splitlines() + source_block = ( "# MAGIC\n" - "# MAGIC An agent should translate this activity from the ADF/ARM JSON below,\n" + "# MAGIC An agent should translate this activity from the source definition below,\n" "# MAGIC then replace the `raise NotImplementedError` cell with the generated code.\n" "# MAGIC\n" - "# MAGIC ```json\n" + "".join(f"# MAGIC {line}\n" for line in arm_lines) + "# MAGIC ```\n" + "# MAGIC ```json\n" + "".join(f"# MAGIC {line}\n" for line in source_lines) + "# MAGIC ```\n" ) content = ( @@ -269,9 +269,9 @@ def _prepare_placeholder(activity: Activity) -> PreparedActivity: "# MAGIC %md\n" f"# MAGIC # Placeholder: {activity.name}\n" "# MAGIC\n" - f"# MAGIC Original ADF activity type: **{original_type}**\n" + f"# MAGIC Original source activity type: **{original_type}**\n" "# MAGIC\n" - f"# MAGIC {comment}\n" + arm_block + "\n# COMMAND ----------\n\n" + f"# MAGIC {comment}\n" + source_block + "\n# COMMAND ----------\n\n" f"raise NotImplementedError(\"Activity '{activity.name}' ({original_type}) needs agentic translation.\")\n" ) diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py index c671d70..1a341e8 100644 --- a/src/flowx/sources/airflow/loader.py +++ b/src/flowx/sources/airflow/loader.py @@ -32,7 +32,9 @@ DbtFactoryActivity, Dependency, ForEachActivity, + NotebookActivity, Pipeline, + PlaceholderActivity, SqlActivity, ) from flowx.sources.airflow import callable_notebook, templating @@ -47,6 +49,11 @@ class _TaskFlowTask: invoked with to the upstream task var it references (TaskFlow's implicit XCom data flow), so the emitted notebook can read that upstream's return value via ``dbutils.jobs.taskValues``. Literal args are preserved when literal and routed to a placeholder when they cannot be resolved safely. + + ``.expand(param=)`` dynamic mapping is captured in ``expand_kwarg`` (the mapped + parameter name) and ``expand_items_json`` (the iterable as a JSON-array literal) when the + iterable is statically knowable; a non-literal iterable leaves ``expand_items_json`` None and + routes the task to the agentic-gap round. """ task_id: str @@ -57,6 +64,8 @@ class _TaskFlowTask: positional_values: dict[int, str] = field(default_factory=dict) keyword_values: dict[str, str] = field(default_factory=dict) unresolved_arguments: list[str] = field(default_factory=list) + expand_kwarg: str | None = None + expand_items_json: str | None = None def _sanitize_task_key(name: str) -> str: @@ -288,15 +297,23 @@ def __init__(self, module: ast.Module) -> None: # TaskFlow: function name -> (FunctionDef, decorator dotted-name) for @task-decorated defs. # Pre-scanned so a @task def defined after the @dag body that uses it is still resolved. self.taskflow_defs: dict[str, tuple[ast.FunctionDef, str]] = {} + # @task_group def names -- a group is a sub-pipeline, not a single renderable task, so an + # invocation routes to a placeholder + gap rather than being expanded here. + self.taskgroup_defs: set[str] = set() for fn in _iter_functions(module): decorator = next( (_decorator_name(d) for d in fn.decorator_list if _decorator_name(d) in _TASK_DECORATORS), None ) if decorator is not None: self.taskflow_defs[fn.name] = (fn, decorator) + elif _has_decorator(fn, _TASK_GROUP_DECORATORS): + self.taskgroup_defs.add(fn.name) # TaskFlow task instances: var name -> _TaskFlowTask (id, def-name, decorator, arg bindings). self.taskflow_tasks: dict[str, _TaskFlowTask] = {} + # @task_group invocations: var name -> (task_id, def-name, is_mapped). + self.taskgroup_calls: dict[str, tuple[str, str, bool]] = {} self._taskflow_counter = 0 + self._taskgroup_counter = 0 # A @dag-decorated function was found (so a bare `@task` file is still recognized as a DAG). self.is_taskflow_dag: bool = False @@ -304,10 +321,11 @@ def functions(self) -> dict[str, ast.FunctionDef]: return self._functions def visit_FunctionDef(self, node: ast.FunctionDef) -> None: - # A @task-decorated function defines a task from its callable; its body is task logic, not DAG - # structure, so don't descend. @dag marks the DAG-defining function: read its config off the - # decorator, then descend so the body's task instances / edges are collected. - if _has_decorator(node, _TASK_DECORATORS): + # A @task- or @task_group-decorated function defines a task / sub-pipeline from its body, + # which is internal logic rather than DAG structure, so don't descend. @dag marks the + # DAG-defining function: read its config off the decorator, then descend so the body's task + # instances / edges are collected. + if _has_decorator(node, _TASK_DECORATORS) or _has_decorator(node, _TASK_GROUP_DECORATORS): return if _has_decorator(node, _DAG_DECORATORS): self.is_taskflow_dag = True @@ -335,27 +353,35 @@ def visit_Assign(self, node: ast.Assign) -> None: self.groups[var] = "__".join(self._group_stack) elif self._register_taskflow_call(node.value, var): pass # a `x = mytask(...)` TaskFlow invocation, captured with var as its key + else: + self._register_taskgroup_call(node.value, var) # a `x = mygroup(...)` @task_group call self.generic_visit(node) def _taskflow_def_name(self, call: ast.Call) -> tuple[str | None, bool, str | None]: - """Resolves a call's underlying ``@task`` def name, unwrapping ``.expand`` / ``.override``. + """Resolves a call's underlying ``@task`` def name, unwrapping the mapping/config chain. - Returns ``(def_name_or_None, is_mapped, override_task_id)``. + Handles ``.expand(...)`` / ``.expand_kwargs(...)`` (both set ``is_mapped``) and the + ``.override(...)`` / ``.partial(...)`` config calls, in any order, so forms like + ``op.partial(...).expand(...)`` resolve. Returns ``(def_name_or_None, is_mapped, override_id)``. """ func = call.func mapped = False override_id: str | None = None while True: if isinstance(func, ast.Attribute): - if func.attr == "expand": + if func.attr in ("expand", "expand_kwargs"): mapped = True func = func.value continue if isinstance(func, ast.Call) and isinstance(func.func, ast.Attribute): - if func.func.attr == "override": + config_call = func.func + if config_call.attr == "override": arguments = {keyword.arg: keyword.value for keyword in func.keywords if keyword.arg} override_id = ops.literal_str(arguments.get("task_id")) - func = func.func.value + func = config_call.value + continue + if config_call.attr == "partial": + func = config_call.value continue break if isinstance(func, ast.Name) and func.id in self.taskflow_defs: @@ -379,8 +405,21 @@ def _register_taskflow_call(self, call: ast.Call, var: str) -> bool: self.calls[var] = call if mapped: self.mapped.add(var) + # ``.expand(param=)`` args live on the outer call; capture the single mapped + # parameter + its literal iterable (Tier 1). A non-literal iterable leaves items None, + # which routes the task to the agentic-gap round in _build_taskflow_task. + self._capture_expand(task, call) + # A mapped iterable OR a .partial(...) fixed arg can be an upstream task's output + # (``process.partial(x=raw).expand(y=vals)``); wire those data-flow edges so the mapped + # task still depends on its producers, whether it lowers to a for_each or a placeholder. + for mapped_arg in _mapping_chain_args(call): + dep = self._resolve_taskflow_arg(mapped_arg) + if dep is not None and dep != var: + self.edges.append((dep, var)) if self._group_stack: self.groups[var] = "__".join(self._group_stack) + if mapped: + return True # Bind each arg that resolves to an upstream task var, and add the data-flow edge. for index, arg in enumerate(call.args): dep = self._resolve_taskflow_arg(arg) @@ -409,6 +448,57 @@ def _register_taskflow_call(self, call: ast.Call, var: str) -> bool: task.keyword_values[kw.arg] = value return True + def _capture_expand(self, task: _TaskFlowTask, call: ast.Call) -> None: + """Captures a ``@task.expand(param=)`` mapping onto *task*. + + Tier 1 (deterministic -> for_each_task): a plain ``.expand(...)`` with exactly one mapped + parameter whose iterable is a literal list, and no ``.partial(...)`` fixed args (a for_each + inner task can't carry them). Anything else -- ``.expand_kwargs``, multiple mapped params, a + ``.partial(...).expand(...)`` chain, or a non-literal iterable -- leaves ``expand_items_json`` + None so _build_taskflow_task routes the task to the agentic-gap round. + """ + if not (isinstance(call.func, ast.Attribute) and call.func.attr == "expand"): + return # .expand_kwargs(...) or other mapping form -> not Tier 1 + if _has_partial_call(call.func.value): + return # .partial(...) fixed args can't be represented on a for_each inner task + keywords = [kw for kw in call.keywords if kw.arg] + if len(keywords) != 1 or len(keywords) != len(call.keywords): + return # 0 / multiple mapped params, or **expand_kwargs -> not Tier 1 + keyword = keywords[0] + task.expand_kwarg = keyword.arg + value = ops.literal_value(keyword.value) + if isinstance(value, list): + # Encode each element as its own JSON text, so the for_each `inputs` is a list of JSON + # strings and the inner notebook's json.loads unambiguously recovers the original value. + # (A bare list like [1, 2, 3] would make `{{input}}` deliver "1"/"2"/"3" -- indistinguishable + # from the string elements ["1", "2", "3"]; wrapping each element removes that ambiguity.) + task.expand_items_json = json.dumps([json.dumps(element) for element in value]) + + def _register_taskgroup_call(self, call: ast.Call, var: str | None) -> bool: + """Records a ``@task_group`` invocation (``pair(...)`` / ``pair.expand(...)``) as a placeholder. + + A ``@task_group`` is a sub-pipeline of tasks, not a single renderable callable, so it can't be + mechanically lowered here -- it's captured (keyed by *var*, or a synthetic name for a bare + call) so an edge to/from it resolves, and emitted as a placeholder + gap for the agentic round. + Returns True when the call resolved to a known group def. + """ + func = call.func + mapped = False + while isinstance(func, ast.Attribute): + if func.attr == "expand": + mapped = True + func = func.value + if not (isinstance(func, ast.Name) and func.id in self.taskgroup_defs): + return False + def_name = func.id + if var is None: + self._taskgroup_counter += 1 + var = f"{def_name}__tg{self._taskgroup_counter}" + self.taskgroup_calls[var] = (var, def_name, mapped) + if self._group_stack: + self.groups[var] = "__".join(self._group_stack) + return True + def _resolve_taskflow_arg(self, arg: ast.expr) -> str | None: """Returns the upstream task var an argument refers to, else None (a literal / unknown). @@ -503,7 +593,7 @@ def visit_Expr(self, node: ast.Expr) -> None: self._taskflow_counter += 1 task_var = f"{def_name}__tf{self._taskflow_counter}" self._register_taskflow_call(value, task_var) - else: + elif not self._register_taskgroup_call(value, None): self._collect_set_dependency(value) self.generic_visit(node) @@ -605,6 +695,39 @@ def _resolve(var: str, *, as_upstream: bool) -> list[str]: return expanded +def _has_partial_call(node: ast.expr) -> bool: + """True when a ``@task`` mapping chain contains a ``.partial(...)`` config call.""" + current: ast.expr = node + while True: + if isinstance(current, ast.Call): + if isinstance(current.func, ast.Attribute) and current.func.attr == "partial": + return True + current = current.func + elif isinstance(current, ast.Attribute): + current = current.value + else: + return False + + +def _mapping_chain_args(node: ast.expr) -> list[ast.expr]: + """Every argument expression across a ``@task`` mapping chain's call nodes. + + Walks ``op.partial(x=up).expand(y=vals)`` (and ``.override(...)``), collecting the args of every + ``.partial`` / ``.expand`` / ``.expand_kwargs`` call so upstream-task references in either the + fixed args or the mapped iterable are found for data-flow edge wiring. + """ + args: list[ast.expr] = [] + current: ast.expr = node + while isinstance(current, (ast.Call, ast.Attribute)): + if isinstance(current, ast.Call): + args.extend(current.args) + args.extend(kw.value for kw in current.keywords) + current = current.func + else: + current = current.value + return args + + def _iter_functions(module: ast.Module) -> list[ast.FunctionDef]: """All FunctionDefs in *module*, including those nested inside a ``@dag`` function body. @@ -727,6 +850,7 @@ def _decorator_kwargs(decorators: list[ast.expr], names: frozenset[str]) -> dict _TASK_DECORATORS: frozenset[str] = frozenset( {"task", "task.branch", "task.virtualenv", "task.short_circuit", "task.sensor", "task.external_python"} ) +_TASK_GROUP_DECORATORS: frozenset[str] = frozenset({"task_group"}) def _has_decorator(func: ast.FunctionDef, names: frozenset[str]) -> bool: @@ -813,6 +937,7 @@ def _task_key(var: str, task_id: str) -> str: # a task_key and dependency edges downstream). var_task_ids: dict[str, str] = {var: tid for var, (tid, _, _) in visitor.operators.items()} var_task_ids.update({var: tf.task_id for var, tf in visitor.taskflow_tasks.items()}) + var_task_ids.update({var: task_id for var, (task_id, _, _) in visitor.taskgroup_calls.items()}) var_to_task_key = {var: _task_key(var, tid) for var, tid in var_task_ids.items()} # Expand group-level edges (`group_a >> group_b`, `task >> group`, ...) into edges between the @@ -941,10 +1066,65 @@ def _dep(upstream_var: str, outcome: str | None) -> str: dep_keys = {var_to_task_key[u] for u in upstreams.get(var, []) if u in var_to_task_key} dep_keys.discard(task_key) depends_on = [Dependency(task_key=k) for k in sorted(dep_keys)] or None + if var in visitor.mapped and tf.expand_items_json is None: + # .expand over a non-literal iterable (e.g. an upstream task's output) can't be lowered to + # a static for_each inputs array -- route to the agentic-gap round instead of silently + # emitting a single-run notebook. + reason = f"mapped parameter {tf.expand_kwarg!r}" if tf.expand_kwarg else "multiple mapped parameters" + func = functions.get(tf.def_name) + placeholder = PlaceholderActivity( + name=tf.task_id, + task_key=task_key, + original_type=f"@{tf.decorator}.expand", + comment=( + f"TaskFlow @{tf.decorator} '{tf.def_name}'.expand() maps over a non-literal iterable " + f"({reason}); translate to a Databricks for_each_task whose inputs reference the " + "upstream task value, iterating the callable." + ), + raw_definition={ + "operator": f"@{tf.decorator}.expand", + "source": ast.get_source_segment(source, func) if func is not None else "", + }, + ) + placeholder.depends_on = depends_on + tasks.append(placeholder) + continue activity = _build_taskflow_task(tf, var_to_task_key, functions, source, task_key) activity.depends_on = depends_on referenced_params |= _convert_activity_templates(activity) - tasks.append(activity) + if var in visitor.mapped and isinstance(activity, NotebookActivity): + # .expand(param=[literal list]) -> a for_each_task iterating the callable notebook; the + # inner notebook reads the mapped parameter from the per-iteration `item` widget. + tasks.append(_wrap_taskflow_in_for_each(activity, tf, task_key, depends_on)) + else: + tasks.append(activity) + + # @task_group invocations: a group is a sub-pipeline of tasks with no single-task lowering, so + # emit a placeholder + gap (never silently drop the whole group) for the agentic round to expand. + for var, (task_id, def_name, mapped) in visitor.taskgroup_calls.items(): + task_key = var_to_task_key[var] + dep_keys = {var_to_task_key[u] for u in upstreams.get(var, []) if u in var_to_task_key} + dep_keys.discard(task_key) + depends_on = [Dependency(task_key=k) for k in sorted(dep_keys)] or None + group_func = functions.get(def_name) + detail = ( + "maps the group over an iterable (one group run per element); translate to a for_each_task " + "whose inner task expands the group's tasks" + if mapped + else "bundles multiple tasks; expand it into its member tasks with their dependencies" + ) + placeholder = PlaceholderActivity( + name=task_id, + task_key=task_key, + original_type="@task_group", + comment=f"Airflow @task_group '{def_name}' {detail}. flowx does not lower task groups.", + raw_definition={ + "operator": "@task_group", + "source": ast.get_source_segment(source, group_func) if group_func is not None else "", + }, + ) + placeholder.depends_on = depends_on + tasks.append(placeholder) # Declare every job parameter -- those referenced in templates plus any from the DAG's # params={...} -- each with a default (Databricks requires one): the params={...} default when @@ -999,6 +1179,29 @@ def _wrap_in_for_each( ) +def _wrap_taskflow_in_for_each( + activity: NotebookActivity, + tf: _TaskFlowTask, + task_key: str, + depends_on: list[Dependency] | None, +) -> ForEachActivity: + """Wraps a mapped ``@task.expand(param=[...])`` notebook in a ForEachActivity (-> for_each_task). + + The literal iterable becomes the for_each ``inputs`` array; the preparer injects each element as + the inner task's ``item`` widget, which the callable notebook reads for the mapped parameter. + """ + activity.task_key = f"{task_key}_iteration" + activity.name = f"{tf.task_id}_iteration" + activity.depends_on = None + return ForEachActivity( + name=tf.task_id, + task_key=task_key, + depends_on=depends_on, + items_expression=tf.expand_items_json or "[]", + inner_activities=[activity], + ) + + # TaskFlow decorators that gate downstream tasks at runtime -- can't lower to a notebook (same # reason BranchPythonOperator/ShortCircuitOperator route to the agentic round). _TASKFLOW_BRANCHING = frozenset({"task.branch", "task.short_circuit"}) @@ -1018,7 +1221,6 @@ def _build_taskflow_task( those bound arguments, and publishes its own return value. Callables that read Airflow task context/XCom, or use a branching decorator, route to a placeholder for the agentic round. """ - from flowx.models.ir import NotebookActivity, PlaceholderActivity func = functions.get(tf.def_name) if func is None: @@ -1099,6 +1301,17 @@ def _reader(dep_var: str) -> str: lines.append(f"{variable} = {_reader(dep_var)}") call_keywords.append(f"{name}={variable}") call_keywords.extend(f"{name}={value}" for name, value in tf.keyword_values.items()) + if tf.expand_kwarg is not None: + # .expand(param=[...]) fan-out: each for_each `inputs` element is the JSON text of the + # original value (see _capture_expand), so json.loads on the injected `item` widget recovers + # it exactly -- ints stay ints and JSON-looking strings stay strings. The except is a defensive + # fallback for an unexpected raw value. + lines.append("_raw_item = dbutils.widgets.get('item')") + lines.append("try:") + lines.append(" _expand_item = json.loads(_raw_item)") + lines.append("except (ValueError, TypeError):") + lines.append(" _expand_item = _raw_item") + call_keywords.append(f"{tf.expand_kwarg}=_expand_item") call_args = ", ".join(call_positional + call_keywords) returns = any(isinstance(n, ast.Return) and n.value is not None for n in ast.walk(func)) diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 1ea7ef5..8f57a0c 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -648,6 +648,15 @@ def test_workspace_paths_suggests_host_from_databricks_linked_service(self, tmp_ payload = json.loads(out.read_text()) assert payload["suggested_hosts"] == ["https://adb-1234.5.azuredatabricks.net"] + def test_workspace_paths_rejects_unknown_source(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): + from flowx.ir_serde import pipeline_to_dict + + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(pipeline_to_dict(Pipeline(name="p", tasks=[_delta_copy()])))) + exit_code = adapter_cli_main(["workspace-paths", str(report_path), "--source", "typo"]) + assert exit_code == 2 + assert "not recognized" in capsys.readouterr().err.lower() + class TestInputsCli: def test_inputs_emits_discover_options(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py index 36d9a50..5610251 100644 --- a/tests/unit/test_airflow_operators.py +++ b/tests/unit/test_airflow_operators.py @@ -1415,6 +1415,189 @@ def test_taskflow_mixed_with_classic_operator(): assert finalize.depends_on[0].task_key == "prep" +def test_taskflow_expand_literal_list_becomes_for_each(): + # @task.expand over a literal list -> for_each_task; the inner notebook reads the per-iteration + # element from the `item` widget (Tier 1, deterministic). + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def process(item):\n return item * 2\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " process.expand(item=[1, 2, 3])\n" + "pipeline()\n" + ) + task = next(t for t in p.tasks if t.task_key.startswith("process")) + assert isinstance(task, ForEachActivity) + # Each element is JSON-encoded individually so the inner notebook's json.loads recovers the exact + # value (ints stay ints, JSON-looking strings stay strings) regardless of {{input}} serialization. + assert task.items_expression == '["1", "2", "3"]' + inner = task.inner_activities[0] + assert isinstance(inner, NotebookActivity) + assert "dbutils.widgets.get('item')" in inner.generated_source + assert "item=_expand_item" in inner.generated_source + compile(inner.generated_source, "", "exec") + + +def test_taskflow_expand_dict_list_becomes_for_each(): + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def process(cfg):\n return cfg\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " process.expand(cfg=[{'a': 1}, {'a': 2}])\n" + "pipeline()\n" + ) + task = next(t for t in p.tasks if t.task_key.startswith("process")) + assert isinstance(task, ForEachActivity) + # Elements are individually JSON-encoded (each is the JSON text of the dict). + assert task.items_expression == '["{\\"a\\": 1}", "{\\"a\\": 2}"]' + + +def test_taskflow_expand_string_elements_round_trip_as_strings(): + # Regression: a list of JSON-looking strings must stay strings, not decode to int/bool/dict. + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def process(item):\n return item\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " process.expand(item=['123', 'true'])\n" + "pipeline()\n" + ) + import json + + task = next(t for t in p.tasks if t.task_key.startswith("process")) + assert isinstance(task, ForEachActivity) + # Simulate the runtime: each inputs element's content is fed to the notebook's json.loads. + decoded = [json.loads(element) for element in json.loads(task.items_expression)] + assert decoded == ["123", "true"] # strings, not 123 / True + + +def test_taskflow_expand_nonliteral_iterable_becomes_placeholder(): + # .expand over an upstream task's output isn't statically knowable -> placeholder + gap, never a + # silent single-run notebook. + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def make():\n return [1, 2, 3]\n" + "@task\n" + "def process(item):\n return item * 2\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " vals = make()\n" + " process.expand(item=vals)\n" + "pipeline()\n" + ) + process = next(t for t in p.tasks if t.task_key.startswith("process")) + assert isinstance(process, PlaceholderActivity) + assert "for_each_task" in process.comment + assert process.raw_definition is not None + # The mapped iterable comes from `vals`, so the dependency edge must survive (not be dropped by + # the mapped-call early return). + assert [d.task_key for d in process.depends_on] == ["vals"] + + +def test_taskflow_partial_expand_becomes_placeholder_not_dropped(): + # .partial(...).expand(...) carries fixed args a for_each inner task can't represent, so it must + # route to a placeholder (not a for_each that silently omits the partial args, nor a silent drop). + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def process(a, b):\n return a + b\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " process.partial(a=1).expand(b=[1, 2, 3])\n" + "pipeline()\n" + ) + assert len(p.tasks) == 1 + task = p.tasks[0] + assert isinstance(task, PlaceholderActivity) + + +def test_taskflow_partial_expand_preserves_upstream_dependency(): + # A .partial(x=upstream) fixed arg is an upstream data-flow dependency; the edge must survive + # even though the mapped task routes to a placeholder. + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def extract():\n return 1\n" + "@task\n" + "def process(x, z):\n return x + z\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " raw = extract()\n" + " process.partial(x=raw).expand(z=[1, 2, 3])\n" + "pipeline()\n" + ) + process = next(t for t in p.tasks if t.task_key.startswith("process")) + assert isinstance(process, PlaceholderActivity) + assert [d.task_key for d in process.depends_on] == ["raw"] + + +def test_taskflow_expand_kwargs_becomes_placeholder(): + # .expand_kwargs([...]) maps whole kwargs dicts (not one param's iterable) -> placeholder, not a + # single-run notebook. + p = _load( + "from airflow.decorators import dag, task\n" + "@task\n" + "def process(a, b):\n return a\n" + "@dag(dag_id='f')\n" + "def pipeline():\n" + " process.expand_kwargs([{'a': 1, 'b': 2}])\n" + "pipeline()\n" + ) + assert len(p.tasks) == 1 + assert isinstance(p.tasks[0], PlaceholderActivity) + + +def test_task_group_mapped_call_becomes_placeholder_not_dropped(): + # A mapped @task_group is a sub-pipeline flowx can't lower; it must become a placeholder + gap, + # never a silently empty pipeline. + p = _load( + "from airflow.decorators import dag, task, task_group\n" + "@task\n" + "def step_a(x):\n return x + 1\n" + "@task_group\n" + "def pair(x):\n return step_a(x)\n" + "@dag(dag_id='g')\n" + "def pipeline():\n" + " pair.expand(x=[1, 2, 3])\n" + "pipeline()\n" + ) + assert len(p.tasks) == 1 + group = p.tasks[0] + assert isinstance(group, PlaceholderActivity) + assert group.original_type == "@task_group" + assert "maps the group over an iterable" in group.comment + assert group.raw_definition is not None + + +def test_task_group_call_preserves_dependency_edges(): + # A @task_group wired with >> must keep its ordering: prep >> grp >> finish. + p = _load( + "from airflow.decorators import dag, task, task_group\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "@task\n" + "def step_a(x):\n return x + 1\n" + "@task_group\n" + "def pair(x):\n return step_a(x)\n" + "@dag(dag_id='g')\n" + "def pipeline():\n" + " prep = PythonOperator(task_id='prep', python_callable=w)\n" + " grp = pair(5)\n" + " finish = PythonOperator(task_id='finish', python_callable=w)\n" + " prep >> grp >> finish\n" + "pipeline()\n" + ) + tasks = _by_key(p) + assert isinstance(tasks["grp"], PlaceholderActivity) + assert [d.task_key for d in tasks["grp"].depends_on] == ["prep"] + assert [d.task_key for d in tasks["finish"].depends_on] == ["grp"] + + def test_multiple_dags_in_one_file_are_loaded_as_separate_pipelines(tmp_path): from flowx.sources.airflow.loader import load_pipelines diff --git a/tests/unit/test_mcp_source_routing.py b/tests/unit/test_mcp_source_routing.py index ce8fa3e..1dd870a 100644 --- a/tests/unit/test_mcp_source_routing.py +++ b/tests/unit/test_mcp_source_routing.py @@ -118,3 +118,9 @@ def test_discover_missing_source_path_errors_clearly(captured, tmp_path: Path): result = server._cmd_discover({"source": "airflow", "output_dir": str(tmp_path)}) assert result["ok"] is False assert "airflow" in result["error"] + + +def test_source_name_rejects_non_string(captured): + # A malformed non-string source must raise (ValueError), not silently coerce 123 -> "123". + with pytest.raises(ValueError, match="must be a string"): + server._source_name({"source": 123}) From f10184561e04376dd80e3a650f148c7813bd3f85 Mon Sep 17 00:00:00 2001 From: Lorenzo Rubio Date: Tue, 21 Jul 2026 23:45:15 +0200 Subject: [PATCH 36/77] fix missing notebook header on bridge notebooks (#8) ## Changes Bridge notebooks (generated by the IfCondition, Switch, and ForEach preparers to evaluate complex ADF expressions that can't be inlined into a task's parameters) were missing the standard `# Databricks notebook source` header that every other flowx-generated notebook has, causing `bundle validate` to reject them as "not a notebook" for any pipeline containing one. - Added `render_bridge_notebook()` to `code_generator.py`, consolidating the three previously independent, byte-for-byte-identical private renderers (`if_condition.py`, `switch.py`, `for_each.py`) into one shared function that prepends the notebook header + command separator. - Updated all three call sites (including both call sites in `for_each.py::_resolve_for_each_inputs_with_bridge`) to use the shared renderer, each passing a `title` naming the source activity. - Removed the three now-dead private renderer functions. ### Linked issues Resolves #7 ### Tests - [ ] manually tested - [x] added unit tests - [ ] added integration tests --- .../preparer/activity_preparers/for_each.py | 37 ++--------- .../activity_preparers/if_condition.py | 29 +-------- .../preparer/activity_preparers/switch.py | 29 +-------- src/flowx/preparer/code_generator.py | 44 +++++++++++++ tests/unit/test_code_generator.py | 59 ++++++++++++++++++ tests/unit/test_preparers.py | 62 +++++++++++++++++++ 6 files changed, 176 insertions(+), 84 deletions(-) diff --git a/src/flowx/preparer/activity_preparers/for_each.py b/src/flowx/preparer/activity_preparers/for_each.py index fdbe6a8..a657062 100644 --- a/src/flowx/preparer/activity_preparers/for_each.py +++ b/src/flowx/preparer/activity_preparers/for_each.py @@ -16,6 +16,7 @@ from flowx.models.dab import DabNotebook, SecretInstruction, SetupTask from flowx.models.ir import TranslationContext from flowx.parser.expression_parser import resolve_expression +from flowx.preparer.code_generator import render_bridge_notebook from flowx.preparer.workflow_preparer import ( PreparedActivity, PreparedWorkflow, @@ -58,11 +59,12 @@ def _resolve_for_each_inputs_with_bridge( value_key = "items" notebook_relative_path = f"notebooks/{bridge_key}.py" base_parameters: dict[str, str] = dict(activity.inputs_bridge_required_parameters) - notebook_source = _render_for_each_inputs_bridge( + notebook_source = render_bridge_notebook( activity.inputs_bridge_notebook_code, list(activity.inputs_bridge_notebook_imports), list(base_parameters.keys()), value_key, + title=f"ForEach inputs bridge: {activity.name}", ) bridge_task: dict[str, Any] = { "task_key": bridge_key, @@ -91,11 +93,12 @@ def _resolve_for_each_inputs_with_bridge( value_key = "items" notebook_relative_path = f"notebooks/{bridge_key}.py" base_parameters = dict(result.required_parameters) - notebook_source = _render_for_each_inputs_bridge( + notebook_source = render_bridge_notebook( result.value, result.imports, list(base_parameters.keys()), value_key, + title=f"ForEach inputs bridge: {activity.name}", ) bridge_task = { "task_key": bridge_key, @@ -111,36 +114,6 @@ def _resolve_for_each_inputs_with_bridge( return items_expression, None, [] -def _render_for_each_inputs_bridge( - notebook_code: str, - imports: list[str], - widget_names: list[str], - value_key: str, -) -> str: - """Generates the Python source for a ForEach-inputs bridge notebook. - - The notebook computes the array value and publishes it via - ``dbutils.jobs.taskValues.set`` so the parent ForEach task can - reference it via ``{{tasks..values.items}}``. - """ - lines: list[str] = [] - seen_imports: set[str] = set() - for imp in imports: - if imp in seen_imports: - continue - seen_imports.add(imp) - lines.append(imp) - if seen_imports: - lines.append("") - for widget in widget_names: - lines.append(f"dbutils.widgets.text('{widget}', '')") - if widget_names: - lines.append("") - lines.append(f"_bridge_value = {notebook_code}") - lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") - return "\n".join(lines) + "\n" - - def _inject_input_parameter(inner_task: dict) -> dict: """Adds ``{{input}}`` as a base_parameter on the inner task. diff --git a/src/flowx/preparer/activity_preparers/if_condition.py b/src/flowx/preparer/activity_preparers/if_condition.py index f69b8b3..eb96072 100644 --- a/src/flowx/preparer/activity_preparers/if_condition.py +++ b/src/flowx/preparer/activity_preparers/if_condition.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any from flowx.models.dab import DabNotebook +from flowx.preparer.code_generator import render_bridge_notebook from flowx.preparer.workflow_preparer import ( PreparedActivity, PreparedArtifacts, @@ -138,11 +139,12 @@ def _build_bridge_task( # Build the bridge notebook source. base_parameters can include widget # bindings the bridge expression depends on. base_parameters: dict[str, str] = dict(activity.bridge_required_parameters) - notebook_source = _render_bridge_notebook( + notebook_source = render_bridge_notebook( activity.bridge_notebook_code, activity.bridge_notebook_imports, list(base_parameters.keys()), value_key, + title=f"IfCondition bridge: {activity.name}", ) bridge_task: dict[str, Any] = { @@ -167,28 +169,3 @@ def _rewrite_bridge_placeholder(operand: str, bridge_value_ref: str | None) -> s # Defensive: translator surfaced a placeholder but no bridge code. return operand return bridge_value_ref - - -def _render_bridge_notebook( - notebook_code: str, - imports: list[str], - widget_names: list[str], - value_key: str, -) -> str: - """Generates the Python source for a condition bridge notebook.""" - lines: list[str] = [] - seen_imports: set[str] = set() - for imp in imports: - if imp in seen_imports: - continue - seen_imports.add(imp) - lines.append(imp) - if seen_imports: - lines.append("") - for widget in widget_names: - lines.append(f"dbutils.widgets.text('{widget}', '')") - if widget_names: - lines.append("") - lines.append(f"_bridge_value = {notebook_code}") - lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") - return "\n".join(lines) + "\n" diff --git a/src/flowx/preparer/activity_preparers/switch.py b/src/flowx/preparer/activity_preparers/switch.py index a9eb353..5f5d4e0 100644 --- a/src/flowx/preparer/activity_preparers/switch.py +++ b/src/flowx/preparer/activity_preparers/switch.py @@ -14,6 +14,7 @@ from flowx.models.ir import TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string from flowx.preparer.activity_preparers.if_condition import inject_outcome_dependency +from flowx.preparer.code_generator import render_bridge_notebook from flowx.preparer.workflow_preparer import ( PreparedActivity, PreparedArtifacts, @@ -220,11 +221,12 @@ def _build_switch_bridge_task( notebook_relative_path = f"notebooks/{bridge_key}.py" base_parameters: dict[str, str] = dict(activity.bridge_required_parameters) - notebook_source = _render_bridge_notebook( + notebook_source = render_bridge_notebook( activity.bridge_notebook_code, activity.bridge_notebook_imports, list(base_parameters.keys()), value_key, + title=f"Switch bridge: {activity.name}", ) bridge_task: dict[str, Any] = { @@ -237,28 +239,3 @@ def _build_switch_bridge_task( bridge_value_ref = f"{{{{tasks.{bridge_key}.values.{value_key}}}}}" notebooks = [DabNotebook(relative_path=notebook_relative_path, content=notebook_source)] return bridge_task, bridge_value_ref, notebooks - - -def _render_bridge_notebook( - notebook_code: str, - imports: list[str], - widget_names: list[str], - value_key: str, -) -> str: - """Generates the Python source for a Switch on-expression bridge notebook.""" - lines: list[str] = [] - seen_imports: set[str] = set() - for imp in imports: - if imp in seen_imports: - continue - seen_imports.add(imp) - lines.append(imp) - if seen_imports: - lines.append("") - for widget in widget_names: - lines.append(f"dbutils.widgets.text('{widget}', '')") - if widget_names: - lines.append("") - lines.append(f"_bridge_value = {notebook_code}") - lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") - return "\n".join(lines) + "\n" diff --git a/src/flowx/preparer/code_generator.py b/src/flowx/preparer/code_generator.py index 6fa73e8..a6ae733 100644 --- a/src/flowx/preparer/code_generator.py +++ b/src/flowx/preparer/code_generator.py @@ -902,6 +902,50 @@ def _command_separator() -> str: return "\n# COMMAND ----------\n\n" +def render_bridge_notebook( + notebook_code: str, + imports: list[str], + widget_names: list[str], + value_key: str, + *, + title: str, +) -> str: + """Generates the Python source for a bridge notebook. + + A bridge notebook evaluates a complex ADF expression that can't be + inlined directly into a task's parameters, then publishes the result + as a task value via ``dbutils.jobs.taskValues.set``. Shared by the + IfCondition, Switch, and ForEach preparers. + + Args: + notebook_code: Python expression to evaluate and publish. + imports: Import statements the expression needs. + widget_names: Names of widgets to declare (bound from base_parameters). + value_key: Task-value key the result is published under. + title: Notebook header title (e.g. naming the source activity). + + Returns: + Complete notebook source, including the Databricks notebook header. + """ + lines: list[str] = [] + seen_imports: set[str] = set() + for imp in imports: + if imp in seen_imports: + continue + seen_imports.add(imp) + lines.append(imp) + if seen_imports: + lines.append("") + for widget in widget_names: + lines.append(f"dbutils.widgets.text('{widget}', '')") + if widget_names: + lines.append("") + lines.append(f"_bridge_value = {notebook_code}") + lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") + body = "\n".join(lines) + "\n" + return _notebook_header(title) + _command_separator() + body + + def _render_query_assignment(query: str) -> str: """Returns ``query = `` source for embedding a lookup query in a notebook. diff --git a/tests/unit/test_code_generator.py b/tests/unit/test_code_generator.py index eb98fc1..1d3e3d6 100644 --- a/tests/unit/test_code_generator.py +++ b/tests/unit/test_code_generator.py @@ -29,6 +29,7 @@ generate_set_variable_notebook, generate_wait_notebook, generate_web_activity_notebook, + render_bridge_notebook, ) # --------------------------------------------------------------------------- @@ -584,3 +585,61 @@ def test_notebook_code_append(self): assert "from datetime import" in content # Should NOT reference widgets.get("value") for code values assert 'dbutils.widgets.get("value")' not in content + + +# --------------------------------------------------------------------------- +# Bridge notebook renderer (shared by IfCondition, Switch, ForEach preparers) +# --------------------------------------------------------------------------- + + +class TestRenderBridgeNotebook: + def test_full_structure_and_order(self): + content = render_bridge_notebook( + "1 + 1", + ["import json"], + ["parent_uid"], + "result", + title="IfCondition bridge: If_ParentUId", + ) + _assert_valid_python(content, "bridge notebook") + assert content.startswith("# Databricks notebook source") + assert "# MAGIC # IfCondition bridge: If_ParentUId" in content + assert "# COMMAND ----------" in content + # Body (after the header/separator) preserves import -> widget -> + # assignment -> publish ordering. + body = content.split("# COMMAND ----------", 1)[1] + import_pos = body.index("import json") + widget_pos = body.index("dbutils.widgets.text('parent_uid', '')") + assign_pos = body.index("_bridge_value = 1 + 1") + publish_pos = body.index("dbutils.jobs.taskValues.set(key='result', value=_bridge_value)") + assert import_pos < widget_pos < assign_pos < publish_pos + + def test_dedups_repeated_imports(self): + content = render_bridge_notebook("1", ["import json", "import json"], [], "result", title="t") + assert content.count("import json") == 1 + + def test_no_imports_omits_leading_blank_line(self): + content = render_bridge_notebook("1", [], ["x"], "result", title="t") + body = content.split("# COMMAND ----------\n\n", 1)[1] + assert body.startswith("dbutils.widgets.text('x', '')") + + def test_no_widgets_omits_blank_line(self): + content = render_bridge_notebook("1", [], [], "result", title="t") + body = content.split("# COMMAND ----------\n\n", 1)[1] + assert body.startswith("_bridge_value = 1") + + def test_notebook_code_inserted_verbatim(self): + # Expressions with quotes/commas must pass through untouched -- the + # renderer does plain string interpolation, no escaping. + code = "str(dbutils.widgets.get('item_type')).split(',')[0]" + content = render_bridge_notebook(code, [], ["item_type"], "result", title="t") + assert f"_bridge_value = {code}" in content + + def test_multiple_widgets_all_declared(self): + content = render_bridge_notebook("1", [], ["a", "b"], "result", title="t") + assert "dbutils.widgets.text('a', '')" in content + assert "dbutils.widgets.text('b', '')" in content + + def test_value_key_used_in_publish_call(self): + content = render_bridge_notebook("1", [], [], "items", title="t") + assert "dbutils.jobs.taskValues.set(key='items', value=_bridge_value)" in content diff --git a/tests/unit/test_preparers.py b/tests/unit/test_preparers.py index f206c73..d8ea058 100644 --- a/tests/unit/test_preparers.py +++ b/tests/unit/test_preparers.py @@ -1205,6 +1205,68 @@ def test_prepare_placeholder_generates_stub(self): assert "NotImplementedError" in prepared.notebooks[0].content +# --------------------------------------------------------------------------- +# Bridge notebook header (BUG_bridge_notebook_missing_header) +# --------------------------------------------------------------------------- + + +class TestBridgeNotebookHeader: + """Every generated bridge notebook must start with the Databricks notebook + header, like every other flowx-generated notebook, or `bundle validate` + rejects it as "not a notebook". Covers all three independent bridge + renderers: if_condition, switch, and for_each's inputs bridge.""" + + def test_if_condition_bridge_notebook_has_header(self): + activity = IfConditionActivity( + **_make_base("If_ParentUId", "if_parentuid"), + op="EQUAL_TO", + left="__bridge__", + right="1", + bridge_notebook_code="str(dbutils.widgets.get('parent_uid'))", + bridge_notebook_imports=[], + bridge_required_parameters={"parent_uid": "{{tasks.init.values.parent_uid}}"}, + ) + prepared = prepare_activity(activity) + bridge_notebooks = [nb for nb in prepared.notebooks if nb.relative_path.endswith("_bridge.py")] + assert len(bridge_notebooks) == 1 + assert bridge_notebooks[0].content.startswith("# Databricks notebook source") + assert "# MAGIC # IfCondition bridge: If_ParentUId" in bridge_notebooks[0].content + + def test_switch_bridge_notebook_has_header(self): + activity = SwitchActivity( + **_make_base("Route", "route"), + on_expression="@toUpper(item().type)", + cases=[SwitchCase(value="FULL", activities=[WaitActivity(**_make_base("W", "w"), wait_time_seconds=1)])], + bridge_notebook_code="str(dbutils.widgets.get('item_type')).upper()", + bridge_notebook_imports=[], + bridge_required_parameters={"item_type": "{{tasks.init.values.item_type}}"}, + ) + prepared = prepare_activity(activity) + bridge_notebooks = [nb for nb in prepared.notebooks if nb.relative_path.endswith("_bridge.py")] + assert len(bridge_notebooks) == 1 + assert bridge_notebooks[0].content.startswith("# Databricks notebook source") + assert "# MAGIC # Switch bridge: Route" in bridge_notebooks[0].content + + def test_for_each_inputs_bridge_notebook_has_header(self): + inner = WaitActivity(**_make_base("Inner", "inner"), wait_time_seconds=1) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@split(variables('fecha'),',')", + inputs_bridge_notebook_code=( + "str(dbutils.jobs.taskValues.get(taskKey='_init_fecha', key='fecha')).split(str(','))" + ), + inputs_bridge_notebook_imports=[], + inputs_bridge_required_parameters={"fecha": "{{tasks._init_fecha.values.fecha}}"}, + inner_activities=[inner], + concurrency=10, + ) + prepared = prepare_activity(activity) + bridge_notebooks = [nb for nb in prepared.notebooks if nb.relative_path.endswith("_bridge.py")] + assert len(bridge_notebooks) == 1 + assert bridge_notebooks[0].content.startswith("# Databricks notebook source") + assert "# MAGIC # ForEach inputs bridge: Loop" in bridge_notebooks[0].content + + # --------------------------------------------------------------------------- # Notebook content validity # --------------------------------------------------------------------------- From fb9c072f7fcb3d28ef9c945fc025e1b9aa6e829b Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Tue, 21 Jul 2026 15:59:22 -0700 Subject: [PATCH 37/77] Document Airflow source in README and address PR review comments README: cover the Airflow source (source-neutral tagline/architecture, new Supported Airflow Operators section, corrected phase descriptions and stale module paths). Review fixes (Copilot): - _has_dependency_cycle guards non-dict tasks/deps so the invariant checker reports findings instead of crashing on malformed bundle YAML - the phase-runner source-path alias now also normalises the equals form (--source-path=/x / --adf-source-path=/x), not just the space form - _timedelta_seconds rounds a sub-second timeout/retry_delay up to 1s instead of truncating to 0 (which silently dropped the value) --- README.md | 70 ++++++++++++++++++------- src/flowx/adapter/__main__.py | 13 ++++- src/flowx/sources/airflow/templating.py | 5 +- src/flowx/validate/bundle_invariants.py | 4 +- tests/unit/test_airflow_operators.py | 13 +++++ tests/unit/test_source_router.py | 22 ++++++++ 6 files changed, 104 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index e06e533..1dfb647 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,16 @@ # flowx -ADF to Databricks Lakeflow Jobs translator, delivered as agent skills. +Orchestrator-to-Databricks Lakeflow Jobs translator, delivered as agent skills. -flowx converts Azure Data Factory (ADF) pipeline definitions into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). It deterministically translates known activity types and falls back to agentic (LLM-assisted) translation for complex or rare types. flowx runs as a set of [agent skills](skills/) usable from Databricks Genie Code, Claude Code, or any tool that supports the Agent Skills standard. +flowx converts a source orchestrator's pipelines — **Azure Data Factory (ADF)** or **Apache +Airflow** — into Databricks Lakeflow Jobs packaged as Declarative Automation Bundles (DABs). It +deterministically translates known activity/operator types and falls back to agentic (LLM-assisted) +translation for complex or rare types. flowx runs as a set of [agent skills](skills/) usable from +Databricks Genie Code, Claude Code, or any tool that supports the Agent Skills standard. + +Both sources emit the same source-neutral Pipeline IR, so the convert-configuration and package +phases are shared; only discovery and translation are source-specific. Pick the source with +`--source {adf,airflow}` (required for discover/convert; package is source-independent). ## Architecture @@ -10,25 +18,25 @@ flowx converts Azure Data Factory (ADF) pipeline definitions into Databricks Lak flowx Pipeline ================== - ADF JSON (UC Volumes / Workspace) - | - v - +------------------+ - | 1. DISCOVER | Parse ADF ARM/JSON exports - | adf_loader.py | -> Typed AST -> metadata/inventory.json - +------------------+ + ADF ARM/JSON (UC Volumes / Workspace) | Airflow DAG .py files + \ | / + v v v + +---------------------------------------------------------------+ + | 1. DISCOVER sources// -> metadata/inventory.json + | (ADF: ARM/JSON parse; Airflow: static ast parse) + +---------------------------------------------------------------+ | v - +------------------+ - | 2. CONVERT | Registry dispatch + topological sort - | engine.py | -> Pipeline IR (deterministic + agentic gaps) - +------------------+ + +---------------------------------------------------------------+ + | 2. CONVERT sources// -> shared Pipeline IR + | (deterministic mappings + agentic gaps) + +---------------------------------------------------------------+ | v - +------------------+ - | 3. PACKAGE | IR -> DAB YAML + notebooks + setup scripts - | dab_writer.py | -> Deployable DABs project - +------------------+ + +---------------------------------------------------------------+ + | 3. PACKAGE bundler/dab_writer.py (source-independent) + | IR -> DAB YAML + notebooks + setup scripts + +---------------------------------------------------------------+ | v databricks bundle validate / deploy @@ -88,7 +96,7 @@ Run the end-to-end migration: Or run individual phases: ``` -/flowx:flowx-discover # Parse ADF JSON, produce inventory + complexity report +/flowx:flowx-discover # Parse the source (ADF JSON / Airflow DAGs), produce inventory + complexity report /flowx:flowx-convert # Deterministic + agentic translation /flowx:flowx-package # Generate DABs project ``` @@ -138,13 +146,35 @@ agent using LLM-assisted reasoning from the activity's ARM JSON. | Script | LLM-assisted (agentic) | | Until | LLM-assisted (agentic) | +## Supported Airflow Operators + +The Airflow source parses DAG `.py` modules **statically** (via `ast`, no Airflow install or DAG +execution) and maps ~35 operator/sensor families to the shared IR. Highlights: + +- **Compute / scripts** — `PythonOperator` (callable → runnable notebook with transitive deps), + `BashOperator` / `SSHOperator` (incl. `spark-submit` lift), `SparkSubmitOperator`, the Databricks + provider operators, and SQL operators (`DatabricksSql*`, `SQLExecuteQueryOperator`, `HiveOperator`, + …) → `sql_task`. +- **TaskFlow API** — `@dag` / `@task`; implicit XCom data flow lowers to `dbutils.jobs.taskValues`. + `@task.expand([literal])` → `for_each_task`; non-literal / `.partial().expand()` / `@task_group` → + placeholder + gap (dependencies preserved — never a silent drop). +- **Sensors** — file/table/time sensors → job triggers or polling notebooks; `ExternalTaskSensor` → + cross-DAG wait; Http/Python/DateTime → polling tasks. +- **dbt** — dbt CLI operators and astronomer-cosmos `DbtDag` / `DbtTaskGroup` → a dbt-factory job + (static per-node explosion by default, or PyDABs via `--dbt-mode pydabs`). +- **Scheduling & semantics** — cron → Quartz, `timedelta` → periodic, `trigger_rule` → `run_if`, + `params={...}` → job parameters, `>>` / `<<` / `set_upstream` / TaskGroup edges. + +Operators without a deterministic mapping become a placeholder recorded in `gaps.json` for the +agentic round. Full matrix: [`skills/flowx-convert/sources/airflow-coverage.md`](skills/flowx-convert/sources/airflow-coverage.md). + ## How It Works ### Phase 1: Discover -Reads ADF JSON definitions from Unity Catalog volumes (or a `/Workspace` Git folder), normalizes ARM template format, parses into typed AST nodes, and classifies each activity as deterministic, agentic, or unsupported. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`. +Parses the source into typed nodes and classifies each activity/operator as deterministic, agentic, or unsupported — ADF JSON from Unity Catalog volumes (or a `/Workspace` Git folder, normalizing ARM template format), or Airflow DAG `.py` modules read statically with `ast`. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`. ### Phase 2: Convert -Applies deterministic translators via registry dispatch, resolves dependencies through topological sort, and threads immutable `TranslationContext` through control-flow visitors. Agentic gaps are flagged for LLM-assisted translation. Produces Pipeline IR. +Applies deterministic translators (ADF activity registry / Airflow operator mapping), resolves dependencies, and flags agentic gaps for LLM-assisted translation. Produces the shared Pipeline IR consumed unchanged by the package phase. ### Phase 3: Package Converts Pipeline IR into a deployable DABs project: `databricks.yml`, per-job YAML resource files, generated Python notebooks, and setup scripts for UC volumes, secrets, and connections. diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py index e9957e5..b0ac1e4 100644 --- a/src/flowx/adapter/__main__.py +++ b/src/flowx/adapter/__main__.py @@ -514,7 +514,18 @@ def _run_phase(phase: str, forward: list[str]) -> int: aliases = {_SOURCE_PATH_FLAG: "--source-dir", source.source_path_flag: "--source-dir"} module = importlib.import_module(module_path) - mapped = [aliases.get(token, token) for token in remaining] + # Alias both the bare form (`--source-path X`) and the equals form (`--source-path=X`) so a + # documented alias works either way; the phase module only knows `--source-dir`. + def _alias(token: str) -> str: + if token in aliases: + return aliases[token] + if token.startswith("--") and "=" in token: + flag, value = token.split("=", 1) + if flag in aliases: + return f"{aliases[flag]}={value}" + return token + + mapped = [_alias(token) for token in remaining] try: return module.main(mapped) or 0 except SystemExit as exit_signal: # e.g. argparse usage error -> parser.error() raises SystemExit diff --git a/src/flowx/sources/airflow/templating.py b/src/flowx/sources/airflow/templating.py index b4fb1d1..c5ae473 100644 --- a/src/flowx/sources/airflow/templating.py +++ b/src/flowx/sources/airflow/templating.py @@ -10,6 +10,7 @@ from __future__ import annotations import ast +import math import re from typing import Any @@ -173,7 +174,9 @@ def _timedelta_seconds(node: ast.expr | None) -> int | None: for kw in node.keywords: if kw.arg in units and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, (int, float)): total += kw.value.value * units[kw.arg] - return int(total) if total > 0 else None + # Round a sub-second total UP to 1s rather than truncating to 0 -- a sub-second timeout/retry_delay + # is better preserved as 1s than silently dropped (int(0.5) == 0 would read as "unset"). + return math.ceil(total) if total > 0 else None def _literal_int(node: ast.expr | None) -> int | None: diff --git a/src/flowx/validate/bundle_invariants.py b/src/flowx/validate/bundle_invariants.py index 4043469..5a88d44 100644 --- a/src/flowx/validate/bundle_invariants.py +++ b/src/flowx/validate/bundle_invariants.py @@ -173,11 +173,13 @@ def _has_dependency_cycle(tasks: list[dict[str, Any]]) -> bool: in_degree: dict[str, int] = {key: 0 for key in keys} adjacency: dict[str, set[str]] = {key: set() for key in keys} for task in tasks: + if not isinstance(task, dict): + continue downstream = task.get("task_key") if not isinstance(downstream, str) or downstream not in key_set: continue for dep in task.get("depends_on") or []: - upstream = dep.get("task_key") + upstream = dep.get("task_key") if isinstance(dep, dict) else None if isinstance(upstream, str) and upstream in key_set and downstream not in adjacency[upstream]: adjacency[upstream].add(downstream) in_degree[downstream] += 1 diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py index 5610251..39c5f51 100644 --- a/tests/unit/test_airflow_operators.py +++ b/tests/unit/test_airflow_operators.py @@ -946,6 +946,19 @@ def test_default_args_apply_retries_timeout_retry_delay(): assert task.min_retry_interval_millis == 300000 +def test_subsecond_timeout_rounds_up_not_dropped(): + # A sub-second execution_timeout must round up to 1s, not truncate to 0 (which reads as "unset"). + p = _load( + "from datetime import timedelta\n" + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d', default_args={'execution_timeout': timedelta(milliseconds=500)}) as dag:\n" + " t = PythonOperator(task_id='t', python_callable=w)\n" + ) + assert _by_key(p)["t"].timeout_seconds == 1 + + def test_per_task_retries_override_default_args(): p = _load( "from airflow import DAG\n" diff --git a/tests/unit/test_source_router.py b/tests/unit/test_source_router.py index a8921f3..22f8b75 100644 --- a/tests/unit/test_source_router.py +++ b/tests/unit/test_source_router.py @@ -87,6 +87,28 @@ def test_airflow_discover_then_convert_route(): assert (out / ".work" / "translation_report.json").exists() +def test_source_path_alias_equals_form_routes(): + # The `--source-path=` equals form must normalise to --source-dir just like the space form, + # or the phase module rejects it with a usage error. + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + rc = _run_phase( + "discover", ["--source", "airflow", f"--source-path={_DAG_FIXTURE}", f"--output-dir={out}"] + ) + assert rc == 0 + assert (out / "metadata" / "inventory.json").exists() + + +def test_source_specific_alias_equals_form_routes(): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + rc = _run_phase( + "discover", ["--source", "airflow", f"--airflow-source-path={_DAG_FIXTURE}", "--output-dir", str(out)] + ) + assert rc == 0 + assert (out / "metadata" / "inventory.json").exists() + + def test_package_is_source_independent(): # package ignores --source and routes to the shared bundler; drive the whole chain. with tempfile.TemporaryDirectory() as tmp: From cbee5eaa4ded9558186e56613d28ec9969498bf7 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Wed, 22 Jul 2026 14:48:06 -0400 Subject: [PATCH 38/77] Mirror public repo content (#19) ## Changes This PR merges public repo changes. ### Linked issues N/A ### Tests - [ ] manually tested - [ ] added unit tests - [ ] added integration tests --------- Co-authored-by: Greg Hansen <163584195+ghanse@users.noreply.github.com> Co-authored-by: service-jira-pub-repo-auto --- .build-constraints.txt | 6 ++--- .../{bug_report.yml => bug.yml} | 0 .../{feature_request.yml => feature.yml} | 0 .github/codecov.yml | 10 +++++++++ .github/dependabot.yml | 10 +++++++++ .github/workflows/docs-release.yml | 6 +++-- .github/workflows/push.yml | 14 ++++++++++++ CODEOWNERS | 1 + CODEOWNERS.txt | 0 Makefile | 2 +- README.md | 22 +++++++++++++++++++ pyproject.toml | 4 +++- 12 files changed, 68 insertions(+), 7 deletions(-) rename .github/ISSUE_TEMPLATE/{bug_report.yml => bug.yml} (100%) rename .github/ISSUE_TEMPLATE/{feature_request.yml => feature.yml} (100%) create mode 100644 .github/codecov.yml create mode 100644 .github/dependabot.yml create mode 100644 CODEOWNERS delete mode 100644 CODEOWNERS.txt diff --git a/.build-constraints.txt b/.build-constraints.txt index ee7a59e..3a6ff79 100644 --- a/.build-constraints.txt +++ b/.build-constraints.txt @@ -1,6 +1,6 @@ -hatchling==1.30.1 \ - --hash=sha256:161eacafb3c6f91526e92116d21426369f2c36e98c36a864f11a96345ad4ee31 \ - --hash=sha256:eee4fd45357f72ebb3d7a42e5d72cfb5e29ed426d79e8836288926c4258d5f2e +hatchling==1.31.0 \ + --hash=sha256:6b48ad4068a482ed7239b3a8215bc55b47aad3345d58dfc94e553c5d2d46211b \ + --hash=sha256:aac80bec8b6fe35e8480f1c335be8910fa210a0e6f735a139be205dadcacb544 packaging==26.2 \ --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.yml rename to .github/ISSUE_TEMPLATE/bug.yml diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.yml rename to .github/ISSUE_TEMPLATE/feature.yml diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000..8ad3f44 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,10 @@ +coverage: + status: + project: + default: + target: auto + threshold: 0.5% # The minimum coverage threshold for the project + patch: + default: + target: auto + threshold: 0.5% # The minimum coverage threshold for the patch diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..fd193b1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "uv" + directory: "/" + cooldown: + default-days: 7 + exclude: + - "databricks*" + schedule: + interval: "daily" diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml index d71e6ab..c46e51c 100644 --- a/.github/workflows/docs-release.yml +++ b/.github/workflows/docs-release.yml @@ -29,7 +29,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: - bun-version: latest + bun-version: 1.3.14 - name: Scrub internal proxy URLs from bun.lock # The Databricks-internal npm proxy is unreachable from public runners; @@ -53,7 +53,9 @@ jobs: deploy: name: Deploy to GitHub Pages needs: build - runs-on: ubuntu-latest + runs-on: + group: databricks-solutions-protected-runner-group + labels: linux-ubuntu-latest permissions: pages: write diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5063847..24baca4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -5,8 +5,21 @@ on: push: branches: [main] +permissions: + contents: read + jobs: + # Gate all downstream jobs behind a single check so PRs from forks (no access to the + # tool environment) and draft PRs do not trigger the expensive acceptance suite. + # PRs from forks are to be tested by the reviewer(s) / maintainer(s) before merging. + not-a-fork: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork + steps: + - run: echo "Not a fork PR, proceeding" + ci: + needs: not-a-fork runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -26,6 +39,7 @@ jobs: || { echo "requirements.txt is stale. Run 'make requirements' (or 'make precommit') and commit it."; exit 1; } fmt: + needs: not-a-fork runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..d11692b --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @databricks-solutions/flowx-maintainers \ No newline at end of file diff --git a/CODEOWNERS.txt b/CODEOWNERS.txt deleted file mode 100644 index e69de29..0000000 diff --git a/Makefile b/Makefile index a99a435..1a82a25 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ lock-dependencies: uv run --exact --all-extras --group yq tomlq -r '.["build-system"].requires[]' pyproject.toml | \ uv pip compile --generate-hashes --universal --no-header - > build-constraints-new.txt mv build-constraints-new.txt .build-constraints.txt - perl -pi -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple"|g' uv.lock + perl -pi -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple"|g; s|url = "https://[^/"]+/packages/|url = "https://files.pythonhosted.org/packages/|g; s|, size = \d+||g' uv.lock $(MAKE) requirements requirements: diff --git a/README.md b/README.md index e06e533..ea4888e 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,28 @@ Or run individual phases: (In Genie Code, invoke the same skills with the `@` prefix, e.g. `@flowx-migrate`.) +## Setup + +`/flowx:flowx-setup` keys off the `DATABRICKS_RUNTIME_VERSION` environment variable +(the same signal the rest of the plugin uses to detect Databricks) and prepares one +of two execution paths: + +- **Local / Claude Code (virtual environment).** The phases run from the plugin's + CLI. Setup runs `scripts/bootstrap.sh`, which creates a `.venv`, installs + `requirements.txt`, and writes the resolved interpreter path to a + `.migration-venv` marker file that the phase skills read. Optionally, a local + (stdio) MCP server can be registered to drive the phases through MCP tools + instead of the CLI. + +- **Databricks Genie Code (MCP server, no virtual environment).** The phases run as + a single `flowx` MCP tool hosted on a Databricks App. Setup runs `app/deploy.sh`, + which stages a self-contained bundle, syncs it to `/Workspace/Shared/mcp-flowx`, + and deploys the `mcp-flowx` app. You then grant app/data access and register the + app under Genie Code **Settings → MCP Servers**. No venv is created on this path. + +Run setup once before any other flowx skill, or again whenever the environment is +missing. + ## Supported ADF Activity Types ### Deterministic (16 types) diff --git a/pyproject.toml b/pyproject.toml index 057e8fb..8d3e242 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,9 @@ yq = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "hatchling>=1.27,<2.0" +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] From 360b1369ac4701c6b1a4360aec157561b1804197 Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:10:45 -0400 Subject: [PATCH 39/77] Merge internal (#14) ## Changes This PR merges fixes missing `# Databricks notebook source` lines for generated notebooks. ### Linked issues N/A ### Tests - [x] manually tested - [x] added unit tests - [ ] added integration tests --------- Co-authored-by: service-jira-pub-repo-auto Co-authored-by: Lorenzo Rubio --- .../preparer/activity_preparers/for_each.py | 37 ++--------- .../activity_preparers/if_condition.py | 29 +-------- .../preparer/activity_preparers/switch.py | 29 +-------- src/flowx/preparer/code_generator.py | 44 +++++++++++++ tests/unit/test_code_generator.py | 59 ++++++++++++++++++ tests/unit/test_preparers.py | 62 +++++++++++++++++++ 6 files changed, 176 insertions(+), 84 deletions(-) diff --git a/src/flowx/preparer/activity_preparers/for_each.py b/src/flowx/preparer/activity_preparers/for_each.py index fdbe6a8..a657062 100644 --- a/src/flowx/preparer/activity_preparers/for_each.py +++ b/src/flowx/preparer/activity_preparers/for_each.py @@ -16,6 +16,7 @@ from flowx.models.dab import DabNotebook, SecretInstruction, SetupTask from flowx.models.ir import TranslationContext from flowx.parser.expression_parser import resolve_expression +from flowx.preparer.code_generator import render_bridge_notebook from flowx.preparer.workflow_preparer import ( PreparedActivity, PreparedWorkflow, @@ -58,11 +59,12 @@ def _resolve_for_each_inputs_with_bridge( value_key = "items" notebook_relative_path = f"notebooks/{bridge_key}.py" base_parameters: dict[str, str] = dict(activity.inputs_bridge_required_parameters) - notebook_source = _render_for_each_inputs_bridge( + notebook_source = render_bridge_notebook( activity.inputs_bridge_notebook_code, list(activity.inputs_bridge_notebook_imports), list(base_parameters.keys()), value_key, + title=f"ForEach inputs bridge: {activity.name}", ) bridge_task: dict[str, Any] = { "task_key": bridge_key, @@ -91,11 +93,12 @@ def _resolve_for_each_inputs_with_bridge( value_key = "items" notebook_relative_path = f"notebooks/{bridge_key}.py" base_parameters = dict(result.required_parameters) - notebook_source = _render_for_each_inputs_bridge( + notebook_source = render_bridge_notebook( result.value, result.imports, list(base_parameters.keys()), value_key, + title=f"ForEach inputs bridge: {activity.name}", ) bridge_task = { "task_key": bridge_key, @@ -111,36 +114,6 @@ def _resolve_for_each_inputs_with_bridge( return items_expression, None, [] -def _render_for_each_inputs_bridge( - notebook_code: str, - imports: list[str], - widget_names: list[str], - value_key: str, -) -> str: - """Generates the Python source for a ForEach-inputs bridge notebook. - - The notebook computes the array value and publishes it via - ``dbutils.jobs.taskValues.set`` so the parent ForEach task can - reference it via ``{{tasks..values.items}}``. - """ - lines: list[str] = [] - seen_imports: set[str] = set() - for imp in imports: - if imp in seen_imports: - continue - seen_imports.add(imp) - lines.append(imp) - if seen_imports: - lines.append("") - for widget in widget_names: - lines.append(f"dbutils.widgets.text('{widget}', '')") - if widget_names: - lines.append("") - lines.append(f"_bridge_value = {notebook_code}") - lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") - return "\n".join(lines) + "\n" - - def _inject_input_parameter(inner_task: dict) -> dict: """Adds ``{{input}}`` as a base_parameter on the inner task. diff --git a/src/flowx/preparer/activity_preparers/if_condition.py b/src/flowx/preparer/activity_preparers/if_condition.py index f69b8b3..eb96072 100644 --- a/src/flowx/preparer/activity_preparers/if_condition.py +++ b/src/flowx/preparer/activity_preparers/if_condition.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any from flowx.models.dab import DabNotebook +from flowx.preparer.code_generator import render_bridge_notebook from flowx.preparer.workflow_preparer import ( PreparedActivity, PreparedArtifacts, @@ -138,11 +139,12 @@ def _build_bridge_task( # Build the bridge notebook source. base_parameters can include widget # bindings the bridge expression depends on. base_parameters: dict[str, str] = dict(activity.bridge_required_parameters) - notebook_source = _render_bridge_notebook( + notebook_source = render_bridge_notebook( activity.bridge_notebook_code, activity.bridge_notebook_imports, list(base_parameters.keys()), value_key, + title=f"IfCondition bridge: {activity.name}", ) bridge_task: dict[str, Any] = { @@ -167,28 +169,3 @@ def _rewrite_bridge_placeholder(operand: str, bridge_value_ref: str | None) -> s # Defensive: translator surfaced a placeholder but no bridge code. return operand return bridge_value_ref - - -def _render_bridge_notebook( - notebook_code: str, - imports: list[str], - widget_names: list[str], - value_key: str, -) -> str: - """Generates the Python source for a condition bridge notebook.""" - lines: list[str] = [] - seen_imports: set[str] = set() - for imp in imports: - if imp in seen_imports: - continue - seen_imports.add(imp) - lines.append(imp) - if seen_imports: - lines.append("") - for widget in widget_names: - lines.append(f"dbutils.widgets.text('{widget}', '')") - if widget_names: - lines.append("") - lines.append(f"_bridge_value = {notebook_code}") - lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") - return "\n".join(lines) + "\n" diff --git a/src/flowx/preparer/activity_preparers/switch.py b/src/flowx/preparer/activity_preparers/switch.py index a9eb353..5f5d4e0 100644 --- a/src/flowx/preparer/activity_preparers/switch.py +++ b/src/flowx/preparer/activity_preparers/switch.py @@ -14,6 +14,7 @@ from flowx.models.ir import TranslationContext from flowx.parser.expression_parser import resolve_expression, resolve_interpolated_string from flowx.preparer.activity_preparers.if_condition import inject_outcome_dependency +from flowx.preparer.code_generator import render_bridge_notebook from flowx.preparer.workflow_preparer import ( PreparedActivity, PreparedArtifacts, @@ -220,11 +221,12 @@ def _build_switch_bridge_task( notebook_relative_path = f"notebooks/{bridge_key}.py" base_parameters: dict[str, str] = dict(activity.bridge_required_parameters) - notebook_source = _render_bridge_notebook( + notebook_source = render_bridge_notebook( activity.bridge_notebook_code, activity.bridge_notebook_imports, list(base_parameters.keys()), value_key, + title=f"Switch bridge: {activity.name}", ) bridge_task: dict[str, Any] = { @@ -237,28 +239,3 @@ def _build_switch_bridge_task( bridge_value_ref = f"{{{{tasks.{bridge_key}.values.{value_key}}}}}" notebooks = [DabNotebook(relative_path=notebook_relative_path, content=notebook_source)] return bridge_task, bridge_value_ref, notebooks - - -def _render_bridge_notebook( - notebook_code: str, - imports: list[str], - widget_names: list[str], - value_key: str, -) -> str: - """Generates the Python source for a Switch on-expression bridge notebook.""" - lines: list[str] = [] - seen_imports: set[str] = set() - for imp in imports: - if imp in seen_imports: - continue - seen_imports.add(imp) - lines.append(imp) - if seen_imports: - lines.append("") - for widget in widget_names: - lines.append(f"dbutils.widgets.text('{widget}', '')") - if widget_names: - lines.append("") - lines.append(f"_bridge_value = {notebook_code}") - lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") - return "\n".join(lines) + "\n" diff --git a/src/flowx/preparer/code_generator.py b/src/flowx/preparer/code_generator.py index 6fa73e8..a6ae733 100644 --- a/src/flowx/preparer/code_generator.py +++ b/src/flowx/preparer/code_generator.py @@ -902,6 +902,50 @@ def _command_separator() -> str: return "\n# COMMAND ----------\n\n" +def render_bridge_notebook( + notebook_code: str, + imports: list[str], + widget_names: list[str], + value_key: str, + *, + title: str, +) -> str: + """Generates the Python source for a bridge notebook. + + A bridge notebook evaluates a complex ADF expression that can't be + inlined directly into a task's parameters, then publishes the result + as a task value via ``dbutils.jobs.taskValues.set``. Shared by the + IfCondition, Switch, and ForEach preparers. + + Args: + notebook_code: Python expression to evaluate and publish. + imports: Import statements the expression needs. + widget_names: Names of widgets to declare (bound from base_parameters). + value_key: Task-value key the result is published under. + title: Notebook header title (e.g. naming the source activity). + + Returns: + Complete notebook source, including the Databricks notebook header. + """ + lines: list[str] = [] + seen_imports: set[str] = set() + for imp in imports: + if imp in seen_imports: + continue + seen_imports.add(imp) + lines.append(imp) + if seen_imports: + lines.append("") + for widget in widget_names: + lines.append(f"dbutils.widgets.text('{widget}', '')") + if widget_names: + lines.append("") + lines.append(f"_bridge_value = {notebook_code}") + lines.append(f"dbutils.jobs.taskValues.set(key='{value_key}', value=_bridge_value)") + body = "\n".join(lines) + "\n" + return _notebook_header(title) + _command_separator() + body + + def _render_query_assignment(query: str) -> str: """Returns ``query = `` source for embedding a lookup query in a notebook. diff --git a/tests/unit/test_code_generator.py b/tests/unit/test_code_generator.py index eb98fc1..1d3e3d6 100644 --- a/tests/unit/test_code_generator.py +++ b/tests/unit/test_code_generator.py @@ -29,6 +29,7 @@ generate_set_variable_notebook, generate_wait_notebook, generate_web_activity_notebook, + render_bridge_notebook, ) # --------------------------------------------------------------------------- @@ -584,3 +585,61 @@ def test_notebook_code_append(self): assert "from datetime import" in content # Should NOT reference widgets.get("value") for code values assert 'dbutils.widgets.get("value")' not in content + + +# --------------------------------------------------------------------------- +# Bridge notebook renderer (shared by IfCondition, Switch, ForEach preparers) +# --------------------------------------------------------------------------- + + +class TestRenderBridgeNotebook: + def test_full_structure_and_order(self): + content = render_bridge_notebook( + "1 + 1", + ["import json"], + ["parent_uid"], + "result", + title="IfCondition bridge: If_ParentUId", + ) + _assert_valid_python(content, "bridge notebook") + assert content.startswith("# Databricks notebook source") + assert "# MAGIC # IfCondition bridge: If_ParentUId" in content + assert "# COMMAND ----------" in content + # Body (after the header/separator) preserves import -> widget -> + # assignment -> publish ordering. + body = content.split("# COMMAND ----------", 1)[1] + import_pos = body.index("import json") + widget_pos = body.index("dbutils.widgets.text('parent_uid', '')") + assign_pos = body.index("_bridge_value = 1 + 1") + publish_pos = body.index("dbutils.jobs.taskValues.set(key='result', value=_bridge_value)") + assert import_pos < widget_pos < assign_pos < publish_pos + + def test_dedups_repeated_imports(self): + content = render_bridge_notebook("1", ["import json", "import json"], [], "result", title="t") + assert content.count("import json") == 1 + + def test_no_imports_omits_leading_blank_line(self): + content = render_bridge_notebook("1", [], ["x"], "result", title="t") + body = content.split("# COMMAND ----------\n\n", 1)[1] + assert body.startswith("dbutils.widgets.text('x', '')") + + def test_no_widgets_omits_blank_line(self): + content = render_bridge_notebook("1", [], [], "result", title="t") + body = content.split("# COMMAND ----------\n\n", 1)[1] + assert body.startswith("_bridge_value = 1") + + def test_notebook_code_inserted_verbatim(self): + # Expressions with quotes/commas must pass through untouched -- the + # renderer does plain string interpolation, no escaping. + code = "str(dbutils.widgets.get('item_type')).split(',')[0]" + content = render_bridge_notebook(code, [], ["item_type"], "result", title="t") + assert f"_bridge_value = {code}" in content + + def test_multiple_widgets_all_declared(self): + content = render_bridge_notebook("1", [], ["a", "b"], "result", title="t") + assert "dbutils.widgets.text('a', '')" in content + assert "dbutils.widgets.text('b', '')" in content + + def test_value_key_used_in_publish_call(self): + content = render_bridge_notebook("1", [], [], "items", title="t") + assert "dbutils.jobs.taskValues.set(key='items', value=_bridge_value)" in content diff --git a/tests/unit/test_preparers.py b/tests/unit/test_preparers.py index f206c73..d8ea058 100644 --- a/tests/unit/test_preparers.py +++ b/tests/unit/test_preparers.py @@ -1205,6 +1205,68 @@ def test_prepare_placeholder_generates_stub(self): assert "NotImplementedError" in prepared.notebooks[0].content +# --------------------------------------------------------------------------- +# Bridge notebook header (BUG_bridge_notebook_missing_header) +# --------------------------------------------------------------------------- + + +class TestBridgeNotebookHeader: + """Every generated bridge notebook must start with the Databricks notebook + header, like every other flowx-generated notebook, or `bundle validate` + rejects it as "not a notebook". Covers all three independent bridge + renderers: if_condition, switch, and for_each's inputs bridge.""" + + def test_if_condition_bridge_notebook_has_header(self): + activity = IfConditionActivity( + **_make_base("If_ParentUId", "if_parentuid"), + op="EQUAL_TO", + left="__bridge__", + right="1", + bridge_notebook_code="str(dbutils.widgets.get('parent_uid'))", + bridge_notebook_imports=[], + bridge_required_parameters={"parent_uid": "{{tasks.init.values.parent_uid}}"}, + ) + prepared = prepare_activity(activity) + bridge_notebooks = [nb for nb in prepared.notebooks if nb.relative_path.endswith("_bridge.py")] + assert len(bridge_notebooks) == 1 + assert bridge_notebooks[0].content.startswith("# Databricks notebook source") + assert "# MAGIC # IfCondition bridge: If_ParentUId" in bridge_notebooks[0].content + + def test_switch_bridge_notebook_has_header(self): + activity = SwitchActivity( + **_make_base("Route", "route"), + on_expression="@toUpper(item().type)", + cases=[SwitchCase(value="FULL", activities=[WaitActivity(**_make_base("W", "w"), wait_time_seconds=1)])], + bridge_notebook_code="str(dbutils.widgets.get('item_type')).upper()", + bridge_notebook_imports=[], + bridge_required_parameters={"item_type": "{{tasks.init.values.item_type}}"}, + ) + prepared = prepare_activity(activity) + bridge_notebooks = [nb for nb in prepared.notebooks if nb.relative_path.endswith("_bridge.py")] + assert len(bridge_notebooks) == 1 + assert bridge_notebooks[0].content.startswith("# Databricks notebook source") + assert "# MAGIC # Switch bridge: Route" in bridge_notebooks[0].content + + def test_for_each_inputs_bridge_notebook_has_header(self): + inner = WaitActivity(**_make_base("Inner", "inner"), wait_time_seconds=1) + activity = ForEachActivity( + **_make_base("Loop", "loop"), + items_expression="@split(variables('fecha'),',')", + inputs_bridge_notebook_code=( + "str(dbutils.jobs.taskValues.get(taskKey='_init_fecha', key='fecha')).split(str(','))" + ), + inputs_bridge_notebook_imports=[], + inputs_bridge_required_parameters={"fecha": "{{tasks._init_fecha.values.fecha}}"}, + inner_activities=[inner], + concurrency=10, + ) + prepared = prepare_activity(activity) + bridge_notebooks = [nb for nb in prepared.notebooks if nb.relative_path.endswith("_bridge.py")] + assert len(bridge_notebooks) == 1 + assert bridge_notebooks[0].content.startswith("# Databricks notebook source") + assert "# MAGIC # ForEach inputs bridge: Loop" in bridge_notebooks[0].content + + # --------------------------------------------------------------------------- # Notebook content validity # --------------------------------------------------------------------------- From 2c04b2a1f7130a153be1dd75b52e2811f40e545a Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Thu, 23 Jul 2026 12:40:21 -0400 Subject: [PATCH 40/77] Add handling for global parameters (#15) ## Changes Add handler methods to resolve global parameters from ADF resources. Parameter is controlled by the user via prompting. The choice is applied to an entire ADF resource during conversion. Users can choose to: - Inject resolved parameter values directly into the code or configuration during conversion - Create bundle variables and parameterize code or configuration with bundle variable references ### Linked issues N/A ### Tests - [x] manually tested - [x] added unit tests - [ ] added integration tests --- AGENTS.md | 11 ++ docs/content/docs/options.mdx | 24 ++++ skills/flowx-convert/SKILL.md | 10 +- src/flowx/adapter/constants.py | 1 + src/flowx/adapter/session.py | 12 ++ src/flowx/bundler/dab_writer.py | 48 ++++++- src/flowx/bundler/prereqs_writer.py | 27 ++++ src/flowx/models/ir.py | 8 ++ src/flowx/parser/adf_loader.py | 100 +++++++++++++- src/flowx/parser/expression_parser.py | 53 +++++-- src/flowx/parser/ir_rewriter.py | 27 +++- src/flowx/preparer/workflow_preparer.py | 2 + .../activity_translators/for_each.py | 1 + src/flowx/translator/engine.py | 74 +++++++++- tests/unit/test_adapter.py | 7 +- tests/unit/test_adf_loader.py | 122 +++++++++++++++++ tests/unit/test_bundler.py | 72 ++++++++++ tests/unit/test_expression_parser.py | 112 ++++++++++++++- tests/unit/test_translators.py | 129 ++++++++++++++++++ 19 files changed, 811 insertions(+), 29 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3bfe895..17e028b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -113,6 +113,17 @@ ExecuteDataFlow, SqlServerStoredProcedure, AzureFunction, WebHook, Custom, Execu - Leaf types return Activity only, control-flow returns (Activity, TranslationContext) - Use `parse_expression()` for ADF expression translation, return None for unsupported +### Naming, docstrings, and comments + +- Spell names out: use unabbreviated variable, parameter, function, and class names + (`parameter_values` not `params`, `whole_reference` not `whole`). Short loop indices and + regex match binders (`match`, `item`) are fine. +- Write docstrings in plain, conversational language aimed at both users and maintainers. + Say what the function does and why in everyday terms; skip jargon and marketing tone. +- Prefer self-documenting code over inline comments. Reserve comments for the non-obvious + *why* (a workaround, a spec quirk, a subtle ordering constraint) -- not for restating what + the code already says. Delete comments that narrate self-evident lines. + ## Adding a New Deterministic Translator 1. Add IR dataclass to `src/flowx/models/ir.py` diff --git a/docs/content/docs/options.mdx b/docs/content/docs/options.mdx index bbfa4f0..05e1649 100644 --- a/docs/content/docs/options.mdx +++ b/docs/content/docs/options.mdx @@ -83,6 +83,30 @@ The following are required to consolidate into a single ingestion pipeline: - The number of metadata rows or objects must be less than 250 +## global_parameter_resolution + +Controls how global parameters (`@pipeline().globalParameters.X`) are translated. This option is +applied uniformly to every pipeline in the input resource. + +| Value | Default | Behavior | +|-------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `literal` | True | Writes each global variable into the translated configuration or notebook code. | +| `bundle_variable` | False | Uses a `${var.X}` reference and declares the global variable as a [DAB bundle variable](https://docs.databricks.com/aws/en/dev-tools/bundles/variables). | + +Under `bundle_variable`, each referenced global becomes a variable in `databricks.yml` with its +factory value as the default, so a deploy with no overrides reproduces the original ADF behavior. +You then change the value per target or at deploy time +(`databricks bundle deploy -t --var "X="`). Globals referenced inside generated +notebook code are wired through the task's `base_parameters` so `${var.X}` still resolves at +runtime. + + +The default values for global parameters are written into `databricks.yml` as plain text. For sensitive +values (e.g. tokens, connection strings, URLs with embedded SAS signatures), clear the default and +supply the value at deploy time using a `--var` argument, a per-target override, or a secret scope instead +of committing sensitive values to the bundle. See `SETUP.md` for a full list of converted global variables. + + ## Discover complexity report The `discover` phase emits `/metadata/profile_report.csv` (default `./flowx_output/metadata/profile_report.csv`), one row per pipeline with the following columns: diff --git a/skills/flowx-convert/SKILL.md b/skills/flowx-convert/SKILL.md index 5cd2b43..c5e141d 100644 --- a/skills/flowx-convert/SKILL.md +++ b/skills/flowx-convert/SKILL.md @@ -95,7 +95,8 @@ Execute the translation engine on all deterministic activities: "$PY" -m flowx.translator.engine \ --source-dir \ --output-dir \ - [--pipeline ] + [--pipeline ] \ + [--global-parameter-resolution literal|bundle_variable] ``` Where: @@ -103,6 +104,13 @@ Where: - `` is the **shared migration output directory** (default: `./flowx_output`) — the same one discover used - `` (optional) — when provided, translates only the named pipeline. **Always pass `--pipeline` when the user has specified a specific pipeline to migrate**, matching the value passed to the discover phase. +- `--global-parameter-resolution` (optional, default `literal`) — how `@pipeline().globalParameters.X` + references resolve, applied to every pipeline. `literal` bakes the factory value in as a literal; + `bundle_variable` emits `${var.X}` and declares the global as a DAB bundle variable whose default is + the factory value, so it can be set at deploy time (`--var X=…` or a per-target override) instead of + being hard-coded into pipeline/activity bodies. Globals referenced inside generated notebook code are + bridged through the task's `base_parameters` so `${var.X}` still resolves. See SETUP.md for the list + of hoisted variables and a plaintext-secret caveat. The translation report and intermediate IR are written to the **transient** `/.work/` folder (`translation_report.json`, per-pipeline IR, `gaps.json`). These are consumed by the steps diff --git a/src/flowx/adapter/constants.py b/src/flowx/adapter/constants.py index 02e30aa..1e77b56 100644 --- a/src/flowx/adapter/constants.py +++ b/src/flowx/adapter/constants.py @@ -31,6 +31,7 @@ INPUT_ADF_RESOURCE_URL: Final[str] = "adf_resource_url" INPUT_OUTPUT_DIR: Final[str] = "output_dir" INPUT_INVENTORY_PATH: Final[str] = "inventory_path" +INPUT_GLOBAL_PARAMETER_RESOLUTION: Final[str] = "global_parameter_resolution" INPUT_TRANSLATION_REPORT_PATH: Final[str] = "translation_report_path" INPUT_OUTPUT_BUNDLE_PATH: Final[str] = "output_bundle_path" INPUT_CATALOG: Final[str] = "catalog" diff --git a/src/flowx/adapter/session.py b/src/flowx/adapter/session.py index 2b7509c..2813ff3 100644 --- a/src/flowx/adapter/session.py +++ b/src/flowx/adapter/session.py @@ -19,6 +19,7 @@ INPUT_BUNDLE_NAME, INPUT_CATALOG, INPUT_DATABRICKS_PROFILE, + INPUT_GLOBAL_PARAMETER_RESOLUTION, INPUT_INSTALL_DASHBOARD, INPUT_INVENTORY_PATH, INPUT_OUTPUT_BUNDLE_PATH, @@ -297,6 +298,17 @@ def _collect_motif_consolidations(self) -> dict[str, MotifConsolidate]: default="./flowx_output", required=False, ), + MigrationInputOption( + option_id=INPUT_GLOBAL_PARAMETER_RESOLUTION, + prompt="How should factory global parameters be resolved?", + description=( + "Applies to every pipeline. 'literal' bakes each @pipeline().globalParameters.X value in as a " + "literal; 'bundle_variable' emits ${var.X} and declares the global as a DAB bundle variable with " + "the factory value as its default, so it can be changed at deploy time." + ), + default="literal", + required=False, + ), ) _PACKAGE_OPTIONS: tuple[MigrationInputOption, ...] = ( diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 362d48a..5573f05 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -121,6 +121,8 @@ def write_bundle( pipeline_resources = _collect_pipeline_resources(workflow) pipeline_variable_declarations = _build_pipeline_variable_declarations(pipeline_resources, catalog, schema) + hoisted_global_variables = _collect_hoisted_global_variables(workflow) + extra_variable_declarations = {**pipeline_variable_declarations, **hoisted_global_variables} # 1. Write databricks.yml. When any task runs on classic compute, spark_version / node_type_id # defaults come from the ADF linked-service configs; when every task is serverless, they're omitted. @@ -133,7 +135,7 @@ def write_bundle( spark_version=inferred_spark_version, node_type_id=inferred_node_type_id, include_cluster_variables=bundle_uses_classic_cluster, - extra_variables=pipeline_variable_declarations, + extra_variables=extra_variable_declarations, ) databricks_yml_path.write_text( yaml.dump( @@ -156,7 +158,8 @@ def write_bundle( resources_dir = output_dir / "resources" resources_dir.mkdir(parents=True, exist_ok=True) job_yml_path = resources_dir / f"{resource_key}.yml" - job_resource = _build_job_resource(workflow, resource_key) + hoisted_global_names = set(hoisted_global_variables) + job_resource = _build_job_resource(workflow, resource_key, hoisted_globals=hoisted_global_names) job_yml_path.write_text( yaml.dump( job_resource, default_flow_style=False, sort_keys=False, allow_unicode=True, Dumper=_BundleYamlDumper @@ -170,7 +173,9 @@ def write_bundle( for inner in workflow.inner_workflows: inner_key = normalize_task_key(inner.name) inner_yml_path = resources_dir / f"{inner_key}.yml" - inner_resource = _build_job_resource(inner, inner_key, extra_notebooks_for_augment=workflow.notebooks) + inner_resource = _build_job_resource( + inner, inner_key, extra_notebooks_for_augment=workflow.notebooks, hoisted_globals=hoisted_global_names + ) inner_yml_path.write_text( yaml.dump( inner_resource, @@ -289,6 +294,7 @@ def write_bundle( manual_schedule_time_of_day=manual_schedule_time_of_day_configs, manual_credentials=manual_credential_configs, neutralized_conditions=list(_neutralized_conditions), + hoisted_global_variables=hoisted_global_variables, ) setup_path = output_dir / "SETUP.md" setup_path.write_text(render_setup_md(prereqs, bundle_name=effective_name), encoding="utf-8") @@ -797,6 +803,26 @@ def _collect_variable_references(value: Any) -> set[str]: return refs +def _collect_hoisted_global_variables(workflow: PreparedWorkflow) -> dict[str, Any]: + """Returns the bundle-variable declarations for hoisted factory globals. + + Merges ``bundle_variables`` from *workflow* and every inner workflow so a + global referenced only inside a nested (run_job_task) workflow is still + declared in the root ``variables:`` block. + + Args: + workflow: The prepared workflow being written. + + Returns: + Mapping of variable name to its DAB declaration dict. + """ + declarations: dict[str, Any] = {} + for inner in workflow.inner_workflows: + declarations.update(inner.bundle_variables) + declarations.update(workflow.bundle_variables) + return declarations + + def _build_pipeline_variable_declarations( pipeline_resources: list[dict[str, Any]], catalog: str, @@ -1204,13 +1230,20 @@ def _collect_all_task_keys(tasks: list[dict[str, Any]]) -> set[str]: return keys -def _augment_base_parameters(tasks: list[dict[str, Any]], notebooks: list[DabNotebook]) -> None: +def _augment_base_parameters( + tasks: list[dict[str, Any]], notebooks: list[DabNotebook], hoisted_globals: set[str] | None = None +) -> None: """Ensure every widget a notebook reads is declared in its base_parameters. Args: tasks: Top-level task dicts (mutated in place). notebooks: Generated notebooks to scan. + hoisted_globals: Names of factory globals hoisted to bundle variables. + A widget matching one of these binds to ``${var.NAME}`` so the + deploy-time bundle variable flows into the notebook; other widgets + default to an empty string as before. """ + hoisted = hoisted_globals or set() notebook_by_relpath = {notebook.relative_path: notebook for notebook in notebooks} def visit(task: dict[str, Any]) -> None: @@ -1223,7 +1256,8 @@ def visit(task: dict[str, Any]) -> None: widgets = set(_WIDGET_REFERENCE.findall(notebook.content)) base_parameters = notebook_task.setdefault("base_parameters", {}) for widget_name in sorted(widgets): - base_parameters.setdefault(widget_name, "") + fallback = "${var." + widget_name + "}" if widget_name in hoisted else "" + base_parameters.setdefault(widget_name, fallback) for_each = task.get("for_each_task") if for_each and isinstance(for_each.get("task"), dict): visit(for_each["task"]) @@ -1238,6 +1272,7 @@ def _build_job_resource( *, attach_clusters: bool = True, extra_notebooks_for_augment: list[DabNotebook] | None = None, + hoisted_globals: set[str] | None = None, ) -> dict[str, Any]: """Builds a job resource dict for a single workflow. @@ -1256,7 +1291,7 @@ def _build_job_resource( # For inner jobs (run_job_task), notebooks live in the parent workflow's list — pass them in so widget # auto-augment can still find the bound notebook and populate base_parameters. augment_scope = list(workflow.notebooks) + list(extra_notebooks_for_augment or []) - _augment_base_parameters(workflow.tasks, augment_scope) + _augment_base_parameters(workflow.tasks, augment_scope, hoisted_globals) # Task values don't cross run_job_task boundaries; such a reference resolves to an empty string at # runtime, so emit it now for SETUP.md §4. C-43: a blanked condition operand is always-true, so record # each neutralised condition for the SETUP.md re-wiring section. @@ -1463,6 +1498,7 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d parameters=parameters or None, translation_configuration=_reconstruct_configuration(pipeline_dict.get("translation_configuration")), schedule=pipeline_dict.get("schedule"), + bundle_variables=pipeline_dict.get("bundle_variables") or {}, ) return pipeline, parameters diff --git a/src/flowx/bundler/prereqs_writer.py b/src/flowx/bundler/prereqs_writer.py index 0149507..5d5834f 100644 --- a/src/flowx/bundler/prereqs_writer.py +++ b/src/flowx/bundler/prereqs_writer.py @@ -131,6 +131,7 @@ class Prereqs: # C-43 (CF5-001 / CF5-002): condition_task operands blanked because they referenced a task in another # job ({task_key, field, original_ref}); a blanked operand is always-true, so the user must re-wire it. neutralized_conditions: list[dict[str, str]] = field(default_factory=list) + hoisted_global_variables: dict[str, dict[str, Any]] = field(default_factory=dict) def is_empty(self) -> bool: """Return ``True`` when nothing needs to happen before ``bundle run``.""" @@ -150,6 +151,7 @@ def is_empty(self) -> bool: and not self.manual_schedule_time_of_day and not self.manual_credentials and not self.neutralized_conditions + and not self.hoisted_global_variables ) @@ -361,6 +363,7 @@ def build_prereqs( manual_schedule_time_of_day: list[dict[str, Any]] | None = None, manual_credentials: list[dict[str, Any]] | None = None, neutralized_conditions: list[dict[str, str]] | None = None, + hoisted_global_variables: dict[str, dict[str, Any]] | None = None, ) -> Prereqs: """Assemble a :class:`Prereqs` from the bundle's generated artifacts. @@ -408,6 +411,7 @@ def build_prereqs( manual_schedule_time_of_day=list(manual_schedule_time_of_day or []), manual_credentials=list(manual_credentials or []), neutralized_conditions=list(neutralized_conditions or []), + hoisted_global_variables=dict(hoisted_global_variables or {}), ) @@ -476,6 +480,29 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: ) lines.append("") + if prereqs.hoisted_global_variables: + lines.append("## Factory global parameters (bundle variables)") + lines.append("") + lines.append( + "These ADF factory global parameters were hoisted to DAB bundle variables " + "(`global_parameter_resolution=bundle_variable`). Each is declared in `databricks.yml` " + "with its factory value as the default, so a deploy with no overrides reproduces the " + "original behaviour. Override per target in `databricks.yml` or at deploy time:" + ) + lines.append("") + lines.append("```bash") + for name in sorted(prereqs.hoisted_global_variables): + lines.append(f'databricks bundle deploy -t --var "{name}="') + lines.append("```") + lines.append("") + lines.append( + "> **Security note:** the factory-value defaults are stored in plaintext in " + "`databricks.yml`. For any sensitive value (tokens, connection strings, URLs with " + "embedded SAS signatures), clear the default and supply it at deploy time via `--var`, " + "a per-target override, or a secret scope instead of committing it to the bundle." + ) + lines.append("") + if prereqs.missing_notebooks: lines.append("## Notebooks to author") lines.append("") diff --git a/src/flowx/models/ir.py b/src/flowx/models/ir.py index aa1afe3..ce6198b 100644 --- a/src/flowx/models/ir.py +++ b/src/flowx/models/ir.py @@ -532,6 +532,8 @@ class Pipeline: tasks: Ordered list of translated activities. tags: System and user-defined tags. not_translatable: Entries describing properties that could not be translated. + bundle_variables: DAB bundle-variable declarations (name -> ``{"description", "default"}``) + for factory globals hoisted under the ``bundle_variable`` resolution policy. """ name: str @@ -541,6 +543,7 @@ class Pipeline: tags: dict[str, str] = field(default_factory=dict) not_translatable: list[dict[str, Any]] = field(default_factory=list) translation_configuration: TranslationConfiguration | None = None + bundle_variables: dict[str, dict[str, Any]] = field(default_factory=dict) @dataclass(frozen=True, slots=True) @@ -562,6 +565,7 @@ class TranslationContext: variable_default_literals: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) global_parameters: MappingProxyType[str, Any] = field(default_factory=lambda: MappingProxyType({})) linked_service_parameters: MappingProxyType[str, Any] = field(default_factory=lambda: MappingProxyType({})) + global_parameter_resolution: str = "literal" def with_activity(self, name: str, activity: Activity) -> TranslationContext: """Return a new context with *activity* added to the cache. @@ -582,6 +586,7 @@ def with_activity(self, name: str, activity: Activity) -> TranslationContext: variable_default_literals=self.variable_default_literals, global_parameters=self.global_parameters, linked_service_parameters=self.linked_service_parameters, + global_parameter_resolution=self.global_parameter_resolution, ) def get_activity(self, activity_name: str) -> Activity | None: @@ -627,6 +632,7 @@ def with_variable( variable_default_literals=self.variable_default_literals, global_parameters=self.global_parameters, linked_service_parameters=self.linked_service_parameters, + global_parameter_resolution=self.global_parameter_resolution, ) def with_variable_types( @@ -661,6 +667,7 @@ def with_variable_types( variable_default_literals=MappingProxyType({**self.variable_default_literals, **(default_literals or {})}), global_parameters=self.global_parameters, linked_service_parameters=self.linked_service_parameters, + global_parameter_resolution=self.global_parameter_resolution, ) def get_variable_task_key(self, variable_name: str) -> str | None: @@ -698,6 +705,7 @@ def with_linked_service_parameters(self, params: dict[str, Any]) -> TranslationC variable_default_literals=self.variable_default_literals, global_parameters=self.global_parameters, linked_service_parameters=MappingProxyType(dict(params)), + global_parameter_resolution=self.global_parameter_resolution, ) def get_global_parameter(self, name: str) -> Any: diff --git a/src/flowx/parser/adf_loader.py b/src/flowx/parser/adf_loader.py index ab760b3..af9917a 100644 --- a/src/flowx/parser/adf_loader.py +++ b/src/flowx/parser/adf_loader.py @@ -104,7 +104,7 @@ def load_adf_definitions(source_dir: Path) -> AdfDefinitions: source_dir = Path(source_dir).resolve() if source_dir.is_file() and source_dir.suffix == ".json": - return _load_arm_template(source_dir) + return _load_arm_template(source_dir, parameters_path=_find_arm_parameters_file(source_dir)) pipelines: list[AdfPipeline] = [] datasets: dict[str, AdfDataset] = {} @@ -569,11 +569,96 @@ def _normalize_arm(data: dict[str, Any]) -> dict[str, Any]: return data -def _load_arm_template(template_path: Path) -> AdfDefinitions: +# Matches an ARM string that is *entirely* a single parameter reference, e.g. +# ``[parameters('ls_kv_central_..._baseUrl')]`` -- captures the parameter name. +_ARM_PARAM_WHOLE_RE = re.compile(r"^\[parameters\('([^']+)'\)\]$") +# Matches a parameter reference embedded inside a larger ARM expression. +_ARM_PARAM_TOKEN_RE = re.compile(r"parameters\('([^']+)'\)") + + +def _find_arm_parameters_file(template_path: Path) -> Path | None: + """Finds the ``*ParametersForFactory.json`` that sits next to an ARM template. + + Args: + template_path: Path to the ARM template JSON file. + + Returns: + The matching parameters file, or ``None`` if there isn't one. + """ + directory = template_path.parent + # ARMTemplateForFactory.json pairs with ARMTemplateParametersForFactory.json. + expected_name = directory / template_path.name.replace("ForFactory", "ParametersForFactory") + if expected_name != template_path and expected_name.is_file(): + return expected_name + other_matches = sorted(directory.glob("*ParametersForFactory.json")) + return other_matches[0] if other_matches else None + + +def _load_arm_parameter_values(parameters_path: Path) -> dict[str, Any]: + """Reads an ``*ParametersForFactory.json`` file into a name -> value map. + + Args: + parameters_path: Path to the ARM parameters file. + + Returns: + Each parameter name mapped to its value. Parameters with no value + (the ones filled in at deploy time) are skipped. + """ + try: + data = json.loads(parameters_path.read_text(encoding="utf-8")) + except Exception: + logger.exception("Failed to read ARM parameters file %s", parameters_path) + return {} + parameters = data.get("parameters", {}) + if not isinstance(parameters, dict): + return {} + return {name: entry["value"] for name, entry in parameters.items() if isinstance(entry, dict) and "value" in entry} + + +def _resolve_arm_parameters(value: Any, parameter_values: dict[str, Any]) -> Any: + """Walks a JSON structure and fills in ``parameters('X')`` references. + + If a string is nothing but a single reference (``[parameters('X')]``), it + becomes the parameter's value as-is, keeping its original type (an int stays + an int). If a reference sits inside a larger ARM expression, only the + ``parameters('X')`` token is swapped for the string value -- the rest of the + expression (e.g. ``concat(...)``) is left alone, since flowx does not run ARM + expressions. References whose name is not in the parameters file are left as-is. + + Args: + value: Any JSON-decoded value (dict, list, string, or scalar). + parameter_values: Parameter name -> value map from the parameters file. + + Returns: + A copy of *value* with every resolvable reference filled in. + """ + if not parameter_values: + return value + if isinstance(value, dict): + return {key: _resolve_arm_parameters(item, parameter_values) for key, item in value.items()} + if isinstance(value, list): + return [_resolve_arm_parameters(item, parameter_values) for item in value] + if isinstance(value, str): + whole_reference = _ARM_PARAM_WHOLE_RE.match(value) + if whole_reference is not None and whole_reference.group(1) in parameter_values: + return parameter_values[whole_reference.group(1)] + return _ARM_PARAM_TOKEN_RE.sub( + lambda match: ( + str(parameter_values[match.group(1)]) if match.group(1) in parameter_values else match.group(0) + ), + value, + ) + return value + + +def _load_arm_template(template_path: Path, parameters_path: Path | None = None) -> AdfDefinitions: """Loads all ADF resources from a single ARM template file. Args: template_path: Path to the ARM template JSON file. + parameters_path: Optional path to the sibling + ``*ParametersForFactory.json`` file. When provided, ``parameters('X')`` + references in resource bodies are resolved to their concrete values. Returns: Parsed :class:`AdfDefinitions`. @@ -581,6 +666,8 @@ def _load_arm_template(template_path: Path) -> AdfDefinitions: data = json.loads(template_path.read_text(encoding="utf-8")) resources = data.get("resources", []) + parameter_values = _load_arm_parameter_values(parameters_path) if parameters_path is not None else {} + pipelines: list[AdfPipeline] = [] datasets: dict[str, AdfDataset] = {} linked_services: dict[str, AdfLinkedService] = {} @@ -589,7 +676,7 @@ def _load_arm_template(template_path: Path) -> AdfDefinitions: for resource in resources: rtype = resource.get("type", "") - props = resource.get("properties", {}) + props = _resolve_arm_parameters(resource.get("properties", {}), parameter_values) raw_name = resource.get("name", "") if "/" in raw_name: name = raw_name.rsplit("/", 1)[-1].strip("'])") @@ -598,7 +685,12 @@ def _load_arm_template(template_path: Path) -> AdfDefinitions: wrapped = {"name": name, "properties": props} - if rtype.endswith("/pipelines"): + if rtype.endswith("/globalparameters"): + try: + global_parameters.update(props) + except Exception: + logger.exception("Failed to parse ARM global-parameters resource %s", name) + elif rtype.endswith("/pipelines"): try: pipelines.append(_parse_pipeline_json(wrapped, fallback_name=name)) except Exception: diff --git a/src/flowx/parser/expression_parser.py b/src/flowx/parser/expression_parser.py index 2bec906..20c6708 100644 --- a/src/flowx/parser/expression_parser.py +++ b/src/flowx/parser/expression_parser.py @@ -244,6 +244,11 @@ def _replace_match(match: re.Match[str]) -> str: return _INTERPOLATION_RE.sub(_replace_match, value) +def _escape_fstring_braces(text: str) -> str: + """Doubles ``{`` / ``}`` so *text* is safe as literal content inside an f-string.""" + return text.replace("{", "{{").replace("}", "}}") + + def resolve_interpolated_string_for_notebook( value: str, context: TranslationContext, @@ -252,26 +257,36 @@ def resolve_interpolated_string_for_notebook( ) -> str: """Resolves ``@{...}`` tokens to Python f-string expressions for notebook code. + The result is meant to be embedded in an f-string by the caller, so every + ``{`` / ``}`` that is *not* part of a generated f-string field (literal + values and the static text around the tokens) is doubled, keeping the + surrounding f-string valid when a value carries braces (e.g. JSON). + Args: value: A string containing ``@{...}`` tokens. context: Translation context for resolving variables. variable_task_keys: Optional explicit variable-name-to-task-key map. Returns: - A string with ``@{...}`` tokens replaced by Python f-string expressions. + A string with ``@{...}`` tokens replaced by Python f-string expressions + and all other braces escaped for safe f-string embedding. """ if not isinstance(value, str) or "@{" not in value: return value - def _replace_match(match: re.Match[str]) -> str: - inner_expr = match.group(1) + def _resolve_token(inner_expr: str) -> str: result = resolve_expression("@" + inner_expr, context, variable_task_keys=variable_task_keys) if result is None: - return match.group(0) + return _escape_fstring_braces("@{" + inner_expr + "}") if result.kind == "literal": - return result.value + # Literal text sits outside any f-string field, so brace characters in the value + # (e.g. a JSON template) must be escaped or they break the surrounding f-string. + return _escape_fstring_braces(result.value) if result.kind == "dab_ref": ref = result.value + var_match = re.match(r"\$\{var\.(\w+)\}", ref) + if var_match: + return "{dbutils.widgets.get('" + var_match.group(1) + "')}" param_match = re.match(r"\{\{job\.parameters\.(\w+)\}\}", ref) if param_match: return "{dbutils.widgets.get('" + param_match.group(1) + "')}" @@ -295,7 +310,16 @@ def _replace_match(match: re.Match[str]) -> str: return ref return "{" + result.value + "}" - return _INTERPOLATION_RE.sub(_replace_match, value) + # Walk the string ourselves rather than using re.sub so the static text *between* @{...} tokens + # (which may itself contain braces, e.g. a surrounding JSON body) also gets f-string-escaped. + parts: list[str] = [] + cursor = 0 + for match in _INTERPOLATION_RE.finditer(value): + parts.append(_escape_fstring_braces(value[cursor : match.start()])) + parts.append(_resolve_token(match.group(1))) + cursor = match.end() + parts.append(_escape_fstring_braces(value[cursor:])) + return "".join(parts) def parse_expression(value: str | dict[str, Any] | int | float | bool, context: TranslationContext) -> str | None: @@ -349,17 +373,24 @@ def _resolve_pipeline_param(expr: str) -> ExpressionResult | None: def _resolve_pipeline_global_param(expr: str, context: TranslationContext) -> ExpressionResult | None: """Resolves ``pipeline().globalParameters.X`` against factory globals. - When ``context.global_parameters`` carries a concrete value for *X* - the expression collapses to a literal so downstream callers (notably - ``concat`` reductions) get the actual factory value baked in. When - no factory value is available we fall back to a job-parameter DAB - ref so the bundle YAML can supply it. + Under the default ``literal`` policy, when ``context.global_parameters`` + carries a concrete value for *X* the expression collapses to a literal + so downstream callers (notably ``concat`` reductions) get the actual + factory value baked in. Under the ``bundle_variable`` policy the + reference lowers to a ``${var.X}`` DAB ref -- even when a concrete value + exists -- so the value becomes a bundle variable (the engine declares it + with the factory value as its default) that can be set at deploy time + rather than being hard-coded into pipeline/activity bodies. When no + factory value is available we fall back to a ``{{job.parameters.X}}`` ref + regardless of policy so the bundle YAML can supply it. """ match = _PIPELINE_GLOBAL_PARAM_RE.match(expr) if match is None: return None param_name = match.group(1) value = context.get_global_parameter(param_name) + if context.global_parameter_resolution == "bundle_variable" and value is not None: + return ExpressionResult(kind="dab_ref", value="${var." + param_name + "}") if value is None: return ExpressionResult(kind="dab_ref", value="{{" + f"job.parameters.{param_name}" + "}}") return ExpressionResult(kind="literal", value=str(value)) diff --git a/src/flowx/parser/ir_rewriter.py b/src/flowx/parser/ir_rewriter.py index 0fc536f..6213fa8 100644 --- a/src/flowx/parser/ir_rewriter.py +++ b/src/flowx/parser/ir_rewriter.py @@ -65,6 +65,8 @@ def rewrite_pipeline_expressions( pipeline: Pipeline, *, warnings: list[str] | None = None, + global_parameters: dict[str, Any] | None = None, + global_parameter_resolution: str = "literal", ) -> Pipeline: """Walks every string field in *pipeline* and rewrites ``@{...}`` tokens. @@ -74,6 +76,13 @@ def rewrite_pipeline_expressions( for every string field that still contains an unresolved ``@{...}`` token after the pass. When ``None`` the rewriter still runs but cannot surface gaps. + global_parameters: Factory-level global parameters, so + ``@{pipeline().globalParameters.X}`` tokens embedded in raw SQL + / REST bodies resolve the same way the per-activity translators + resolve them. + global_parameter_resolution: ``"literal"`` or ``"bundle_variable"`` -- + forwarded to the context so this pass honours the same + global-parameter policy as ``translate_pipeline``. Returns: A new :class:`Pipeline` whose activities have had their string @@ -91,18 +100,30 @@ def rewrite_pipeline_expressions( - Unknown field types (ints, bools, None, custom dataclasses beyond Activity/SwitchCase) pass through unchanged. """ - context = _build_context_from_pipeline(pipeline) + context = _build_context_from_pipeline( + pipeline, + global_parameters=global_parameters, + global_parameter_resolution=global_parameter_resolution, + ) sink: list[str] = warnings if warnings is not None else [] rewritten_tasks = [_rewrite_activity(activity, context, sink) for activity in pipeline.tasks] return dataclasses.replace(pipeline, tasks=rewritten_tasks) -def _build_context_from_pipeline(pipeline: Pipeline) -> TranslationContext: +def _build_context_from_pipeline( + pipeline: Pipeline, + global_parameters: dict[str, Any] | None = None, + global_parameter_resolution: str = "literal", +) -> TranslationContext: """Builds a TranslationContext whose variable_cache is keyed by every SetVariable / AppendVariable activity in the pipeline. Args: pipeline: Translated pipeline IR. + global_parameters: Factory-level global parameters to seed so + ``@{pipeline().globalParameters.X}`` tokens resolve here too. + global_parameter_resolution: ``"literal"`` or ``"bundle_variable"`` + global-parameter policy carried on the context. Returns: A :class:`TranslationContext` with ``variable_cache`` populated. @@ -125,6 +146,8 @@ def visit(activities: list[Activity]) -> None: activity_cache=MappingProxyType({}), registry=MappingProxyType({}), variable_cache=MappingProxyType(variable_cache), + global_parameters=MappingProxyType(dict(global_parameters or {})), + global_parameter_resolution=global_parameter_resolution, ) diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py index 6148cbf..d86efcb 100644 --- a/src/flowx/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -66,6 +66,7 @@ class PreparedWorkflow: # C-10 (SCHED-001): serialised schedule / trigger spec the bundler # renders as ``schedule:`` / ``trigger:`` on the emitted DAB job. schedule: dict[str, Any] | None = None + bundle_variables: dict[str, dict[str, Any]] = field(default_factory=dict) def run_if_from_adf_outcomes(outcomes: list[str | None]) -> str | None: @@ -397,6 +398,7 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: pipeline_resources=list(artifacts.pipeline_resources), parameter_approximations=list(artifacts.parameter_approximations), schedule=pipeline.schedule, + bundle_variables=dict(pipeline.bundle_variables), ) diff --git a/src/flowx/translator/activity_translators/for_each.py b/src/flowx/translator/activity_translators/for_each.py index 12866d1..671e37b 100644 --- a/src/flowx/translator/activity_translators/for_each.py +++ b/src/flowx/translator/activity_translators/for_each.py @@ -81,6 +81,7 @@ def translate( variable_value_cache=context.variable_value_cache, global_parameters=context.global_parameters, linked_service_parameters=context.linked_service_parameters, + global_parameter_resolution=context.global_parameter_resolution, ) inner_activities, _ = translate_activities_fn(child_adf_activities, child_context, definitions) diff --git a/src/flowx/translator/engine.py b/src/flowx/translator/engine.py index b45e30e..e2cdb05 100644 --- a/src/flowx/translator/engine.py +++ b/src/flowx/translator/engine.py @@ -7,7 +7,7 @@ import logging import re from collections import defaultdict -from dataclasses import asdict +from dataclasses import asdict, replace from datetime import datetime from pathlib import Path from types import MappingProxyType @@ -91,6 +91,7 @@ def translate_pipeline( definitions: AdfDefinitions, *, motif_consolidations: dict[str, str] | None = None, + global_parameter_resolution: str = "literal", ) -> TranslationReport: """Translates an ADF pipeline into a Databricks pipeline IR. @@ -104,6 +105,13 @@ def translate_pipeline( consolidates every detected motif. When provided, only motifs whose id maps to ``"consolidate"`` are collapsed; the rest remain as the original activity-by-activity translation. + global_parameter_resolution: How ``@pipeline().globalParameters.X`` + references resolve. ``"literal"`` (default) bakes the concrete + factory value in as a literal; ``"bundle_variable"`` lowers each + reference to ``${var.X}`` and declares the global as a DAB + bundle variable (with the factory value as its default) so the + value is set at deploy time instead of being hard-coded into + pipeline/activity bodies. Returns: :class:`TranslationReport` containing the translated :class:`Pipeline`, @@ -124,6 +132,7 @@ def translate_pipeline( registry=MappingProxyType(TRANSLATOR_REGISTRY), variable_cache=MappingProxyType({}), global_parameters=MappingProxyType(dict(definitions.global_parameters)), + global_parameter_resolution=global_parameter_resolution, ) # C-41 (CF5-001): seed declared variable types so the IfCondition fallback recognises Boolean @@ -193,7 +202,15 @@ def translate_pipeline( # Whole-IR expression rewrite: catches @{...} tokens the per-activity translators missed (raw SQL # WHERE, REST bodies, dataset folder paths, ...). Unresolved tokens become translation warnings. - pipeline_ir = rewrite_pipeline_expressions(pipeline_ir, warnings=warnings) + pipeline_ir = rewrite_pipeline_expressions( + pipeline_ir, + warnings=warnings, + global_parameters=definitions.global_parameters, + global_parameter_resolution=global_parameter_resolution, + ) + + if global_parameter_resolution == "bundle_variable": + pipeline_ir = _declare_referenced_globals_as_bundle_variables(pipeline_ir, definitions.global_parameters) # Motif detection. Collapsing is gated on motif_consolidations: None preserves back-compat (collapse # every detected motif), otherwise only motifs mapped to "consolidate" are collapsed. @@ -227,6 +244,44 @@ def translate_pipeline( ) +_VAR_REF_RE: re.Pattern[str] = re.compile(r"\$\{var\.([A-Za-z0-9_]+)\}") + + +def _unwrap_global_value(raw_value: Any) -> Any: + """Returns a factory global's plain value, unwrapping the ARM ``{"type", "value"}`` shape.""" + if isinstance(raw_value, dict) and "value" in raw_value: + return raw_value["value"] + return raw_value + + +def _declare_referenced_globals_as_bundle_variables( + pipeline_ir: Pipeline, global_parameters: dict[str, Any] +) -> Pipeline: + """Declares a bundle variable for every global that was turned into ``${var.X}``. + + Args: + pipeline_ir: Translated pipeline IR (already expression-rewritten). + global_parameters: Factory-level global parameters. + + Returns: + A new :class:`Pipeline` whose ``bundle_variables`` holds a declaration + (with the factory value as its ``default``) for each hoisted global, or + the pipeline unchanged when none are referenced. + """ + referenced_names = set(_VAR_REF_RE.findall(json.dumps(_pipeline_to_dict(pipeline_ir), default=str))) + declarations = { + name: { + "description": f"Factory global parameter '{name}' (override at deploy time).", + "default": _unwrap_global_value(global_parameters[name]), + } + for name in sorted(referenced_names) + if name in global_parameters + } + if not declarations: + return pipeline_ir + return replace(pipeline_ir, bundle_variables={**pipeline_ir.bundle_variables, **declarations}) + + _OPT_IN_ONLY_MOTIFS: frozenset[str] = frozenset({"activity_and_notify"}) @@ -1275,6 +1330,8 @@ def _pipeline_to_dict(pipeline: Pipeline) -> dict[str, Any]: "tags": pipeline.tags, "tasks": [_activity_to_dict(task) for task in pipeline.tasks], } + if pipeline.bundle_variables: + result["bundle_variables"] = pipeline.bundle_variables if pipeline.translation_configuration is not None: result["translation_configuration"] = _configuration_to_dict(pipeline.translation_configuration) return result @@ -1706,6 +1763,17 @@ def main(argv: list[str] | None = None) -> int: default=None, help="Translate only the named pipeline (default: all).", ) + parser.add_argument( + "--global-parameter-resolution", + choices=("literal", "bundle_variable"), + default="literal", + help=( + "How @pipeline().globalParameters.X references resolve. 'literal' (default) bakes the " + "factory value in as a literal; 'bundle_variable' emits ${var.X} and declares the global " + "as a DAB bundle variable with the factory value as its default, so it can be set at " + "deploy time." + ), + ) parser.add_argument( "--debug", action="store_true", @@ -1770,7 +1838,7 @@ def main(argv: list[str] | None = None) -> int: if args.pipeline and pipeline.name != args.pipeline: continue - report = translate_pipeline(pipeline, definitions) + report = translate_pipeline(pipeline, definitions, global_parameter_resolution=args.global_parameter_resolution) total_deterministic += report.deterministic_count total_agentic += report.agentic_count total_unsupported += report.unsupported_count diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index f283c1a..37e89cf 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -535,9 +535,14 @@ def test_convert_session_lists_expected_options(self): from flowx.adapter import MigrationInputSession session = MigrationInputSession(phase="convert") - ids = [q.option_id for q in session.pending().options] + options = session.pending().options + ids = [option.option_id for option in options] assert "inventory_path" in ids assert "adf_source_path" in ids + assert "global_parameter_resolution" in ids + # The default must match the engine's accepted values (literal / bundle_variable). + resolution_option = next(o for o in options if o.option_id == "global_parameter_resolution") + assert resolution_option.default == "literal" def test_package_session_lists_expected_options(self): from flowx.adapter import MigrationInputSession diff --git a/tests/unit/test_adf_loader.py b/tests/unit/test_adf_loader.py index 8213742..a4c6dc0 100644 --- a/tests/unit/test_adf_loader.py +++ b/tests/unit/test_adf_loader.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json + from flowx.models.adf_ast import ( AdfDefinitions, TranslationStrategy, @@ -9,8 +11,11 @@ from flowx.parser.adf_loader import ( AGENTIC_TYPES, DETERMINISTIC_TYPES, + _find_arm_parameters_file, + _load_arm_parameter_values, _normalize_arm, _parse_pipeline_json, + _resolve_arm_parameters, build_inventory, classify_activity, clear_stale_outputs, @@ -294,6 +299,123 @@ def test_normalize_arm_passthrough(self): assert result is data +class TestLoadArmTemplate: + """Single-file ARM template ingestion, including global params and parameter resolution.""" + + def _write_template(self, tmp_path, resources): + template = tmp_path / "ARMTemplateForFactory.json" + template.write_text( + json.dumps( + { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "resources": resources, + } + ), + encoding="utf-8", + ) + return template + + def test_global_parameters_resource_is_loaded(self, tmp_path): + """A standalone /globalparameters resource populates global_parameters.""" + template = self._write_template( + tmp_path, + [ + { + "type": "Microsoft.DataFactory/factories/globalparameters", + "name": "[concat(parameters('factoryName'), '/default')]", + "properties": {"env": {"type": "string", "value": "prod"}}, + } + ], + ) + defs = load_adf_definitions(template) + assert defs.global_parameters == {"env": {"type": "string", "value": "prod"}} + + def test_parameter_references_are_resolved(self, tmp_path): + """parameters('X') references resolve from the sibling parameters file.""" + self._write_template( + tmp_path, + [ + { + "type": "Microsoft.DataFactory/factories/linkedServices", + "name": "[concat(parameters('factoryName'), '/ls_kv')]", + "properties": { + "type": "AzureKeyVault", + "typeProperties": {"baseUrl": "[parameters('ls_kv_baseUrl')]"}, + }, + } + ], + ) + (tmp_path / "ARMTemplateParametersForFactory.json").write_text( + json.dumps({"parameters": {"ls_kv_baseUrl": {"value": "https://kv.vault.azure.net/"}}}), + encoding="utf-8", + ) + defs = load_adf_definitions(tmp_path / "ARMTemplateForFactory.json") + ls = defs.linked_services["ls_kv"] + assert ls.properties["typeProperties"]["baseUrl"] == "https://kv.vault.azure.net/" + + def test_unknown_parameter_reference_is_left_intact(self, tmp_path): + """A reference with no matching parameter value is not substituted.""" + template = self._write_template( + tmp_path, + [ + { + "type": "Microsoft.DataFactory/factories/linkedServices", + "name": "[concat(parameters('factoryName'), '/ls_x')]", + "properties": { + "type": "AzureKeyVault", + "typeProperties": {"baseUrl": "[parameters('missing')]"}, + }, + } + ], + ) + defs = load_adf_definitions(template) + assert defs.linked_services["ls_x"].properties["typeProperties"]["baseUrl"] == "[parameters('missing')]" + + +class TestArmParameterHelpers: + """Direct coverage for the ARM parameters-file discovery / substitution helpers.""" + + def test_find_parameters_file_exact_name(self, tmp_path): + template = tmp_path / "ARMTemplateForFactory.json" + template.write_text("{}", encoding="utf-8") + params = tmp_path / "ARMTemplateParametersForFactory.json" + params.write_text("{}", encoding="utf-8") + assert _find_arm_parameters_file(template) == params + + def test_find_parameters_file_glob_fallback(self, tmp_path): + template = tmp_path / "weird.json" + template.write_text("{}", encoding="utf-8") + params = tmp_path / "myParametersForFactory.json" + params.write_text("{}", encoding="utf-8") + assert _find_arm_parameters_file(template) == params + + def test_find_parameters_file_none_when_absent(self, tmp_path): + template = tmp_path / "only.json" + template.write_text("{}", encoding="utf-8") + assert _find_arm_parameters_file(template) is None + + def test_load_parameter_values_skips_entries_without_value(self, tmp_path): + params = tmp_path / "p.json" + params.write_text( + json.dumps({"parameters": {"a": {"value": "1"}, "b": {"type": "string"}}}), + encoding="utf-8", + ) + assert _load_arm_parameter_values(params) == {"a": "1"} + + def test_resolve_whole_string_ref_preserves_type(self): + # A whole-string reference returns the value verbatim (int stays int). + assert _resolve_arm_parameters("[parameters('n')]", {"n": 5}) == 5 + + def test_resolve_embedded_ref_substitutes_token_only(self): + # An embedded reference replaces just the parameters('x') token with the string value; + # surrounding ARM syntax (here the brackets) is left intact since flowx does not evaluate it. + assert _resolve_arm_parameters("prefix-[parameters('x')]-suffix", {"x": "REAL"}) == "prefix-[REAL]-suffix" + + def test_resolve_recurses_into_nested_structures(self): + obj = {"a": ["[parameters('x')]", {"b": "[parameters('x')]"}]} + assert _resolve_arm_parameters(obj, {"x": "V"}) == {"a": ["V", {"b": "V"}]} + + # --------------------------------------------------------------------------- # clear_stale_outputs # --------------------------------------------------------------------------- diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index 90ecb02..001bf73 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -1034,3 +1034,75 @@ def test_switch_bridge_fields_preserved(self): assert isinstance(task, SwitchActivity) assert task.bridge_notebook_code == "result = item.get('type', 'default').upper()" assert task.bridge_required_parameters == {"item": "{{tasks.upstream.values.row}}"} + + +class TestHoistedGlobalVariables: + """bundle_variable resolution: hoisted globals declared as DAB variables + SETUP note.""" + + def _workflow_with_hoisted_global(self): + pipeline = Pipeline( + name="pl_hoisted", + tasks=[ + NotebookActivity( + name="Run NB", + task_key="run_nb", + notebook_path="/Shared/ETL/transform", + base_parameters={"env": "dev"}, + ) + ], + bundle_variables={ + "env": {"description": "Factory global parameter 'env' (override at deploy time).", "default": "prod"} + }, + ) + return prepare_workflow(pipeline) + + def test_variable_declared_in_databricks_yml(self, tmp_path): + wf = self._workflow_with_hoisted_global() + write_bundle(wf, tmp_path, catalog="main", schema="ingest") + content = yaml.safe_load((tmp_path / "databricks.yml").read_text()) + assert content["variables"]["env"]["default"] == "prod" + + def test_setup_md_lists_hoisted_variable_and_secret_note(self, tmp_path): + wf = self._workflow_with_hoisted_global() + write_bundle(wf, tmp_path, catalog="main", schema="ingest") + setup = (tmp_path / "SETUP.md").read_text() + assert "Factory global parameters (bundle variables)" in setup + assert "--var" in setup + assert "Security note" in setup + + def test_augment_base_parameters_binds_hoisted_widget_to_var_ref(self): + """A notebook widget matching a hoisted global binds to ${var.X}, others to ''.""" + from flowx.bundler.dab_writer import _augment_base_parameters + from flowx.models.dab import DabNotebook + + notebook = DabNotebook( + relative_path="nb/run.py", + content="env = dbutils.widgets.get('env')\nother = dbutils.widgets.get('other')\n", + ) + tasks = [{"notebook_task": {"notebook_path": "../src/nb/run.py"}}] + _augment_base_parameters(tasks, [notebook], hoisted_globals={"env"}) + base = tasks[0]["notebook_task"]["base_parameters"] + assert base["env"] == "${var.env}" + assert base["other"] == "" + + def test_prepare_workflow_threads_bundle_variables(self): + """prepare_workflow copies Pipeline.bundle_variables onto the PreparedWorkflow.""" + pipeline = Pipeline( + name="pl", + tasks=[NotebookActivity(name="n", task_key="n", notebook_path="/x")], + bundle_variables={"env": {"description": "d", "default": "prod"}}, + ) + wf = prepare_workflow(pipeline) + assert wf.bundle_variables == {"env": {"description": "d", "default": "prod"}} + + def test_prereqs_not_empty_when_only_hoisted_globals(self): + """is_empty() must return False when the only prereq is hoisted globals, + otherwise render_setup_md would skip the section entirely.""" + from flowx.bundler.prereqs_writer import Prereqs, render_setup_md + + prereqs = Prereqs(hoisted_global_variables={"env": {"description": "d", "default": "prod"}}) + assert not prereqs.is_empty() + md = render_setup_md(prereqs, bundle_name="b") + assert "Factory global parameters (bundle variables)" in md + assert "Security note" in md + assert "env=" in md diff --git a/tests/unit/test_expression_parser.py b/tests/unit/test_expression_parser.py index 5df2cc4..a842eff 100644 --- a/tests/unit/test_expression_parser.py +++ b/tests/unit/test_expression_parser.py @@ -9,6 +9,7 @@ parse_expression, parse_expression_for_dab, resolve_expression, + resolve_interpolated_string_for_notebook, ) @@ -887,9 +888,10 @@ def test_parse_expression_for_dab_returns_none_for_non_expression(self): class TestGlobalParameters: """Change expr-resolver-globalparams-and-wrappers (P0).""" - def _ctx_with_globals(self, **globals_) -> TranslationContext: + def _ctx_with_globals(self, *, resolution: str = "literal", **globals_) -> TranslationContext: return TranslationContext( global_parameters=MappingProxyType(dict(globals_)), + global_parameter_resolution=resolution, ) def test_global_parameter_resolves_to_literal(self): @@ -928,6 +930,114 @@ def test_concat_with_globals_resolves_fully(self): assert result.kind == "literal" assert result.value == "/Volumes/datahub01t/x/myjar.jar" + def test_global_parameter_bundle_variable_mode_emits_var_ref(self): + ctx = self._ctx_with_globals(resolution="bundle_variable", env_variable="t") + result = resolve_expression("@pipeline().globalParameters.env_variable", ctx) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "${var.env_variable}" + + def test_global_parameter_bundle_variable_mode_dict_value(self): + ctx = self._ctx_with_globals(resolution="bundle_variable", env_variable={"type": "string", "value": "t"}) + result = resolve_expression("@pipeline().globalParameters.env_variable", ctx) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "${var.env_variable}" + + def test_global_parameter_bundle_variable_missing_still_falls_back_to_job_param(self): + ctx = self._ctx_with_globals(resolution="bundle_variable") + result = resolve_expression("@pipeline().globalParameters.absent", ctx) + assert result is not None + assert result.kind == "dab_ref" + assert result.value == "{{job.parameters.absent}}" + + def test_concat_with_globals_bundle_variable_mode_bridges_widgets(self): + ctx = self._ctx_with_globals(resolution="bundle_variable", env_variable="t", libFileName="myjar.jar") + expr = ( + "@concat('/Volumes/datahub01', pipeline().globalParameters.env_variable, " + "'/x/', pipeline().globalParameters.libFileName)" + ) + result = resolve_expression(expr, ctx) + assert result is not None + assert result.kind == "notebook_code" + assert "dbutils.widgets.get('env_variable')" in result.value + assert "dbutils.widgets.get('libFileName')" in result.value + assert result.required_parameters == { + "env_variable": "${var.env_variable}", + "libFileName": "${var.libFileName}", + } + + def test_notebook_interpolation_bridges_var_ref_to_widget(self): + ctx = self._ctx_with_globals(resolution="bundle_variable", env_variable="t") + resolved = resolve_interpolated_string_for_notebook("path/@{pipeline().globalParameters.env_variable}/end", ctx) + assert resolved == "path/{dbutils.widgets.get('env_variable')}/end" + + def test_function_wrapped_global_literal_mode_bakes_value_into_python(self): + """A global inside a Python-requiring function (e.g. @split) lowers to + notebook_code with the factory value baked in as a string literal.""" + ctx = self._ctx_with_globals(path="/a/b/c") + result = resolve_expression("@split(pipeline().globalParameters.path, '/')", ctx) + assert result is not None + assert result.kind == "notebook_code" + assert "'/a/b/c'" in result.value + assert "dbutils.widgets.get" not in result.value + + def test_function_wrapped_global_bundle_variable_mode_uses_widget(self): + """The same function wrapper under bundle_variable resolves the global to + a widget read (bound to ${var.X}) rather than a baked-in literal.""" + ctx = self._ctx_with_globals(resolution="bundle_variable", env="PROD") + result = resolve_expression("@toLower(pipeline().globalParameters.env)", ctx) + assert result is not None + assert result.kind == "notebook_code" + assert "dbutils.widgets.get('env')" in result.value + assert result.required_parameters == {"env": "${var.env}"} + + def test_function_wrapped_global_in_interpolated_string_literal_mode(self): + """In literal mode a function-wrapped global inside @{...} bakes the factory + value into the f-string expression rather than reading a widget.""" + ctx = self._ctx_with_globals(env="PROD") + resolved = resolve_interpolated_string_for_notebook( + "prefix-@{toLower(pipeline().globalParameters.env)}-suffix", ctx + ) + assert resolved == "prefix-{str('PROD').lower()}-suffix" + + def test_function_wrapped_global_in_interpolated_string_bundle_variable_mode(self): + """A function-wrapped global inside @{...} lowers to an f-string widget read.""" + ctx = self._ctx_with_globals(resolution="bundle_variable", env="PROD") + resolved = resolve_interpolated_string_for_notebook( + "prefix-@{toLower(pipeline().globalParameters.env)}-suffix", ctx + ) + assert resolved == "prefix-{str(dbutils.widgets.get('env')).lower()}-suffix" + + def test_braces_in_resolved_literal_are_escaped_for_fstring(self): + """A global whose literal value contains braces must be doubled so the caller's + f-string stays valid (would otherwise be read as an f-string field).""" + ctx = self._ctx_with_globals(tmpl="prefix-{region}-suffix") + resolved = resolve_interpolated_string_for_notebook("x=@{pipeline().globalParameters.tmpl}/end", ctx) + assert resolved == "x=prefix-{{region}}-suffix/end" + # And it round-trips through an actual f-string back to the original value. + assert eval(f"f{__import__('json').dumps(resolved)}") == "x=prefix-{region}-suffix/end" # noqa: S307 + + def test_braces_in_surrounding_literal_text_are_escaped(self): + """Braces in the static text around a token (e.g. a JSON body) are also escaped, + while the resolved widget field stays a single-brace f-string expression.""" + ctx = self._ctx_with_globals(resolution="bundle_variable", env="PROD") + resolved = resolve_interpolated_string_for_notebook('{"e": "@{pipeline().globalParameters.env}"}', ctx) + assert resolved == '{{"e": "{dbutils.widgets.get(\'env\')}"}}' + + def test_resolution_policy_survives_context_with_methods(self): + """Every with_* reconstruction preserves global_parameter_resolution.""" + ctx = self._ctx_with_globals(resolution="bundle_variable", env="PROD") + ctx = ctx.with_activity("a", object()) # type: ignore[arg-type] + ctx = ctx.with_variable("v", "task_v") + ctx = ctx.with_variable_types({"v": "String"}) + ctx = ctx.with_linked_service_parameters({"p": "1"}) + assert ctx.global_parameter_resolution == "bundle_variable" + # And the global is still resolvable after all the reconstructions. + result = resolve_expression("@pipeline().globalParameters.env", ctx) + assert result is not None + assert result.value == "${var.env}" + class TestNoopWrappers: """Change expr-resolver-globalparams-and-wrappers (P0): @json/@string/@array.""" diff --git a/tests/unit/test_translators.py b/tests/unit/test_translators.py index 92d52d4..04af300 100644 --- a/tests/unit/test_translators.py +++ b/tests/unit/test_translators.py @@ -1664,6 +1664,135 @@ def test_translate_switch_function_call_routes_through_bridge(self): assert result.on_expression.startswith("__BRIDGE__::") +class TestGlobalParameterResolution: + """Engine-level global_parameter_resolution policy (literal vs bundle_variable).""" + + def _pipeline_with_global(self): + from flowx.models.adf_ast import AdfPipeline + + pipeline = AdfPipeline( + name="pl_globals", + activities=[ + _make_activity( + "Run NB", + "DatabricksNotebook", + { + "notebookPath": "/Shared/x", + "libraries": [{"jar": "@pipeline().globalParameters.libPath"}], + }, + ), + ], + ) + definitions = AdfDefinitions( + pipelines=[pipeline], + datasets={}, + linked_services={}, + triggers=[], + global_parameters={"libPath": "/Volumes/my.jar"}, + ) + return pipeline, definitions + + def test_literal_mode_bakes_value_and_declares_no_variable(self): + pipeline, definitions = self._pipeline_with_global() + report = translate_pipeline(pipeline, definitions, global_parameter_resolution="literal") + assert report.pipeline.bundle_variables == {} + notebook_task = next(t for t in report.pipeline.tasks if t.name == "Run NB") + assert notebook_task.libraries == [{"jar": "/Volumes/my.jar"}] + + def test_bundle_variable_mode_hoists_and_declares_variable(self): + pipeline, definitions = self._pipeline_with_global() + report = translate_pipeline(pipeline, definitions, global_parameter_resolution="bundle_variable") + assert "libPath" in report.pipeline.bundle_variables + assert report.pipeline.bundle_variables["libPath"]["default"] == "/Volumes/my.jar" + notebook_task = next(t for t in report.pipeline.tasks if t.name == "Run NB") + assert notebook_task.libraries == [{"jar": "${var.libPath}"}] + + def test_default_policy_is_literal(self): + pipeline, definitions = self._pipeline_with_global() + report = translate_pipeline(pipeline, definitions) + assert report.pipeline.bundle_variables == {} + notebook_task = next(t for t in report.pipeline.tasks if t.name == "Run NB") + assert notebook_task.libraries == [{"jar": "/Volumes/my.jar"}] + + def test_bundle_variables_survive_report_round_trip(self): + """bundle_variables serialize via _pipeline_to_dict and reconstruct via pipeline_dict_to_ir.""" + import json + + from flowx.bundler.dab_writer import pipeline_dict_to_ir + from flowx.translator.engine import _pipeline_to_dict + + pipeline, definitions = self._pipeline_with_global() + report = translate_pipeline(pipeline, definitions, global_parameter_resolution="bundle_variable") + # Full JSON round-trip, mirroring how the convert report reaches the package phase. + serialized = json.loads(json.dumps(_pipeline_to_dict(report.pipeline), default=str)) + reconstructed, _ = pipeline_dict_to_ir(serialized) + assert reconstructed.bundle_variables == report.pipeline.bundle_variables + assert reconstructed.bundle_variables["libPath"]["default"] == "/Volumes/my.jar" + + def test_global_inside_foreach_inherits_policy(self): + """A global referenced inside a ForEach inner activity resolves per the parent policy.""" + from flowx.models.adf_ast import AdfPipeline + + inner = _make_activity( + "InnerNB", + "DatabricksNotebook", + {"notebookPath": "/x", "libraries": [{"jar": "@pipeline().globalParameters.libPath"}]}, + ) + foreach = _make_activity( + "FE", + "ForEach", + {"items": {"value": "@pipeline().parameters.list", "type": "Expression"}}, + activities=[inner], + ) + pipeline = AdfPipeline(name="pl_fe", activities=[foreach]) + definitions = AdfDefinitions( + pipelines=[pipeline], + datasets={}, + linked_services={}, + triggers=[], + global_parameters={"libPath": "/Volumes/x.jar"}, + ) + report = translate_pipeline(pipeline, definitions, global_parameter_resolution="bundle_variable") + fe_task = next(t for t in report.pipeline.tasks if t.name == "FE") + inner_nb = fe_task.inner_activities[0] + assert inner_nb.libraries == [{"jar": "${var.libPath}"}] + assert "libPath" in report.pipeline.bundle_variables + + def test_global_in_raw_sql_body_resolved_by_whole_ir_rewrite(self): + """A global embedded in a raw SQL query is resolved by the whole-IR rewrite pass in both modes.""" + from flowx.models.adf_ast import AdfPipeline + + copy = _make_activity( + "Cp", + "Copy", + { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT * FROM t WHERE env='@{pipeline().globalParameters.env}'", + }, + "sink": {"type": "DeltaSink"}, + }, + ) + pipeline = AdfPipeline(name="pl_sql", activities=[copy]) + definitions = AdfDefinitions( + pipelines=[pipeline], + datasets={}, + linked_services={}, + triggers=[], + global_parameters={"env": "PROD"}, + ) + + literal = translate_pipeline(pipeline, definitions, global_parameter_resolution="literal") + literal_query = literal.pipeline.tasks[0].source_properties["sqlReaderQuery"] + assert "PROD" in literal_query + assert "${var." not in literal_query + + hoisted = translate_pipeline(pipeline, definitions, global_parameter_resolution="bundle_variable") + hoisted_query = hoisted.pipeline.tasks[0].source_properties["sqlReaderQuery"] + assert "${var.env}" in hoisted_query + assert "env" in hoisted.pipeline.bundle_variables + + class TestVariableInitTasks: """C-05 (VAREX-002): init SetVariable tasks for default-valued variables.""" From ea6ae18b119170455ec917c117368f75b434212e Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Wed, 29 Jul 2026 12:04:43 -0700 Subject: [PATCH 41/77] update dbt-factory to 0.3.1 --- src/flowx/dbt/manifest.py | 81 ++++++++++++------- .../activity_preparers/dbt_factory.py | 12 +-- tests/unit/test_dbt_factory_preparer.py | 11 +-- tests/unit/test_dbt_manifest.py | 51 +++++++++--- 4 files changed, 104 insertions(+), 51 deletions(-) diff --git a/src/flowx/dbt/manifest.py b/src/flowx/dbt/manifest.py index a48e850..e522d00 100644 --- a/src/flowx/dbt/manifest.py +++ b/src/flowx/dbt/manifest.py @@ -2,9 +2,10 @@ This is the deterministic core of dbt-factory mode: it turns the stable dbt-core manifest artifact into an ordered list of :class:`DbtNode` objects, one -per dbt model / seed / snapshot / test, with the dependency edges between them -pruned to the exploded set. It performs no I/O beyond reading the manifest file -and needs no dbt install, so it is unit-testable against a synthetic manifest. +per dbt model / seed / snapshot / data test / unit test, with the dependency +edges between them pruned to the exploded set. It performs no I/O beyond reading +the manifest file and needs no dbt install, so it is unit-testable against a +synthetic manifest. Both renderers (static explosion and the PyDABs deploy-time hook) consume the same :class:`DbtNode` list, so the "one IR node, two renderers" contract holds. @@ -17,16 +18,25 @@ from dataclasses import dataclass, field from pathlib import Path -# dbt resource_types that dbt-factory turns into their own orchestrator task. deps/docs and source -# definitions are not runnable nodes; snapshots/seeds/models/tests are. +# dbt resource_types that become their own orchestrator task. deps/docs and source definitions are +# not runnable nodes; snapshots/seeds/models/data tests are (all live under manifest["nodes"]). _RUNNABLE_RESOURCE_TYPES: frozenset[str] = frozenset({"model", "seed", "snapshot", "test"}) -# The dbt command each runnable resource_type maps to (dbt-factory's model->run, seed->seed, etc.). +# Unit tests live under a separate top-level manifest["unit_tests"] key (dbt >= 1.8), not under +# "nodes". They run under `dbt test`, gate like data tests, and are exploded when "test" is in scope. +_UNIT_TEST_RESOURCE_TYPE = "unit_test" + +# Resource types that gate downstream nodes: a downstream model waits for the tests (data and unit) +# on its upstream models, and a test never waits for another test. +_TEST_RESOURCE_TYPES: frozenset[str] = frozenset({"test", _UNIT_TEST_RESOURCE_TYPE}) + +# The dbt command each runnable resource_type maps to (model->run, seed->seed, unit_test->test, etc.). _RESOURCE_TYPE_TO_COMMAND: dict[str, str] = { "model": "run", "seed": "seed", "snapshot": "snapshot", "test": "test", + _UNIT_TEST_RESOURCE_TYPE: "test", } # FQN components go into a `--select fqn:a.b.c` selector; restrict to characters dbt's own selector @@ -40,7 +50,7 @@ class DbtNode: Attributes: unique_id: dbt manifest unique_id (e.g. ``model.pkg.stg_orders``). - resource_type: ``model`` / ``seed`` / ``snapshot`` / ``test``. + resource_type: ``model`` / ``seed`` / ``snapshot`` / ``test`` / ``unit_test``. name: dbt node name. command: dbt subcommand for this node (``run`` / ``seed`` / ...). selector: The ``fqn:`` selector that resolves to exactly this node. @@ -79,6 +89,20 @@ def _fqn_selector(fqn: list[str]) -> str: return "fqn:" + ".".join(fqn) +def _runnable_node(unique_id: str, node: dict, resource_type: str) -> DbtNode: + """Builds a :class:`DbtNode` from a manifest entry of a runnable resource_type.""" + name = node.get("name", unique_id) + fqn = node.get("fqn") or [name] + return DbtNode( + unique_id=unique_id, + resource_type=resource_type, + name=name, + command=_RESOURCE_TYPE_TO_COMMAND[resource_type], + selector=_fqn_selector(fqn), + task_key=_sanitize_task_key(resource_type, name), + ) + + def load_dbt_nodes(manifest_path: Path, *, resource_types: set[str] | None = None) -> list[DbtNode]: """Reads a dbt manifest and returns its runnable nodes as task specs. @@ -91,9 +115,8 @@ def load_dbt_nodes(manifest_path: Path, *, resource_types: set[str] | None = Non macros, or filtered-out nodes are dropped). Raises: - ValueError: When the test factory would be enabled but the - manifest carries unit tests (dbt-factory 0.2.1 silently drops - them), or when a node's fqn contains unsafe characters. + ValueError: When a node's fqn contains unsafe characters, or two + nodes sanitize to the same task key. """ manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) return explode_manifest(manifest, resource_types=resource_types) @@ -106,46 +129,42 @@ def explode_manifest(manifest: dict, *, resource_types: set[str] | None = None) manifest dict without touching the filesystem. """ nodes: dict[str, dict] = manifest.get("nodes", {}) + unit_tests: dict[str, dict] = manifest.get("unit_tests", {}) enabled_types = _RUNNABLE_RESOURCE_TYPES if resource_types is None else _RUNNABLE_RESOURCE_TYPES & resource_types + # Unit tests run under `dbt test`, so they are in scope exactly when data tests are. + unit_tests_enabled = "test" in enabled_types runnable: dict[str, DbtNode] = {} + # The raw manifest entry for each exploded node, keyed by unique_id, so edge pruning can read a + # unit test's depends_on (unit tests live under a separate top-level key) the same way it reads a + # regular node's. + manifest_entry: dict[str, dict] = {} for unique_id, node in nodes.items(): resource_type = node.get("resource_type", "") if resource_type not in enabled_types: continue - fqn = node.get("fqn") or [node.get("name", unique_id)] - runnable[unique_id] = DbtNode( - unique_id=unique_id, - resource_type=resource_type, - name=node.get("name", unique_id), - command=_RESOURCE_TYPE_TO_COMMAND[resource_type], - selector=_fqn_selector(fqn), - task_key=_sanitize_task_key(resource_type, node.get("name", unique_id)), - ) - - # Fail closed on unit tests: dbt-factory 0.2.1 does not emit unit-test tasks, so a manifest that - # carries them would silently lose coverage if any test task is exploded. - if manifest.get("unit_tests") and any(n.resource_type == "test" for n in runnable.values()): - raise ValueError( - "Manifest declares unit_tests, which dbt-factory 0.2.1 does not explode into tasks. " - "Refusing to emit an incomplete dbt job (fail-closed)." - ) + runnable[unique_id] = _runnable_node(unique_id, node, resource_type) + manifest_entry[unique_id] = node + if unit_tests_enabled: + for unique_id, node in unit_tests.items(): + runnable[unique_id] = _runnable_node(unique_id, node, _UNIT_TEST_RESOURCE_TYPE) + manifest_entry[unique_id] = node # Prune dependency edges to the exploded set. dbt nodes depend on sources, macros, and each other; # only edges between two runnable nodes become task dependencies. task_key_by_uid = {uid: dbt_node.task_key for uid, dbt_node in runnable.items()} tests_by_tested_uid: dict[str, list[str]] = {} for test_uid, test_node in runnable.items(): - if test_node.resource_type != "test": + if test_node.resource_type not in _TEST_RESOURCE_TYPES: continue - for tested_uid in nodes[test_uid].get("depends_on", {}).get("nodes") or []: + for tested_uid in manifest_entry[test_uid].get("depends_on", {}).get("nodes") or []: tests_by_tested_uid.setdefault(tested_uid, []).append(test_node.task_key) for uid, dbt_node in runnable.items(): - upstream_uids = nodes[uid].get("depends_on", {}).get("nodes") or [] + upstream_uids = manifest_entry[uid].get("depends_on", {}).get("nodes") or [] dependencies = [ task_key_by_uid[upstream_uid] for upstream_uid in upstream_uids if upstream_uid in task_key_by_uid ] - if dbt_node.resource_type != "test": + if dbt_node.resource_type not in _TEST_RESOURCE_TYPES: dependencies.extend( test_key for upstream_uid in upstream_uids for test_key in tests_by_tested_uid.get(upstream_uid, []) ) diff --git a/src/flowx/preparer/activity_preparers/dbt_factory.py b/src/flowx/preparer/activity_preparers/dbt_factory.py index abbe934..eedccaf 100644 --- a/src/flowx/preparer/activity_preparers/dbt_factory.py +++ b/src/flowx/preparer/activity_preparers/dbt_factory.py @@ -82,7 +82,7 @@ def _pydabs_pyproject_source() -> str: 'requires-python = ">=3.10"\n' "dependencies = [\n" ' "databricks-bundles>=1.0.0,<2.0.0",\n' - ' "databricks-dbt-factory==0.2.1",\n' + ' "databricks-dbt-factory==0.3.1",\n' ' "dbt-databricks==1.12.2",\n' ' "dbt-core==1.11.12",\n' "]\n" @@ -336,14 +336,14 @@ def _pydabs_hook_source(activity: DbtFactoryActivity) -> str: "from databricks.bundles.jobs import Job\n" "from databricks_dbt_factory.DbtFactory import DbtFactory\n" "from databricks_dbt_factory.DbtTask import DbtTaskOptions, TaskType\n" - "from databricks_dbt_factory.SpecsHandler import SpecsHandler\n" "from databricks_dbt_factory.TaskFactory import (\n" " DbtDependencyResolver,\n" " ModelTaskFactory,\n" " SeedTaskFactory,\n" " SnapshotTaskFactory,\n" " TestTaskFactory,\n" - ")\n\n" + ")\n" + "from databricks_dbt_factory.Utils import read_dbt_manifest\n\n" f"MANIFEST_PATH = {'src/dbt_project/target/manifest.json'!r}\n" f"PROJECT_DIR = {'../dbt_project'!r}\n" f"PROFILES_DIR = {'../dbt_profiles'!r}\n" @@ -369,15 +369,15 @@ def _pydabs_hook_source(activity: DbtFactoryActivity) -> str: " for name in RESOURCE_TYPES\n" " }\n\n" "def load_resources(bundle: Bundle) -> Resources:\n" - " manifest = SpecsHandler.read_dbt_manifest(MANIFEST_PATH)\n" + " manifest = read_dbt_manifest(MANIFEST_PATH)\n" " task_factories = _task_factories()\n" " resources = Resources()\n" - " factory = DbtFactory(SpecsHandler(), task_factories, bundle_tests=False)\n" + " factory = DbtFactory(task_factories, bundle_tests=False)\n" " tasks = factory.create_tasks(manifest)\n" " environment = {\n" " 'environment_key': 'Default',\n" " 'spec': {\n" - " 'environment_version': '4',\n" + " 'environment_version': '5',\n" " 'dependencies': ['dbt-databricks==1.12.2', 'dbt-core==1.11.12'],\n" " },\n" " }\n" diff --git a/tests/unit/test_dbt_factory_preparer.py b/tests/unit/test_dbt_factory_preparer.py index b6e1fbd..f10dbaf 100644 --- a/tests/unit/test_dbt_factory_preparer.py +++ b/tests/unit/test_dbt_factory_preparer.py @@ -171,10 +171,11 @@ def test_pydabs_emits_hook_module_and_no_inner_job(tmp_path): } <= hook_paths hook = next(nb for nb in prepared.notebooks if nb.relative_path.endswith("_dbt_job.py")) assert "load_resources" in hook.content - assert "from databricks_dbt_factory.SpecsHandler import SpecsHandler" in hook.content - assert "DbtFactory(SpecsHandler(), task_factories" in hook.content - import_block = hook.content.split("from databricks_dbt_factory", maxsplit=1)[1].split("\n\n", maxsplit=1)[0] - assert "read_dbt_manifest" not in import_block + assert "from databricks_dbt_factory.Utils import read_dbt_manifest" in hook.content + assert "DbtFactory(task_factories" in hook.content + # 0.3.1 dropped SpecsHandler; the manifest reader is a module-level Utils function. + assert "SpecsHandler" not in hook.content + assert "read_dbt_manifest(MANIFEST_PATH)" in hook.content runner = next(nb for nb in prepared.notebooks if nb.relative_path == "notebooks/run_dbt_command.py") assert "dbt_commands" in runner.content assert "project_directory" in runner.content @@ -271,7 +272,7 @@ def test_pydabs_bundle_wires_python_resources_and_setup(tmp_path): assert (tmp_path / "resources" / "__init__.py").exists() assert not (tmp_path / "src" / "resources").exists() pyproject = (tmp_path / "pyproject.toml").read_text() - assert "databricks-dbt-factory==0.2.1" in pyproject + assert "databricks-dbt-factory==0.3.1" in pyproject assert "dbt-databricks==1.12.2" in pyproject setup = (tmp_path / "SETUP.md").read_text() assert "dbt factory (PyDABs mode)" in setup diff --git a/tests/unit/test_dbt_manifest.py b/tests/unit/test_dbt_manifest.py index b3b217e..f1f50d8 100644 --- a/tests/unit/test_dbt_manifest.py +++ b/tests/unit/test_dbt_manifest.py @@ -27,6 +27,10 @@ def _test(name, fqn, deps=None): return {"resource_type": "test", "name": name, "fqn": fqn, "depends_on": {"nodes": deps or []}} +def _unit_test(name, fqn, model_uid): + return {"resource_type": "unit_test", "name": name, "fqn": fqn, "depends_on": {"nodes": [model_uid]}} + + def _manifest(nodes, unit_tests=None): return {"nodes": nodes, "unit_tests": unit_tests or {}} @@ -116,22 +120,51 @@ def test_output_is_sorted_by_task_key(): assert keys == sorted(keys) -def test_rejects_unit_tests_when_test_node_present(): +def test_unit_tests_explode_into_their_own_test_command_tasks(): manifest = _manifest( - {"test.p.t": _test("t", ["p", "t"])}, - unit_tests={"unit_test.p.a": {}}, + {"model.p.stg": _model("stg", ["p", "staging", "stg"])}, + unit_tests={ + "unit_test.p.stg.check_amount": _unit_test( + "check_amount", ["p", "staging", "stg", "check_amount"], "model.p.stg" + ) + }, ) - with pytest.raises(ValueError, match="unit_test"): - explode_manifest(manifest) + by_key = {node.task_key: node for node in explode_manifest(manifest)} + + unit = by_key["unit_test_check_amount"] + assert unit.command == "test" + assert unit.selector == "fqn:p.staging.stg.check_amount" + # The unit test gates on the model it targets, like a data test. + assert unit.depends_on == ["model_stg"] -def test_allows_unit_tests_when_no_test_node(): - # A manifest with unit_tests but no exploded test node is fine (nothing dropped). + +def test_downstream_model_waits_for_unit_tests_on_its_upstream_model(): + manifest = _manifest( + { + "model.p.stg": _model("stg", ["p", "stg"]), + "model.p.fct": _model("fct", ["p", "fct"], deps=["model.p.stg"]), + }, + unit_tests={ + "unit_test.p.stg.check": _unit_test("check", ["p", "stg", "check"], "model.p.stg"), + }, + ) + + by_key = {node.task_key: node for node in explode_manifest(manifest)} + + assert by_key["model_fct"].depends_on == ["model_stg", "unit_test_check"] + + +def test_unit_tests_dropped_when_test_scope_excluded(): + # `dbt run` (resource_types={"model"}) does not run tests, so unit tests are out of scope too. manifest = _manifest( {"model.p.stg": _model("stg", ["p", "stg"])}, - unit_tests={"unit_test.p.a": {}}, + unit_tests={"unit_test.p.stg.check": _unit_test("check", ["p", "stg", "check"], "model.p.stg")}, ) - explode_manifest(manifest) # no raise + + keys = {node.task_key for node in explode_manifest(manifest, resource_types={"model"})} + + assert keys == {"model_stg"} def test_rejects_unsafe_fqn_characters(): From ef12a9540bcee37f6c5532d1c73a0e34da55c204 Mon Sep 17 00:00:00 2001 From: Alex Nastetsky Date: Wed, 29 Jul 2026 12:42:10 -0400 Subject: [PATCH 42/77] Emit sync.include in generated databricks.yml so gitignored output deploys DABs derives its deploy sync set by honoring .gitignore. The default output dir (./flowx_output) is commonly gitignored, so bundle deploy would upload zero files and leave the job's notebooks missing, even though the job deployed. A nested .gitignore negation can't recover this, so a DAB-level sync.include is the only reliable override. _build_databricks_yml now always emits sync.include: [src/**]; adds a bundler test. Co-authored-by: Isaac --- src/flowx/bundler/dab_writer.py | 11 +++++++++++ tests/unit/test_bundler.py | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 362d48a..b9ac835 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -635,6 +635,17 @@ def _build_databricks_yml( "include": [ "resources/*.yml", ], + # Force the generated notebook sources into the deploy sync set. DABs derives its + # sync set by honoring .gitignore, and the default output dir (./flowx_output) is + # commonly gitignored, which would otherwise make `bundle deploy` upload zero files + # and leave the job's notebooks missing. A nested .gitignore negation can't recover + # this (git won't re-include a path under an excluded parent), so sync.include is the + # only reliable override. Harmless when the dir isn't ignored. + "sync": { + "include": [ + "src/**", + ], + }, "targets": { "dev": { "mode": "development", diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index 90ecb02..ca764c6 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -92,6 +92,13 @@ def test_databricks_yml_structure(self, tmp_path): assert "dev" in content["targets"] assert "prod" in content["targets"] + def test_databricks_yml_sync_includes_src(self, tmp_path): + """databricks.yml forces src/** into the sync set so a gitignored output dir still uploads notebooks.""" + wf = _simple_workflow("my_pipeline") + write_bundle(wf, tmp_path) + content = yaml.safe_load((tmp_path / "databricks.yml").read_text()) + assert content["sync"]["include"] == ["src/**"] + def test_job_resource_yml_exists(self, tmp_path): """A job resource YAML is created under resources/.""" wf = _simple_workflow("my_job") From 26c437500fadcc76afcd4ede64ea94c4f38a0c8d Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Wed, 29 Jul 2026 13:28:21 -0700 Subject: [PATCH 43/77] Fix sensor codegen escaping and dbt-chain ordering loss Escape sensor path/table_name/description via repr() so a value with a quote no longer emits uncompilable notebook source. Collapse dbt chains so the factory absorbs every dbt op's external upstream (not just the first) and a task sandwiched between two dbt ops keeps its ordering without forming a cycle. --- src/flowx/sources/airflow/loader.py | 55 +++++++++++++++++++++- src/flowx/sources/airflow/operators.py | 6 +-- tests/unit/test_airflow_operators.py | 64 +++++++++++++++++++++++++- 3 files changed, 120 insertions(+), 5 deletions(-) diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py index 1a341e8..edf2f91 100644 --- a/src/flowx/sources/airflow/loader.py +++ b/src/flowx/sources/airflow/loader.py @@ -977,9 +977,52 @@ def _task_key(var: str, task_id: str) -> str: # downstream task that depended on a later dbt op (e.g. `dbt_test`) points at the factory task # rather than a task_key that was never emitted (which would dangle). dbt_vars = [var for var, (_, op, _) in visitor.operators.items() if op in ops.DBT_CLI_OPERATORS] + dbt_var_set = set(dbt_vars) dbt_factory_key = var_to_task_key[dbt_vars[0]] if dbt_vars else None dbt_key_remap = {var_to_task_key[v]: dbt_factory_key for v in dbt_vars} if dbt_factory_key else {} + # Non-dbt tasks reachable *downstream* from the collapsed dbt set. Because every dbt op folds into + # one factory task, a task that sat between two dbt ops (e.g. `dbt_seed >> task_b >> dbt_run`) is + # downstream of the factory; the factory therefore cannot depend on it without forming a cycle, + # but it must still depend on the factory and gate whatever followed it. + downstream_of_factory: set[str] = set() + if dbt_factory_key: + adjacency: dict[str, list[str]] = {v: [] for v in var_task_ids} + for downstream_var, ups in upstreams.items(): + for upstream_var in ups: + adjacency.setdefault(upstream_var, []).append(downstream_var) + stack = list(dbt_vars) + seen_ds: set[str] = set(dbt_vars) + while stack: + for nxt in adjacency.get(stack.pop(), []): + if nxt not in seen_ds: + seen_ds.add(nxt) + stack.append(nxt) + downstream_of_factory = {var_to_task_key[v] for v in seen_ds if v not in dbt_var_set} + + def _sandwiched_before(dbt_var: str) -> set[str]: + """Non-dbt tasks that fed *dbt_var* (through the collapsed dbt chain) and sit downstream of the + factory. A task consuming a later dbt op must still wait for these, since the collapse drops + the intermediate dbt op they fed.""" + result: set[str] = set() + for upstream_var in upstreams.get(dbt_var, []): + if upstream_var in dbt_var_set: + result |= _sandwiched_before(upstream_var) + elif var_to_task_key[upstream_var] in downstream_of_factory: + result.add(var_to_task_key[upstream_var]) + return result + + # The factory absorbs every dbt op's external (non-dbt) upstream that is not itself downstream of + # the factory -- not just the first dbt op's, so a later dbt op's upstream is not silently dropped. + factory_dep_keys: set[str] = set() + for dbt_var in dbt_vars: + for upstream_var in upstreams.get(dbt_var, []): + if upstream_var in dbt_var_set: + continue + key = var_to_task_key[upstream_var] + if key not in downstream_of_factory: + factory_dep_keys.add(key) + def _dep(upstream_var: str, outcome: str | None) -> str: key = var_to_task_key[upstream_var] return dbt_key_remap.get(key, key) @@ -995,6 +1038,11 @@ def _dep(upstream_var: str, outcome: str | None) -> str: # Remap dbt-chain upstreams to the single factory key and drop self-edges (a dbt op # depending on another dbt op in the same collapsed chain). dep_keys = {_dep(u, outcome) for u in upstreams[var]} + # A task consuming a later dbt op must also wait for any non-dbt task that sat between two dbt + # ops (the collapse folds away the intermediate dbt op that carried that ordering). + for upstream_var in upstreams[var]: + if upstream_var in dbt_var_set: + dep_keys |= _sandwiched_before(upstream_var) dep_keys.discard(task_key if operator not in ops.DBT_CLI_OPERATORS else dbt_factory_key) depends_on = [Dependency(task_key=k, outcome=outcome) for k in sorted(dep_keys)] or None @@ -1008,13 +1056,18 @@ def _dep(upstream_var: str, outcome: str | None) -> str: if emitted_dbt: continue emitted_dbt = True + # The factory gates on every dbt op's external upstreams (not just the first op's), minus + # any that are downstream of the factory itself (a sandwiched task, which would cycle). + factory_depends_on = [ + Dependency(task_key=k, outcome=outcome) for k in sorted(factory_dep_keys) + ] or None dbt_kwargs = [visitor.operators[v][2] for v in dbt_vars] tasks.append( _build_dbt_factory( task_id, task_key, dbt_kwargs, - depends_on, + factory_depends_on, dbt_mode, operator_types=[visitor.operators[dbt_var][1] for dbt_var in dbt_vars], ) diff --git a/src/flowx/sources/airflow/operators.py b/src/flowx/sources/airflow/operators.py index 9e713b8..d307947 100644 --- a/src/flowx/sources/airflow/operators.py +++ b/src/flowx/sources/airflow/operators.py @@ -189,7 +189,7 @@ def _poll_body(operator: str, check_expr: str, description: str, poke: int, time return ( f"POKE_INTERVAL = {poke} # seconds\n" + f"TIMEOUT = {timeout} # seconds\n" - + f'DESCRIPTION = "{description}"\n\n' + + f"DESCRIPTION = {description!r}\n\n" + "def _condition_met():\n" + f" # {operator} poke: returns truthy once the awaited condition holds.\n" + f" return {check_expr}\n\n" @@ -236,7 +236,7 @@ def _build_file_sensor(ctx: OperatorContext) -> Activity: " except Exception:\n" " return False\n\n" ) - loop = _poll_body(ctx.operator, f'_path_exists("{path}")', f"file at {path}", poke, timeout) + loop = _poll_body(ctx.operator, f"_path_exists({path!r})", f"file at {path}", poke, timeout) return NotebookActivity( name=ctx.task_id, task_key=ctx.task_key, @@ -267,7 +267,7 @@ def _build_table_sensor(ctx: OperatorContext) -> Activity: ) desc = "SQL sensor condition" elif table_name is not None: - check = f'spark.catalog.tableExists("{table_name}")' + check = f"spark.catalog.tableExists({table_name!r})" header = _notebook_header(ctx.task_id, ctx.operator) + "import time\n\n" desc = f"table {table_name}" else: diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py index 39c5f51..7d9dde8 100644 --- a/tests/unit/test_airflow_operators.py +++ b/tests/unit/test_airflow_operators.py @@ -651,6 +651,68 @@ def test_dbt_chain_downstream_dep_rewired_to_factory_key(): assert [d.task_key for d in tasks["publish"].depends_on] == [factory_key] +def test_dbt_chain_absorbs_every_dbt_ops_upstream(): + # An external task feeding a LATER dbt op must gate the single collapsed factory -- not be dropped + # because only the first dbt op's upstreams were absorbed. + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "from airflow_dbt.operators.dbt_operator import DbtRunOperator, DbtTestOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " ingest = PythonOperator(task_id='ingest', python_callable=w)\n" + " seed_src = PythonOperator(task_id='seed_src', python_callable=w)\n" + " r = DbtRunOperator(task_id='run', dir='/opt/proj')\n" + " t = DbtTestOperator(task_id='test', dir='/opt/proj')\n" + " ingest >> r\n" + " seed_src >> t\n" + " r >> t\n" + ) + factory = next(t for t in p.tasks if isinstance(t, DbtFactoryActivity)) + assert sorted(d.task_key for d in factory.depends_on) == ["ingest", "seed_src"] + + +def test_dbt_chain_preserves_sandwiched_task_ordering(): + # A non-dbt task between two dbt ops (seed >> mid >> run) is downstream of the collapsed factory, + # so a task consuming the later dbt op must still wait for it -- and no cycle is formed. + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "from airflow_dbt.operators.dbt_operator import DbtSeedOperator, DbtRunOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " s = DbtSeedOperator(task_id='seed', dir='/opt/proj')\n" + " mid = PythonOperator(task_id='mid', python_callable=w)\n" + " r = DbtRunOperator(task_id='run', dir='/opt/proj')\n" + " tail = PythonOperator(task_id='tail', python_callable=w)\n" + " s >> mid >> r >> tail\n" + ) + tasks = _by_key(p) + factory_key = next(t.task_key for t in p.tasks if isinstance(t, DbtFactoryActivity)) + # `mid` gates on the factory; `tail` (consumer of the vanished `run`) waits for BOTH the factory + # and `mid`, preserving the mid->tail ordering without depending on itself (no cycle). + assert [d.task_key for d in tasks["mid"].depends_on] == [factory_key] + assert sorted(d.task_key for d in tasks["tail"].depends_on) == sorted([factory_key, "mid"]) + + +def test_table_sensor_escapes_quotes_in_table_name(): + # A table_name (or file path) carrying a double quote must not break the generated notebook: the + # value goes through repr(), so the source still compiles. + p = _load( + "from airflow import DAG\n" + "from airflow.providers.databricks.sensors.databricks_partition import DatabricksPartitionSensor\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " prep = PythonOperator(task_id='prep', python_callable=w)\n" + ' wait = DatabricksPartitionSensor(task_id=\'wait\', table_name=\'main.silver.we"ird\')\n' + " prep >> wait\n" + ) + wait = _by_key(p)["wait"] + assert isinstance(wait, NotebookActivity) + compile(wait.generated_source, "", "exec") # would raise before the repr() fix + + def test_dbt_factory_explodes_manifest_into_tasks(tmp_path): # End-to-end: a real (synthetic) manifest must explode into per-node tasks, not an empty job. import json @@ -780,7 +842,7 @@ def test_mid_dag_sensor_retained_as_polling_task(): assert set(tasks) == {"prep", "wait", "go"} wait = tasks["wait"] assert isinstance(wait, NotebookActivity) - assert 'spark.catalog.tableExists("main.silver.events")' in wait.generated_source + assert "spark.catalog.tableExists('main.silver.events')" in wait.generated_source compile(wait.generated_source, "", "exec") From 42552cc098f87dcca3c7cb3f73819a90c5b9d524 Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Wed, 29 Jul 2026 13:51:47 -0700 Subject: [PATCH 44/77] Map Airflow execution date to a backfillable run_date parameter Route {{ ds }}/execution_date/logical_date macros to an overridable run_date job parameter (both op_kwargs and SQL paths) instead of an inline start_time ref, so a native Databricks backfill can override it per replayed window. Default the parameter to the scheduled trigger time on cron/periodic jobs (start_time drifts on delay/retry) and to start_time on event-triggered/unscheduled jobs. Surface catchup=True as a SETUP.md native-backfill note. --- src/flowx/bundler/dab_writer.py | 2 + src/flowx/bundler/prereqs_writer.py | 21 +++++++ src/flowx/preparer/workflow_preparer.py | 5 ++ src/flowx/sources/airflow/loader.py | 31 ++++++++- src/flowx/sources/airflow/templating.py | 84 +++++++++++++++---------- tests/unit/test_airflow_operators.py | 69 ++++++++++++++++++-- tests/unit/test_airflow_templating.py | 49 +++++++++++++++ tests/unit/test_bundler.py | 17 +++++ 8 files changed, 237 insertions(+), 41 deletions(-) create mode 100644 tests/unit/test_airflow_templating.py diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 7c098ec..7cdaf3a 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -288,6 +288,7 @@ def _write_generated(notebooks: list[DabNotebook]) -> None: task.config for task in workflow.setup_tasks if task.type == "manual_schedule_time_of_day" ] manual_credential_configs = [task.config for task in workflow.setup_tasks if task.type == "manual_credential"] + airflow_backfill_configs = [task.config for task in workflow.setup_tasks if task.type == "airflow_backfill"] pydabs_dbt_factory_configs = [task.config for task in workflow.setup_tasks if task.type == "pydabs_dbt_factory"] for inner in workflow.inner_workflows: pydabs_dbt_factory_configs.extend( @@ -327,6 +328,7 @@ def _write_generated(notebooks: list[DabNotebook]) -> None: manual_credentials=manual_credential_configs, neutralized_conditions=list(_neutralized_conditions), pydabs_dbt_factories=pydabs_dbt_factory_configs, + airflow_backfills=airflow_backfill_configs, ) setup_path = output_dir / "SETUP.md" setup_path.write_text(render_setup_md(prereqs, bundle_name=effective_name), encoding="utf-8") diff --git a/src/flowx/bundler/prereqs_writer.py b/src/flowx/bundler/prereqs_writer.py index a0bc593..e79fc40 100644 --- a/src/flowx/bundler/prereqs_writer.py +++ b/src/flowx/bundler/prereqs_writer.py @@ -134,6 +134,9 @@ class Prereqs: # dbt-factory PyDABs hooks; each entry is the SetupTask config dict ({hook_module, job_key, # manifest_path, note}). The user must `pip install databricks-dbt-factory` before deploy. pydabs_dbt_factories: list[dict[str, Any]] = field(default_factory=list) + # Airflow catchup=True jobs; each entry is the SetupTask config dict ({pipeline}). History is + # replayed via a native Databricks backfill overriding the run_date parameter, not a DABs setting. + airflow_backfills: list[dict[str, Any]] = field(default_factory=list) def is_empty(self) -> bool: """Return ``True`` when nothing needs to happen before ``bundle run``.""" @@ -154,6 +157,7 @@ def is_empty(self) -> bool: and not self.manual_credentials and not self.neutralized_conditions and not self.pydabs_dbt_factories + and not self.airflow_backfills ) @@ -366,6 +370,7 @@ def build_prereqs( manual_credentials: list[dict[str, Any]] | None = None, neutralized_conditions: list[dict[str, str]] | None = None, pydabs_dbt_factories: list[dict[str, Any]] | None = None, + airflow_backfills: list[dict[str, Any]] | None = None, ) -> Prereqs: """Assemble a :class:`Prereqs` from the bundle's generated artifacts. @@ -414,6 +419,7 @@ def build_prereqs( manual_credentials=list(manual_credentials or []), neutralized_conditions=list(neutralized_conditions or []), pydabs_dbt_factories=list(pydabs_dbt_factories or []), + airflow_backfills=list(airflow_backfills or []), ) @@ -682,6 +688,21 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append(f"| `{pipeline}` | `{frequency}` | `{interval}` | `{tod_spec}` |") lines.append("") + if prereqs.airflow_backfills: + lines.append("## Backfill (Airflow catchup)") + lines.append("") + lines.append( + "The DAG(s) below set `catchup=True`, so Airflow backfilled missed intervals. There is " + "no equivalent DABs schedule setting. To replay history, run a " + "[native Databricks backfill](https://docs.databricks.com/aws/en/jobs/backfill-jobs), which " + "overrides the `run_date` job parameter with `{{backfill.iso_date}}` per replayed window " + "(the run_date parameter is emitted for exactly this reason)." + ) + lines.append("") + for entry in sorted(prereqs.airflow_backfills, key=lambda config: config.get("pipeline", "")): + lines.append(f"- `{entry.get('pipeline', '')}`") + lines.append("") + if prereqs.manual_credentials: lines.append("## Manual credential setup") lines.append("") diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py index e94adad..ab50ccd 100644 --- a/src/flowx/preparer/workflow_preparer.py +++ b/src/flowx/preparer/workflow_preparer.py @@ -376,6 +376,11 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow: ) ) + # Airflow catchup=True has no DABs schedule setting; surface that history is replayed via a native + # Databricks backfill, which overrides the run_date job parameter with {{backfill.iso_date}}. + if pipeline.tags.get("airflow_catchup") == "true": + setup_tasks_out.append(SetupTask(type="airflow_backfill", config={"pipeline": pipeline.name})) + # C-39 (LSC4-004): ADF auth modes with no Databricks equivalent (MSI, CredentialReference) make the # default_cluster fall back to single_user_name: ${workspace.current_user.userName}; flag it via SetupTask. seen_auth: set[tuple[str, str]] = set() diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py index edf2f91..9a5cde5 100644 --- a/src/flowx/sources/airflow/loader.py +++ b/src/flowx/sources/airflow/loader.py @@ -282,6 +282,9 @@ def __init__(self, module: ast.Module) -> None: self.schedule_interval: str | None = None self.schedule_node: ast.expr | None = None self.timezone: str | None = None + # DAG catchup= flag: True means Airflow backfills missed intervals, which maps to a native + # Databricks backfill overriding the run_date parameter rather than any DABs schedule setting. + self.catchup: bool = False self.default_args: dict[str, ast.expr] = {} # DAG-level params={...} defaults (param name -> literal default), so emitted job parameters # carry a Databricks-required default rather than an empty placeholder. @@ -561,6 +564,7 @@ def _apply_dag_kwargs(self, kwargs: dict[str, ast.expr]) -> None: kwargs.get("schedule") ) self.timezone = _extract_timezone(kwargs.get("start_date")) or _extract_timezone(kwargs.get("timezone")) + self.catchup = ops.literal_value(kwargs.get("catchup")) is True # default_args is a dict literal of DAG-wide task settings (retries, timeouts, email). default_args = kwargs.get("default_args") if isinstance(default_args, ast.Dict): @@ -1181,18 +1185,24 @@ def _dep(upstream_var: str, outcome: str | None) -> str: # Declare every job parameter -- those referenced in templates plus any from the DAG's # params={...} -- each with a default (Databricks requires one): the params={...} default when - # present, else an empty string so the bundle still validates. + # present; a logical-date parameter (run_date/execution_date/...) its schedule-aware time ref so a + # native backfill can override it per window; else an empty string so the bundle still validates. param_names = referenced_params | set(visitor.dag_params) parameters = [ - {"name": name, "default": visitor.dag_params[name] if visitor.dag_params.get(name) is not None else ""} + {"name": name, "default": _declared_param_default(name, visitor.dag_params, schedule)} for name in sorted(param_names) ] or None + tags = {"source": "airflow", "dag_id": visitor.dag_id or ""} + if visitor.catchup: + # Airflow catchup=True has no DABs schedule setting; it maps to running a native Databricks + # backfill, which overrides the run_date job parameter with {{backfill.iso_date}} per window. + tags["airflow_catchup"] = "true" return Pipeline( name=visitor.dag_id or Path(dag_path).stem, tasks=tasks, parameters=parameters, schedule=schedule, - tags={"source": "airflow", "dag_id": visitor.dag_id or ""}, + tags=tags, ) @@ -1375,6 +1385,21 @@ def _reader(dep_var: str) -> str: return "\n".join(lines) + "\n" +def _declared_param_default(name: str, dag_params: dict[str, Any], schedule: dict[str, object] | None) -> Any: + """Returns the Databricks-required default for a declared job parameter. + + A DAG ``params={...}`` default wins. A logical-date parameter (``run_date`` etc., from an Airflow + ``{{ ds }}``/``execution_date`` macro) defaults to its schedule-aware time ref so a native backfill + can override it per replayed window. Everything else defaults to an empty string. + """ + if dag_params.get(name) is not None: + return dag_params[name] + field = templating.DATE_PARAM_FIELDS.get(name) + if field is not None: + return templating.date_param_default(field, schedule) + return "" + + def _convert_activity_templates(activity: Activity) -> set[str]: """Converts Airflow Jinja in an activity's parameter fields to DAB refs. diff --git a/src/flowx/sources/airflow/templating.py b/src/flowx/sources/airflow/templating.py index c5ae473..ed312c0 100644 --- a/src/flowx/sources/airflow/templating.py +++ b/src/flowx/sources/airflow/templating.py @@ -14,21 +14,43 @@ import re from typing import Any -# Airflow Jinja macros -> Databricks job dynamic-value references. Only macros with an exact -# Databricks equivalent are mapped; date macros -> the job start time, run_id -> the run id. -# ``ds_nodash``/``ts_nodash`` have no dashless dynamic-value form, so they are intentionally NOT -# mapped -- they're left untouched (surfaced as an unresolved reference) rather than emitting an -# invalid ref. +# Airflow date/time macros carrying the run's *logical date* -> a named job parameter (not an inline +# time ref), so a native Databricks backfill can override the parameter per replayed window with +# {{backfill.iso_date}}. Each maps to (parameter_name, time_field); the loader assigns the parameter a +# schedule-aware default (see `date_param_default`). ``ds_nodash``/``ts_nodash`` have no dashless +# dynamic-value form, so they are intentionally NOT mapped -- they're left untouched (surfaced as an +# unresolved reference) rather than emitting an invalid ref. +_DATE_MACRO_PARAM: dict[str, tuple[str, str]] = { + "ds": ("run_date", "iso_date"), + "ts": ("run_timestamp", "iso_datetime"), + "data_interval_start": ("data_interval_start", "iso_datetime"), + "data_interval_end": ("data_interval_end", "iso_datetime"), + "execution_date": ("execution_date", "iso_datetime"), + "logical_date": ("logical_date", "iso_datetime"), +} + +# job parameter name -> its time field, so the loader can default each to the right granularity. +DATE_PARAM_FIELDS: dict[str, str] = {param: field for param, field in _DATE_MACRO_PARAM.values()} + +# Non-date macros with an exact Databricks equivalent, mapped inline (no backfill relevance). _MACRO_TO_DAB_REF: dict[str, str] = { - "ds": "{{job.start_time.iso_date}}", - "ts": "{{job.start_time.iso_datetime}}", - "data_interval_start": "{{job.start_time.iso_datetime}}", - "data_interval_end": "{{job.start_time.iso_datetime}}", - "execution_date": "{{job.start_time.iso_datetime}}", - "logical_date": "{{job.start_time.iso_datetime}}", "run_id": "{{job.run_id}}", } + +def date_param_default(field: str, schedule: dict[str, object] | None) -> str: + """Returns the default dynamic-value ref for a logical-date job parameter. + + On a cron/periodic schedule the logical date is the scheduled trigger time + (``{{job.trigger.time...}}``) -- ``start_time`` would drift with queue delay and retries. On an + event-triggered job (``file_arrival``/``table_update``/``continuous``) or an unscheduled job there + is no scheduled trigger time, so approximate with the run's start time. A native backfill overrides + the parameter regardless of this default. + """ + kind = schedule.get("kind") if schedule else None + base = "{{job.trigger.time." if kind in ("schedule", "periodic") else "{{job.start_time." + return f"{base}{field}}}}}" + # {{ params.X }} / {{ var.value.X }} / {{ dag_run.conf['X'] }} -> {{job.parameters.X}} _PARAM_PATTERNS: list[re.Pattern[str]] = [ re.compile(r"^params\.([A-Za-z_][A-Za-z0-9_]*)$"), @@ -43,15 +65,20 @@ def convert_template(value: str) -> tuple[str, set[str]]: """Converts Airflow Jinja in *value* to DAB dynamic-value references. - Returns ``(converted_value, referenced_param_names)``. Date/system macros map - to ``{{job.start_time.*}}`` refs; ``params.X`` / ``var.value.X`` / ``dag_run.conf['X']`` - map to ``{{job.parameters.X}}`` and X is reported so the pipeline can declare it. - An unrecognised expression is left as-is (so nothing is silently corrupted). + Returns ``(converted_value, referenced_param_names)``. A logical-date macro (``ds``, + ``execution_date``, ...) maps to ``{{job.parameters.run_date}}`` (etc.) so a native backfill can + override it; ``params.X`` / ``var.value.X`` / ``dag_run.conf['X']`` map to ``{{job.parameters.X}}``; + ``run_id`` maps to its inline ref. Referenced parameter names are reported so the pipeline can + declare them. An unrecognised expression is left as-is (so nothing is silently corrupted). """ params: set[str] = set() def _sub(match: re.Match[str]) -> str: expr = match.group(1).strip() + if expr in _DATE_MACRO_PARAM: + name, _ = _DATE_MACRO_PARAM[expr] + params.add(name) + return "{{job.parameters." + name + "}}" if expr in _MACRO_TO_DAB_REF: return _MACRO_TO_DAB_REF[expr] for pattern in _PARAM_PATTERNS: @@ -65,17 +92,6 @@ def _sub(match: re.Match[str]) -> str: return _JINJA.sub(_sub, value), params -# Airflow macro -> the sql_task.parameters name + the DAB dynamic value it resolves to. Databricks -# requires dynamic references in SQL to go through named :markers + sql_task.parameters, never inline. -_SQL_MACRO_PARAM: dict[str, tuple[str, str]] = { - "ds": ("run_date", "{{job.start_time.iso_date}}"), - "ts": ("run_timestamp", "{{job.start_time.iso_datetime}}"), - "data_interval_start": ("data_interval_start", "{{job.start_time.iso_datetime}}"), - "data_interval_end": ("data_interval_end", "{{job.start_time.iso_datetime}}"), - "execution_date": ("execution_date", "{{job.start_time.iso_datetime}}"), - "logical_date": ("logical_date", "{{job.start_time.iso_datetime}}"), - "run_id": ("run_id", "{{job.run_id}}"), -} _SQL_IDENTIFIER_CONTEXT = re.compile( r"(?:\bFROM|\bJOIN|\bINTO|\bUPDATE|\bTABLE|\bVIEW|\bSCHEMA|\bCATALOG)\s*$", re.IGNORECASE, @@ -86,9 +102,10 @@ def convert_sql_template(sql: str) -> tuple[str, dict[str, str]]: """Rewrites Airflow Jinja in *sql* to ``:name`` markers + a ``sql_task.parameters`` map. Databricks requires dynamic references in a ``sql_task`` to be passed through named parameters, - not interpolated into the SQL text. ``{{ ds }}`` -> ``:run_date`` with - ``{"run_date": "{{job.start_time.iso_date}}"}``; ``{{ params.x }}`` -> ``:x`` with - ``{"x": "{{job.parameters.x}}"}``. Unknown expressions are left untouched. + not interpolated into the SQL text. A logical-date macro ``{{ ds }}`` -> ``:run_date`` with + ``{"run_date": "{{job.parameters.run_date}}"}`` (a job parameter, so a native backfill can override + it); ``{{ params.x }}`` -> ``:x`` with ``{"x": "{{job.parameters.x}}"}``; ``run_id`` binds to its + inline ref. Unknown expressions are left untouched. Returns ``(sql_with_markers, parameters)``. """ @@ -100,10 +117,13 @@ def _marker(name: str, match: re.Match[str]) -> str: def _sub(match: re.Match[str]) -> str: expr = match.group(1).strip() - if expr in _SQL_MACRO_PARAM: - name, ref = _SQL_MACRO_PARAM[expr] - parameters[name] = ref + if expr in _DATE_MACRO_PARAM: + name, _ = _DATE_MACRO_PARAM[expr] + parameters[name] = "{{job.parameters." + name + "}}" return _marker(name, match) + if expr in _MACRO_TO_DAB_REF: + parameters["run_id"] = _MACRO_TO_DAB_REF[expr] + return _marker("run_id", match) for pattern in _PARAM_PATTERNS: m = pattern.match(expr) if m: diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py index 7d9dde8..34c1026 100644 --- a/tests/unit/test_airflow_operators.py +++ b/tests/unit/test_airflow_operators.py @@ -963,19 +963,76 @@ def test_jinja_macros_convert_to_dab_refs_and_collect_params(): "from airflow import DAG\n" "from airflow.operators.python import PythonOperator\n" "def w(date=None, env=None):\n pass\n" - "with DAG(dag_id='d') as dag:\n" + "with DAG(dag_id='d', schedule_interval='0 6 * * *') as dag:\n" " t = PythonOperator(task_id='t', python_callable=w,\n" " op_kwargs={'date': '{{ ds }}', 'env': '{{ params.env }}'})\n" ) task = _by_key(p)["t"] # op_kwargs are JSON-encoded into the internal __flowx_op_kwargs widget; Jinja inside the - # values is still converted to DAB refs. + # values is still converted to DAB refs. {{ ds }} routes through a run_date job parameter (so a + # native backfill can override it), not an inline start_time ref. kwargs_json = task.base_parameters["__flowx_op_kwargs"] - assert "{{job.start_time.iso_date}}" in kwargs_json + assert "{{job.parameters.run_date}}" in kwargs_json assert "{{job.parameters.env}}" in kwargs_json - # The referenced param is declared on the pipeline with a (Databricks-required) default; the - # internal __flowx_ widget is NOT declared. - assert p.parameters == [{"name": "env", "default": ""}] + # Referenced params are declared with a (Databricks-required) default; run_date defaults to the + # scheduled trigger time on a cron job. The internal __flowx_ widget is NOT declared. + assert p.parameters == [ + {"name": "env", "default": ""}, + {"name": "run_date", "default": "{{job.trigger.time.iso_date}}"}, + ] + + +def test_execution_date_on_event_triggered_job_defaults_to_start_time(): + # A cron+sensor collapses to a file_arrival trigger -- no scheduled trigger time exists, so the + # run_date parameter approximates with the run start time (still overridable by a backfill). + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor\n" + "def w(date=None):\n pass\n" + "with DAG(dag_id='d') as dag:\n" + " wait = S3KeySensor(task_id='wait', bucket_key='s3://b/landing/')\n" + " t = PythonOperator(task_id='t', python_callable=w, op_kwargs={'date': '{{ execution_date }}'})\n" + " wait >> t\n" + ) + assert (p.schedule or {}).get("kind") == "file_arrival" + assert p.parameters == [{"name": "execution_date", "default": "{{job.start_time.iso_datetime}}"}] + + +def test_catchup_true_tags_pipeline_for_native_backfill(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d', schedule_interval='0 6 * * *', catchup=True) as dag:\n" + " t = PythonOperator(task_id='t', python_callable=w)\n" + ) + assert p.tags.get("airflow_catchup") == "true" + + +def test_catchup_false_leaves_no_backfill_tag(): + p = _load( + "from airflow import DAG\n" + "from airflow.operators.python import PythonOperator\n" + "def w():\n pass\n" + "with DAG(dag_id='d', schedule_interval='0 6 * * *', catchup=False) as dag:\n" + " t = PythonOperator(task_id='t', python_callable=w)\n" + ) + assert "airflow_catchup" not in p.tags + + +def test_dag_param_named_run_date_keeps_user_default(): + # An explicit params={'run_date': ...} default wins over the schedule-aware backfill default. + p = _load( + "from airflow import DAG\n" + "from airflow.models.param import Param\n" + "from airflow.operators.python import PythonOperator\n" + "def w(date=None):\n pass\n" + "with DAG(dag_id='d', schedule_interval='0 6 * * *', params={'run_date': Param('2024-01-01')}) as dag:\n" + " t = PythonOperator(task_id='t', python_callable=w, op_kwargs={'date': '{{ ds }}'})\n" + ) + run_date = next(param for param in p.parameters if param["name"] == "run_date") + assert run_date["default"] == "2024-01-01" def test_unsupported_airflow_macro_becomes_placeholder(): diff --git a/tests/unit/test_airflow_templating.py b/tests/unit/test_airflow_templating.py new file mode 100644 index 0000000..1c311d3 --- /dev/null +++ b/tests/unit/test_airflow_templating.py @@ -0,0 +1,49 @@ +"""Unit tests for Airflow Jinja -> DAB reference conversion (flowx.sources.airflow.templating).""" + +from __future__ import annotations + +from flowx.sources.airflow.templating import ( + convert_sql_template, + convert_template, + date_param_default, +) + + +def test_execution_date_macros_route_through_job_parameters(): + # Logical-date macros become an overridable job parameter (not an inline start_time ref) so a + # native Databricks backfill can override them per replayed window. + assert convert_template("{{ ds }}") == ("{{job.parameters.run_date}}", {"run_date"}) + assert convert_template("{{ execution_date }}") == ("{{job.parameters.execution_date}}", {"execution_date"}) + assert convert_template("{{ logical_date }}") == ("{{job.parameters.logical_date}}", {"logical_date"}) + + +def test_run_id_macro_stays_inline(): + # run_id has no backfill relevance -- it maps to its inline dynamic ref and declares no parameter. + assert convert_template("{{ run_id }}") == ("{{job.run_id}}", set()) + + +def test_dashless_macro_left_untouched(): + # ds_nodash has no dynamic-value form; leave it as an (unresolved) reference rather than emitting + # an invalid ref. + assert convert_template("{{ ds_nodash }}") == ("{{ ds_nodash }}", set()) + + +def test_sql_execution_date_binds_a_job_parameter(): + marked, params = convert_sql_template("SELECT * FROM t WHERE d = {{ ds }}") + assert marked == "SELECT * FROM t WHERE d = :run_date" + assert params == {"run_date": "{{job.parameters.run_date}}"} + + +def test_sql_run_id_binds_inline_ref(): + marked, params = convert_sql_template("SELECT '{{ run_id }}'") + assert marked == "SELECT ':run_id'" + assert params == {"run_id": "{{job.run_id}}"} + + +def test_date_param_default_is_schedule_aware(): + # Cron/periodic jobs have a scheduled trigger time (correct on normal runs, no start-time drift); + # event-triggered or unscheduled jobs approximate with the run start time. + assert date_param_default("iso_date", {"kind": "schedule"}) == "{{job.trigger.time.iso_date}}" + assert date_param_default("iso_datetime", {"kind": "periodic"}) == "{{job.trigger.time.iso_datetime}}" + assert date_param_default("iso_date", {"kind": "file_arrival"}) == "{{job.start_time.iso_date}}" + assert date_param_default("iso_date", None) == "{{job.start_time.iso_date}}" diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index 371eb9c..f90cb81 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -540,6 +540,23 @@ def test_neutralized_condition_renders_setup_section(self): assert "{{tasks._init_continue.values.continue}}" in md assert "`branch`" in md + def test_airflow_backfill_renders_setup_section(self): + # An Airflow catchup=True DAG surfaces a native-backfill section in SETUP.md so the + # run_date override path is documented rather than silently lost. + from flowx.bundler.prereqs_writer import build_prereqs, render_setup_md + + prereqs = build_prereqs( + notebooks=[], + tasks=[], + known_bundle_jobs=set(), + airflow_backfills=[{"pipeline": "daily_etl"}], + ) + assert not prereqs.is_empty() + md = render_setup_md(prereqs, bundle_name="b") + assert "Backfill (Airflow catchup)" in md + assert "{{backfill.iso_date}}" in md + assert "`daily_etl`" in md + def test_recurses_into_for_each_task_body(self): from flowx.bundler.dab_writer import _strip_dangling_task_value_refs From 119483294b7cb256a15c11671fbbebb8119cfdbc Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Wed, 29 Jul 2026 14:59:48 -0700 Subject: [PATCH 45/77] Resolve Airflow macros in BashOperator commands BashOperator/SSHOperator commands left {{ ds }} and friends inline, unlike PythonOperator. Convert each macro to a $name shell variable fed by a job-parameter widget exported to the shell environment (a DAB dynamic-value ref does not resolve inside %sh source), so run_date/ run_id/params resolve at run time with the same backfill-aware defaults. --- src/flowx/sources/airflow/loader.py | 13 +++--- src/flowx/sources/airflow/operators.py | 47 +++++++++++++------- src/flowx/sources/airflow/templating.py | 57 +++++++++++++++++++++++++ tests/unit/test_airflow_operators.py | 41 ++++++++++++++++++ tests/unit/test_airflow_templating.py | 24 +++++++++++ 5 files changed, 161 insertions(+), 21 deletions(-) diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py index 9a5cde5..2b9191c 100644 --- a/src/flowx/sources/airflow/loader.py +++ b/src/flowx/sources/airflow/loader.py @@ -1388,15 +1388,16 @@ def _reader(dep_var: str) -> str: def _declared_param_default(name: str, dag_params: dict[str, Any], schedule: dict[str, object] | None) -> Any: """Returns the Databricks-required default for a declared job parameter. - A DAG ``params={...}`` default wins. A logical-date parameter (``run_date`` etc., from an Airflow - ``{{ ds }}``/``execution_date`` macro) defaults to its schedule-aware time ref so a native backfill - can override it per replayed window. Everything else defaults to an empty string. + A DAG ``params={...}`` default wins. A macro-derived parameter (``run_date`` etc. from an Airflow + ``{{ ds }}``/``execution_date`` macro, or ``run_id``) gets its schedule-aware / inline default so + the value resolves at run time (and a native backfill can override a logical date). Everything else + defaults to an empty string. """ if dag_params.get(name) is not None: return dag_params[name] - field = templating.DATE_PARAM_FIELDS.get(name) - if field is not None: - return templating.date_param_default(field, schedule) + macro_default = templating.macro_param_default(name, schedule) + if macro_default is not None: + return macro_default return "" diff --git a/src/flowx/sources/airflow/operators.py b/src/flowx/sources/airflow/operators.py index d307947..316fb71 100644 --- a/src/flowx/sources/airflow/operators.py +++ b/src/flowx/sources/airflow/operators.py @@ -29,7 +29,7 @@ SparkPythonActivity, SqlActivity, ) -from flowx.sources.airflow import callable_notebook +from flowx.sources.airflow import callable_notebook, templating from flowx.utils import normalize_task_key # -------------------------------------------------------------------------------------- @@ -160,9 +160,24 @@ def notebook_from_callable( return callable_notebook.render(func, source, op_args=op_args, op_kwargs=op_kwargs) -def _sh_notebook(task_id: str, command: str) -> str: +def _sh_notebook(task_id: str, command: str, env_widgets: dict[str, str] | None = None) -> str: + """Renders a bash command as a ``%sh`` notebook. + + Airflow macros in the command are converted to ``$name`` shell variables (see + :func:`templating.convert_shell_template`); ``env_widgets`` maps each such name to the DAB + dynamic-value ref its widget resolves to. A Python cell reads those widgets and exports them as + environment variables so the following ``%sh`` cell (a subshell that inherits ``os.environ``) can + reference them -- a DAB ref does not resolve inside ``%sh`` source directly. + """ + header = _notebook_header(task_id, "BashOperator") + if env_widgets: + export = ["import os"] + for name in sorted(env_widgets): + export.append(f"dbutils.widgets.text({name!r}, '')") + export.append(f"os.environ[{name!r}] = dbutils.widgets.get({name!r})") + header += "\n".join(export) + "\n\n# COMMAND ----------\n\n" lines = "".join(f"# MAGIC {line}\n" for line in command.splitlines()) - return _notebook_header(task_id, "BashOperator") + "# MAGIC %sh\n" + lines + return header + "# MAGIC %sh\n" + lines # Airflow sensor defaults (seconds): poke every 60s, give up after 7 days. @@ -513,18 +528,25 @@ def _build_python(ctx: OperatorContext) -> Activity: ) +def _sh_notebook_activity(ctx: OperatorContext, command: str) -> NotebookActivity: + """Builds a %sh NotebookActivity, converting Airflow macros in the command to shell vars fed by + job-parameter widgets so ``{{ ds }}`` and friends resolve at run time.""" + converted, env_widgets = templating.convert_shell_template(command) + return NotebookActivity( + name=ctx.task_id, + task_key=ctx.task_key, + notebook_path=f"notebooks/{ctx.task_key}.py", + generated_source=_sh_notebook(ctx.task_id, converted, env_widgets), + ) + + def _build_bash(ctx: OperatorContext) -> Activity: command = literal_str(ctx.kwargs.get("bash_command")) if command is not None: submit = parse_spark_submit(command) if submit is not None: return _spark_activity_from_submit(ctx, submit, "BashOperator spark-submit") - return NotebookActivity( - name=ctx.task_id, - task_key=ctx.task_key, - notebook_path=f"notebooks/{ctx.task_key}.py", - generated_source=_sh_notebook(ctx.task_id, command), - ) + return _sh_notebook_activity(ctx, command) return _placeholder(ctx, "BashOperator command is not a string literal; supply the command manually.") @@ -535,12 +557,7 @@ def _build_ssh(ctx: OperatorContext) -> Activity: if submit is not None: # The SSH hop is eliminated -- Databricks runs Spark natively. return _spark_activity_from_submit(ctx, submit, "SSHOperator spark-submit") - return NotebookActivity( - name=ctx.task_id, - task_key=ctx.task_key, - notebook_path=f"notebooks/{ctx.task_key}.py", - generated_source=_sh_notebook(ctx.task_id, command), - ) + return _sh_notebook_activity(ctx, command) return _placeholder(ctx, "SSHOperator command is not a string literal; supply the command manually.") diff --git a/src/flowx/sources/airflow/templating.py b/src/flowx/sources/airflow/templating.py index ed312c0..0c10fea 100644 --- a/src/flowx/sources/airflow/templating.py +++ b/src/flowx/sources/airflow/templating.py @@ -51,6 +51,23 @@ def date_param_default(field: str, schedule: dict[str, object] | None) -> str: base = "{{job.trigger.time." if kind in ("schedule", "periodic") else "{{job.start_time." return f"{base}{field}}}}}" + +def macro_param_default(name: str, schedule: dict[str, object] | None) -> str | None: + """Returns the Databricks-required default for a macro-derived job parameter, or None. + + A logical-date parameter (``run_date`` etc.) gets its schedule-aware time ref; ``run_id`` gets the + inline run-id ref (bash/env-var threading forces even ``run_id`` through a job parameter, and its + default must resolve to the run id rather than an empty string). Any other name is not + macro-derived, so this returns None and the caller falls back to its own default. + """ + field = DATE_PARAM_FIELDS.get(name) + if field is not None: + return date_param_default(field, schedule) + if name == "run_id": + return _MACRO_TO_DAB_REF["run_id"] + return None + + # {{ params.X }} / {{ var.value.X }} / {{ dag_run.conf['X'] }} -> {{job.parameters.X}} _PARAM_PATTERNS: list[re.Pattern[str]] = [ re.compile(r"^params\.([A-Za-z_][A-Za-z0-9_]*)$"), @@ -135,6 +152,46 @@ def _sub(match: re.Match[str]) -> str: return _JINJA.sub(_sub, sql), parameters +# Shell-safe env var name from a job-parameter name (bash disallows the same characters as the param +# patterns already restrict, so this is a straight pass-through kept for intent/clarity). +def _shell_var(name: str) -> str: + return name + + +def convert_shell_template(command: str) -> tuple[str, dict[str, str]]: + """Rewrites Airflow Jinja in a bash command to ``$NAME`` shell variable references. + + A DAB dynamic-value ref (``{{job.parameters.X}}``) only resolves in a task *parameter* value, not + inside ``%sh`` notebook source, so a bash macro can't be replaced inline. Instead each recognised + macro becomes a ``$name`` shell variable the runner notebook exports from a widget of the same + name. ``{{ ds }}`` -> ``$run_date``; ``{{ params.x }}`` -> ``$x``; ``run_id`` -> ``$run_id``. + Unknown expressions are left untouched. + + Returns ``(command_with_shell_vars, {name: dynamic_value_ref})`` where each ref is what the widget + of that name must resolve to (a job parameter, or an inline ref for run_id). + """ + bindings: dict[str, str] = {} + + def _sub(match: re.Match[str]) -> str: + expr = match.group(1).strip() + if expr in _DATE_MACRO_PARAM: + name, _ = _DATE_MACRO_PARAM[expr] + bindings[name] = "{{job.parameters." + name + "}}" + return f"${_shell_var(name)}" + if expr in _MACRO_TO_DAB_REF: + bindings["run_id"] = _MACRO_TO_DAB_REF[expr] + return f"${_shell_var('run_id')}" + for pattern in _PARAM_PATTERNS: + m = pattern.match(expr) + if m: + name = m.group(1) + bindings[name] = "{{job.parameters." + name + "}}" + return f"${_shell_var(name)}" + return match.group(0) + + return _JINJA.sub(_sub, command), bindings + + def convert_params(value: Any) -> tuple[Any, set[str]]: """Recursively converts templates in a str / list / dict value. diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py index 34c1026..727a328 100644 --- a/tests/unit/test_airflow_operators.py +++ b/tests/unit/test_airflow_operators.py @@ -182,6 +182,47 @@ def test_bash_operator_becomes_sh_notebook(): task = _by_key(p)["clean"] assert isinstance(task, NotebookActivity) assert "%sh" in task.generated_source + # No macros -> no widget prelude, just the %sh cell. + assert "dbutils.widgets" not in task.generated_source + + +def test_bash_operator_macros_thread_through_shell_env_vars(): + # A BashOperator with Airflow macros must resolve them at run time: each macro becomes a $var fed + # by a job-parameter widget exported to the shell env, not a literal left in the command. + p = _load( + "from airflow import DAG\n" + "from airflow.operators.bash import BashOperator\n" + "with DAG(dag_id='d', schedule_interval='0 6 * * *') as dag:\n" + " t = BashOperator(task_id='run',\n" + " bash_command='python /opt/etl.py --date {{ ds }} --env {{ params.env }}')\n" + ) + task = _by_key(p)["run"] + assert isinstance(task, NotebookActivity) + # Macros converted to shell variables; the raw {{ ... }} is gone from the %sh cell. + assert "--date $run_date" in task.generated_source + assert "--env $env" in task.generated_source + assert "{{ ds }}" not in task.generated_source + # The widgets are declared and exported to the environment before the %sh cell. + assert "os.environ['run_date'] = dbutils.widgets.get('run_date')" in task.generated_source + compile("\n".join(task.generated_source.split("# MAGIC %sh")[0].splitlines()), "
", "exec")
+    # run_date declared as a job parameter with the schedule-aware default (backfill-overridable).
+    params = {param["name"]: param["default"] for param in p.parameters}
+    assert params["run_date"] == "{{job.trigger.time.iso_date}}"
+    assert params["env"] == ""
+
+
+def test_bash_operator_run_id_macro_defaults_to_run_id_ref():
+    # run_id has no user default: threaded through a widget whose default resolves to the run id.
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='d') as dag:\n"
+        "    t = BashOperator(task_id='run', bash_command='echo {{ run_id }}')\n"
+    )
+    task = _by_key(p)["run"]
+    assert "echo $run_id" in task.generated_source
+    params = {param["name"]: param["default"] for param in p.parameters}
+    assert params["run_id"] == "{{job.run_id}}"
 
 
 def test_bash_operator_wrapping_spark_submit_becomes_spark_task():
diff --git a/tests/unit/test_airflow_templating.py b/tests/unit/test_airflow_templating.py
index 1c311d3..fe4cc6a 100644
--- a/tests/unit/test_airflow_templating.py
+++ b/tests/unit/test_airflow_templating.py
@@ -3,9 +3,11 @@
 from __future__ import annotations
 
 from flowx.sources.airflow.templating import (
+    convert_shell_template,
     convert_sql_template,
     convert_template,
     date_param_default,
+    macro_param_default,
 )
 
 
@@ -47,3 +49,25 @@ def test_date_param_default_is_schedule_aware():
     assert date_param_default("iso_datetime", {"kind": "periodic"}) == "{{job.trigger.time.iso_datetime}}"
     assert date_param_default("iso_date", {"kind": "file_arrival"}) == "{{job.start_time.iso_date}}"
     assert date_param_default("iso_date", None) == "{{job.start_time.iso_date}}"
+
+
+def test_shell_template_threads_macros_through_named_vars():
+    command, bindings = convert_shell_template("etl.py --date {{ ds }} --run {{ run_id }} --env {{ params.env }}")
+    assert command == "etl.py --date $run_date --run $run_id --env $env"
+    assert bindings == {
+        "run_date": "{{job.parameters.run_date}}",
+        "run_id": "{{job.run_id}}",
+        "env": "{{job.parameters.env}}",
+    }
+
+
+def test_shell_template_leaves_unknown_expressions():
+    command, bindings = convert_shell_template("echo {{ some.unknown }}")
+    assert command == "echo {{ some.unknown }}"
+    assert bindings == {}
+
+
+def test_macro_param_default_covers_date_and_run_id_and_none():
+    assert macro_param_default("run_date", {"kind": "schedule"}) == "{{job.trigger.time.iso_date}}"
+    assert macro_param_default("run_id", None) == "{{job.run_id}}"
+    assert macro_param_default("env", None) is None  # a user param, not macro-derived

From fa5c14c8db504057b0cd5a10abe48e805c91abde Mon Sep 17 00:00:00 2001
From: Zanita Rahimi 
Date: Thu, 30 Jul 2026 14:44:01 +0200
Subject: [PATCH 46/77] docs: fix broken internal links missing /flowx base
 path (#16)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The docs site is served under basePath `/flowx` (docs/next.config.mjs),
but two internal links omit it and 404 on the published site:

- `guide.mdx` → `[Configuration options](/docs/options)` (the
`record-results`/`install-dashboard` reference)
- `architecture.mdx` →
`[Installation](/docs/installation#running-flowx-as-an-mcp-server)`

Both now use the `/flowx/docs/...` form, matching the links in
`index.mdx`.
---
 docs/content/docs/architecture.mdx | 2 +-
 docs/content/docs/guide.mdx        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx
index fbdb0f8..a8e1b67 100644
--- a/docs/content/docs/architecture.mdx
+++ b/docs/content/docs/architecture.mdx
@@ -78,7 +78,7 @@ The MCP server runs in whichever transport fits the calling tool. This is chosen
                                                                        own service principal
 ```
 
-See [Installation](/docs/installation#running-flowx-as-an-mcp-server) for the exact commands and the [app README](https://github.com/databricks-solutions/flowx/tree/main/app) for deployment details.
+See [Installation](/flowx/docs/installation#running-flowx-as-an-mcp-server) for the exact commands and the [app README](https://github.com/databricks-solutions/flowx/tree/main/app) for deployment details.
 
 
 A Databricks App can't read the user's workspace / UC Volume files (`/Volumes/...` is **not** auto-mounted). Two ways to get data in/out of the `flowx` tool:
diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx
index d758899..d84d0d6 100644
--- a/docs/content/docs/guide.mdx
+++ b/docs/content/docs/guide.mdx
@@ -81,7 +81,7 @@ Run the setup scripts and populate secret values before deploying and running pi
 When running with workspace auth (e.g. Genie Code), `package` can optionally persist this run's
 coverage to a Unity Catalog table — one row per pipeline stamped with a UUID `run_id`, `run_date`,
 and `run_by` (`record-results`) — and install a published AI/BI coverage dashboard over that table
-(`install-dashboard`). See [Configuration options](/docs/options) for details.
+(`install-dashboard`). See [Configuration options](/flowx/docs/options) for details.
 
 
 

From 98dbc239905ddea7afaaae2d53e968482e7d0734 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Mon, 3 Aug 2026 11:44:54 -0700
Subject: [PATCH 47/77] Fix silent-wrong-output paths found in review

- Bind %sh macro widgets to their job parameters; unbound widgets were
  backfilled with '' so commands ran with blank values.
- Route an unresolvable python_callable to a placeholder instead of a
  bodyless notebook counted as fully covered.
- Never emit both day-of-month and day-of-week in Quartz (rejected), and
  split week-wrapping weekday ranges that became descending ranges.
- Take for_each inputs from .expand() kwargs only, not a list-valued
  .partial() fixed arg.
- Recurse into for_each bodies when collecting gaps and classifying the
  inventory so nested placeholders are not dropped.
- Fix the golden-bundle quoting assertion and drop dead code.
---
 src/flowx/sources/airflow/convert.py          |  5 +-
 src/flowx/sources/airflow/discover.py         |  9 +-
 src/flowx/sources/airflow/loader.py           | 83 ++++++++++---------
 src/flowx/sources/airflow/operators.py        | 45 ++++++----
 src/flowx/sources/airflow/templating.py       | 18 ++--
 .../integration/test_airflow_golden_bundle.py |  3 +-
 tests/unit/test_airflow_operators.py          | 55 ++++++++++++
 tests/unit/test_airflow_templating.py         | 22 ++++-
 8 files changed, 171 insertions(+), 69 deletions(-)

diff --git a/src/flowx/sources/airflow/convert.py b/src/flowx/sources/airflow/convert.py
index b3d57fb..9137c8f 100644
--- a/src/flowx/sources/airflow/convert.py
+++ b/src/flowx/sources/airflow/convert.py
@@ -15,6 +15,7 @@
 from pathlib import Path
 
 from flowx import ir_serde
+from flowx.adapter.predicates import walk_activities
 from flowx.ir_serde import pipeline_to_dict
 from flowx.models.ir import PlaceholderActivity
 from flowx.sources.airflow.loader import load_pipelines
@@ -105,7 +106,9 @@ def _collect_gaps(pipelines: list) -> list[dict]:
     """
     gaps: list[dict] = []
     for pipeline in pipelines:
-        for task in pipeline.tasks:
+        # Descend into for_each bodies: a mapped operator's placeholder lives in inner_activities, and
+        # a gap the agentic round never sees is guidance generated and dropped.
+        for task in walk_activities(pipeline.tasks):
             if isinstance(task, PlaceholderActivity):
                 gaps.append(
                     {
diff --git a/src/flowx/sources/airflow/discover.py b/src/flowx/sources/airflow/discover.py
index fce6d93..a867785 100644
--- a/src/flowx/sources/airflow/discover.py
+++ b/src/flowx/sources/airflow/discover.py
@@ -16,6 +16,7 @@
 from pathlib import Path
 from typing import Any
 
+from flowx.adapter.predicates import walk_activities
 from flowx.models.ir import NotebookActivity, Pipeline, PlaceholderActivity
 from flowx.sources.adf.loader import clear_stale_outputs
 from flowx.sources.airflow.loader import load_pipelines
@@ -24,9 +25,13 @@
 
 
 def _classify(pipeline: Pipeline) -> list[dict[str, str]]:
-    """Classifies each task in *pipeline* for the inventory."""
+    """Classifies each task in *pipeline* for the inventory.
+
+    Descends into for_each bodies so a mapped operator's nested placeholder is counted as agentic
+    rather than being invisible to the inventory (which would report full coverage).
+    """
     items: list[dict[str, str]] = []
-    for task in pipeline.tasks:
+    for task in walk_activities(pipeline.tasks):
         if isinstance(task, NotebookActivity):
             strategy = "deterministic"
         elif isinstance(task, PlaceholderActivity):
diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py
index 2b9191c..2720cae 100644
--- a/src/flowx/sources/airflow/loader.py
+++ b/src/flowx/sources/airflow/loader.py
@@ -70,8 +70,6 @@ class _TaskFlowTask:
 
 def _sanitize_task_key(name: str) -> str:
     """Converts an Airflow task_id into a valid Databricks task key."""
-    import re
-
     key = re.sub(r"[^a-zA-Z0-9_-]", "_", name)
     key = re.sub(r"_+", "_", key).strip("_")
     return key or "unnamed"
@@ -118,7 +116,14 @@ def _shift_part(part: str) -> str:
             step = "/" + step
         if "-" in part:
             lo, _, hi = part.partition("-")
-            return f"{_shift_token(lo)}-{_shift_token(hi)}{step}"
+            shifted_lo, shifted_hi = _shift_token(lo), _shift_token(hi)
+            # A range that wraps the week in Unix numbering (e.g. 5-0, Fri-Sun) shifts to a descending
+            # range Quartz reads as empty; split it at the week boundary instead (6-7,1).
+            if shifted_lo.isdigit() and shifted_hi.isdigit() and int(shifted_lo) > int(shifted_hi):
+                head = shifted_lo if shifted_lo == "7" else f"{shifted_lo}-7"
+                tail = shifted_hi if shifted_hi == "1" else f"1-{shifted_hi}"
+                return f"{head},{tail}{step}"
+            return f"{shifted_lo}-{shifted_hi}{step}"
         return f"{_shift_token(part)}{step}"
 
     return ",".join(_shift_part(p) for p in dow.split(","))
@@ -139,7 +144,12 @@ def _cron_to_quartz(cron: str) -> str | None:
     minute, hour, dom, month, dow = fields
     if dow not in ("*", "?"):
         dow = _shift_weekday_field(dow)
-    if dow == "*" and dom != "*":
+    if dom != "*" and dow not in ("*", "?"):
+        # Unix cron ORs a restricted day-of-month with a restricted day-of-week; Quartz cannot express
+        # both (it rejects the expression outright). Keep the day-of-week and drop the day-of-month so
+        # the job is still valid -- narrower than the Airflow schedule, and flagged for review.
+        dom = "?"
+    elif dow == "*" and dom != "*":
         dow = "?"
     elif dom == "*":
         dom = "?"
@@ -270,9 +280,7 @@ class _DagVisitor(ast.NodeVisitor):
     """Collects operator calls, dependency edges, and the DAG's schedule."""
 
     def __init__(self, module: ast.Module) -> None:
-        self._functions: dict[str, ast.FunctionDef] = {
-            node.name: node for node in _iter_functions(module) if isinstance(node, ast.FunctionDef)
-        }
+        self._functions: dict[str, ast.FunctionDef] = {node.name: node for node in _iter_functions(module)}
         # task variable name -> (task_id, operator, kwargs)
         self.operators: dict[str, tuple[str, str, dict[str, ast.expr]]] = {}
         # task variable name -> the operator's ast.Call node (for source-slicing placeholders)
@@ -297,6 +305,9 @@ def __init__(self, module: ast.Module) -> None:
         self.group_vars: dict[str, str] = {}
         # task variable names defined via dynamic mapping (.expand()) -> wrapped in a for_each
         self.mapped: set[str] = set()
+        # mapped var -> the kwarg names passed to .expand(). Only these fan out; a list-valued
+        # .partial() arg is a fixed value and must not be mistaken for the mapped iterable.
+        self.expand_kwargs: dict[str, list[str]] = {}
         # TaskFlow: function name -> (FunctionDef, decorator dotted-name) for @task-decorated defs.
         # Pre-scanned so a @task def defined after the @dag body that uses it is still resolved.
         self.taskflow_defs: dict[str, tuple[ast.FunctionDef, str]] = {}
@@ -343,7 +354,7 @@ def visit_Assign(self, node: ast.Assign) -> None:
             var = node.targets[0].id
             direct = _direct_operator_call(node.value)
             mapped = None if direct is not None else _mapped_operator_call(node.value)
-            call = direct or mapped
+            call = direct or (mapped[0] if mapped is not None else None)
             if call is not None and isinstance(call.func, ast.Name):
                 construct = call.func.id
                 kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg}
@@ -352,6 +363,7 @@ def visit_Assign(self, node: ast.Assign) -> None:
                 self.calls[var] = call
                 if mapped is not None:
                     self.mapped.add(var)
+                    self.expand_kwargs[var] = mapped[1]
                 if self._group_stack:
                     self.groups[var] = "__".join(self._group_stack)
             elif self._register_taskflow_call(node.value, var):
@@ -648,7 +660,6 @@ def _collect_set_dependency(self, call: ast.Call) -> None:
 
 def _expand_group_edges(
     edges: list[tuple[str, str]],
-    operators: dict[str, tuple[str, str, dict[str, ast.expr]]],
     groups: dict[str, str],
     group_vars: dict[str, str],
 ) -> list[tuple[str, str]]:
@@ -745,11 +756,6 @@ def _iter_functions(module: ast.Module) -> list[ast.FunctionDef]:
     return found
 
 
-def _referenced_names(node: ast.expr) -> set[str]:
-    """Every bare Name id loaded anywhere in *node* (for TaskFlow data-flow edge detection)."""
-    return {n.id for n in ast.walk(node) if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Load)}
-
-
 def _names_in(node: ast.expr) -> list[str]:
     """Returns the task-variable names in a Name or a ``[Name, ...]`` list node."""
     if isinstance(node, ast.Name):
@@ -767,17 +773,6 @@ def _literal_argument_source(node: ast.expr) -> str | None:
         return None
 
 
-def _flatten_shift_nodes(node: ast.expr) -> list[ast.expr]:
-    """Flattens a ``>>`` / ``<<`` chain into its per-position operand nodes, left to right.
-
-    ``a >> [b, c] >> d()`` becomes ``[Name('a'), List([b, c]), Call(d)]``; the caller resolves each
-    position to task vars (registering an inline TaskFlow call along the way).
-    """
-    if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.RShift, ast.LShift)):
-        return _flatten_shift_nodes(node.left) + _flatten_shift_nodes(node.right)
-    return [node]
-
-
 def _direct_operator_call(node: ast.Call) -> ast.Call | None:
     """Returns *node* if it is a direct ``SomeOperator(...)`` / ``SomeSensor(...)`` call."""
     if isinstance(node.func, ast.Name) and _is_task_construct(node.func.id):
@@ -785,13 +780,14 @@ def _direct_operator_call(node: ast.Call) -> ast.Call | None:
     return None
 
 
-def _mapped_operator_call(node: ast.Call) -> ast.Call | None:
+def _mapped_operator_call(node: ast.Call) -> tuple[ast.Call, list[str]] | None:
     """Returns the underlying operator call for a dynamic-mapping ``.expand(...)`` chain.
 
-    Handles ``Op(...).expand(...)`` and ``Op.partial(...).expand(...)``. The returned
-    Call's keywords are the merged operator kwargs (partial args + expand args), and its
-    ``.func`` is the operator Name, so the caller treats it like a direct operator call.
-    The mapped kwargs let the loader wrap the operator in a for_each_task.
+    Handles ``Op(...).expand(...)`` and ``Op.partial(...).expand(...)``. Returns
+    ``(merged_call, expand_kwarg_names)``: the Call's keywords are the merged operator kwargs
+    (partial args + expand args) and its ``.func`` is the operator Name, so the caller treats it like a
+    direct operator call. The expand kwarg names are returned separately because only those are
+    fanned out -- a list-valued ``.partial()`` arg is a fixed value, not the mapped iterable.
     """
     if not (isinstance(node.func, ast.Attribute) and node.func.attr == "expand"):
         return None
@@ -814,7 +810,7 @@ def _mapped_operator_call(node: ast.Call) -> ast.Call | None:
         args=[],
         keywords=list(inner.keywords) + list(node.keywords),
     )
-    return merged
+    return merged, [kw.arg for kw in node.keywords if kw.arg]
 
 
 def _is_task_construct(name: str) -> bool:
@@ -947,7 +943,7 @@ def _task_key(var: str, task_id: str) -> str:
     # Expand group-level edges (`group_a >> group_b`, `task >> group`, ...) into edges between the
     # groups' boundary tasks: leaves of the upstream group -> roots of the downstream group, matching
     # Airflow's TaskGroup dependency semantics. A non-group var resolves to itself.
-    edges = _expand_group_edges(visitor.edges, visitor.operators, visitor.groups, visitor.group_vars)
+    edges = _expand_group_edges(visitor.edges, visitor.groups, visitor.group_vars)
 
     # Build the upstream adjacency in dependency terms, then drop structural nodes
     # (Dummy/Empty and lifted root sensors) by rewiring their downstreams to their upstreams.
@@ -1112,7 +1108,11 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
 
         if var in visitor.mapped:
             # Dynamic mapping (.expand()) -> a for_each_task iterating the mapped operator.
-            tasks.append(_wrap_in_for_each(activity, task_id, task_key, depends_on, kwargs))
+            tasks.append(
+                _wrap_in_for_each(
+                    activity, task_id, task_key, depends_on, kwargs, visitor.expand_kwargs.get(var) or []
+                )
+            )
         else:
             tasks.append(activity)
 
@@ -1212,18 +1212,21 @@ def _wrap_in_for_each(
     task_key: str,
     depends_on: list[Dependency] | None,
     kwargs: dict[str, ast.expr],
+    expand_kwargs: list[str],
 ) -> ForEachActivity:
     """Wraps a dynamically-mapped operator in a ForEachActivity (-> for_each_task).
 
-    Airflow ``.expand(x=[...])`` fans a task out over an iterable. The for_each's
-    ``inputs`` is the first list-valued expand kwarg (rendered as a JSON array literal
-    when it is a static list; otherwise ``{{job.parameters...}}`` is left for review).
-    The mapped operator becomes the single inner activity, re-keyed so it doesn't
-    collide with the for_each task key.
+    Airflow ``.expand(x=[...])`` fans a task out over an iterable. The for_each's ``inputs`` is the
+    first list-valued kwarg passed to ``.expand()`` -- restricted to *expand* kwargs because a
+    list-valued ``.partial()`` arg is a fixed value, and taking it would fan the task out over the
+    wrong list. The mapped operator becomes the single inner activity, re-keyed so it doesn't collide
+    with the for_each task key.
     """
     items = "[]"
-    for key, node in kwargs.items():
-        if key in ("task_id", "group_id"):
+    candidates = expand_kwargs or [key for key in kwargs if key not in ("task_id", "group_id")]
+    for key in candidates:
+        node = kwargs.get(key)
+        if node is None or key in ("task_id", "group_id"):
             continue
         value = ops.literal_value(node)
         if isinstance(value, list):
diff --git a/src/flowx/sources/airflow/operators.py b/src/flowx/sources/airflow/operators.py
index 316fb71..64de9a8 100644
--- a/src/flowx/sources/airflow/operators.py
+++ b/src/flowx/sources/airflow/operators.py
@@ -459,12 +459,17 @@ def parse_spark_submit(command: str) -> _SparkSubmit | None:
 
 
 def _spark_activity_from_submit(ctx: OperatorContext, submit: _SparkSubmit, note: str) -> Activity:
-    """Builds a Spark JAR/Python activity from a parsed spark-submit."""
+    """Builds a Spark JAR/Python activity from a parsed spark-submit.
+
+    ``note`` records which operator the spark-submit came from and is carried as the activity
+    description so the emitted task states its provenance.
+    """
     app = submit.application or ""
     if submit.java_class or app.endswith(".jar"):
         activity: Activity = SparkJarActivity(
             name=ctx.task_id,
             task_key=ctx.task_key,
+            description=f"Migrated from Airflow {note}.",
             main_class_name=submit.java_class or "UNKNOWN_MAIN_CLASS",
             parameters=submit.app_args or None,
             libraries=[{"jar": app}] if app else None,
@@ -473,6 +478,7 @@ def _spark_activity_from_submit(ctx: OperatorContext, submit: _SparkSubmit, note
         activity = SparkPythonActivity(
             name=ctx.task_id,
             task_key=ctx.task_key,
+            description=f"Migrated from Airflow {note}.",
             python_file=app or f"../src/{ctx.task_key}.py",
             parameters=submit.app_args or None,
         )
@@ -486,17 +492,25 @@ def _spark_activity_from_submit(ctx: OperatorContext, submit: _SparkSubmit, note
 
 def _build_python(ctx: OperatorContext) -> Activity:
     func = ctx.functions.get(callable_name(ctx.kwargs.get("python_callable")) or "")
+    # The callable is not defined in this DAG module (commonly imported from a helper package), so
+    # there is no source to render -- route it to the agentic-gap round rather than emitting a
+    # notebook with no body.
+    if func is None:
+        return _placeholder(
+            ctx,
+            f"{ctx.operator} python_callable could not be resolved in the DAG module "
+            "(likely imported from another module); port the callable manually.",
+        )
     # A callable that reads Airflow task context (**context / ti) or XCom can't run as a plain
     # notebook -- route it to the agentic-gap round instead of emitting code that fails at runtime.
-    if func is not None:
-        reason = callable_notebook.airflow_runtime_reason(func, ctx.source)
-        if reason is not None:
-            return _placeholder(
-                ctx,
-                f"Airflow {ctx.operator} {reason}. flowx has no Airflow runtime to supply it; "
-                "translate manually -- pass upstream data via job parameters or map XCom to "
-                "dbutils.jobs.taskValues (set in the producer, get in the consumer).",
-            )
+    reason = callable_notebook.airflow_runtime_reason(func, ctx.source)
+    if reason is not None:
+        return _placeholder(
+            ctx,
+            f"Airflow {ctx.operator} {reason}. flowx has no Airflow runtime to supply it; "
+            "translate manually -- pass upstream data via job parameters or map XCom to "
+            "dbutils.jobs.taskValues (set in the producer, get in the consumer).",
+        )
     op_kwargs_node = ctx.kwargs.get("op_kwargs")
     op_args_node = ctx.kwargs.get("op_args")
     op_kwargs = literal_value(op_kwargs_node)
@@ -509,9 +523,7 @@ def _build_python(ctx: OperatorContext) -> Activity:
         op_args = list(op_args)
     has_kwargs = isinstance(op_kwargs, dict)
     has_args = isinstance(op_args, list)
-    generated = (
-        notebook_from_callable(func, ctx.source, op_args=has_args, op_kwargs=has_kwargs) if func is not None else None
-    )
+    generated = notebook_from_callable(func, ctx.source, op_args=has_args, op_kwargs=has_kwargs)
     # op_args/op_kwargs pass as JSON widgets so lists/numbers/nested objects survive; the notebook
     # json.loads() them and splats into the call.
     base_parameters: dict[str, str] = {}
@@ -530,13 +542,18 @@ def _build_python(ctx: OperatorContext) -> Activity:
 
 def _sh_notebook_activity(ctx: OperatorContext, command: str) -> NotebookActivity:
     """Builds a %sh NotebookActivity, converting Airflow macros in the command to shell vars fed by
-    job-parameter widgets so ``{{ ds }}`` and friends resolve at run time."""
+    job-parameter widgets so ``{{ ds }}`` and friends resolve at run time.
+
+    ``base_parameters`` must carry the dynamic-value ref for each widget: an unbound widget is
+    backfilled with an empty string by the bundler, which would run the command with blank values.
+    """
     converted, env_widgets = templating.convert_shell_template(command)
     return NotebookActivity(
         name=ctx.task_id,
         task_key=ctx.task_key,
         notebook_path=f"notebooks/{ctx.task_key}.py",
         generated_source=_sh_notebook(ctx.task_id, converted, env_widgets),
+        base_parameters=dict(env_widgets) or None,
     )
 
 
diff --git a/src/flowx/sources/airflow/templating.py b/src/flowx/sources/airflow/templating.py
index 0c10fea..e54dcdc 100644
--- a/src/flowx/sources/airflow/templating.py
+++ b/src/flowx/sources/airflow/templating.py
@@ -152,12 +152,6 @@ def _sub(match: re.Match[str]) -> str:
     return _JINJA.sub(_sub, sql), parameters
 
 
-# Shell-safe env var name from a job-parameter name (bash disallows the same characters as the param
-# patterns already restrict, so this is a straight pass-through kept for intent/clarity).
-def _shell_var(name: str) -> str:
-    return name
-
-
 def convert_shell_template(command: str) -> tuple[str, dict[str, str]]:
     """Rewrites Airflow Jinja in a bash command to ``$NAME`` shell variable references.
 
@@ -177,16 +171,16 @@ def _sub(match: re.Match[str]) -> str:
         if expr in _DATE_MACRO_PARAM:
             name, _ = _DATE_MACRO_PARAM[expr]
             bindings[name] = "{{job.parameters." + name + "}}"
-            return f"${_shell_var(name)}"
+            return f"${name}"
         if expr in _MACRO_TO_DAB_REF:
             bindings["run_id"] = _MACRO_TO_DAB_REF[expr]
-            return f"${_shell_var('run_id')}"
+            return "$run_id"
         for pattern in _PARAM_PATTERNS:
             m = pattern.match(expr)
             if m:
                 name = m.group(1)
                 bindings[name] = "{{job.parameters." + name + "}}"
-                return f"${_shell_var(name)}"
+                return f"${name}"
         return match.group(0)
 
     return _JINJA.sub(_sub, command), bindings
@@ -292,7 +286,11 @@ def pick(key: str) -> ast.expr | None:
 
 
 def email_on_failure(dag_default_args: dict[str, ast.expr], task_kwargs: dict[str, ast.expr]) -> list[str]:
-    """Returns email recipients when email_on_failure is set (for a job-level notification note)."""
+    """Returns email recipients when email_on_failure is set (for a job-level notification note).
+
+    TODO: not wired up yet -- the shared IR has no email-notification field, so carrying these through
+    to a job's ``email_notifications`` needs an IR addition (tracked separately).
+    """
     on_failure = task_kwargs.get("email_on_failure", dag_default_args.get("email_on_failure"))
     if isinstance(on_failure, ast.Constant) and on_failure.value is False:
         return []
diff --git a/tests/integration/test_airflow_golden_bundle.py b/tests/integration/test_airflow_golden_bundle.py
index 7e47bbe..7dba9e6 100644
--- a/tests/integration/test_airflow_golden_bundle.py
+++ b/tests/integration/test_airflow_golden_bundle.py
@@ -80,7 +80,8 @@ def test_root_file_sensor_is_polling_task(bundle_dir: Path):
 
 def test_mid_dag_table_sensor_is_polling_task(bundle_dir: Path):
     src = (bundle_dir / "src" / "notebooks" / "wait_partition.py").read_text()
-    assert 'spark.catalog.tableExists("main.analytics.raw_orders")' in src
+    # The table name is emitted through repr() so a name containing a quote can't break the source.
+    assert f"spark.catalog.tableExists({'main.analytics.raw_orders'!r})" in src
     assert "POKE_INTERVAL = 60" in src
     ast.parse(src)
 
diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py
index 727a328..8e0e3c1 100644
--- a/tests/unit/test_airflow_operators.py
+++ b/tests/unit/test_airflow_operators.py
@@ -205,12 +205,33 @@ def test_bash_operator_macros_thread_through_shell_env_vars():
     # The widgets are declared and exported to the environment before the %sh cell.
     assert "os.environ['run_date'] = dbutils.widgets.get('run_date')" in task.generated_source
     compile("\n".join(task.generated_source.split("# MAGIC %sh")[0].splitlines()), "
", "exec")
+    # Each widget must be BOUND to its job parameter: an unbound widget is backfilled with an empty
+    # string by the bundler, so the command would silently run with blank values.
+    assert task.base_parameters == {
+        "run_date": "{{job.parameters.run_date}}",
+        "env": "{{job.parameters.env}}",
+    }
     # run_date declared as a job parameter with the schedule-aware default (backfill-overridable).
     params = {param["name"]: param["default"] for param in p.parameters}
     assert params["run_date"] == "{{job.trigger.time.iso_date}}"
     assert params["env"] == ""
 
 
+def test_python_operator_with_unresolvable_callable_becomes_placeholder():
+    # python_callable imported from another module has no source to render -- it must become a
+    # placeholder (a real gaps.json entry), not a bodyless notebook counted as deterministic.
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.python import PythonOperator\n"
+        "import my_module\n"
+        "with DAG(dag_id='d') as dag:\n"
+        "    a = PythonOperator(task_id='a', python_callable=my_module.etl_step)\n"
+    )
+    task = _by_key(p)["a"]
+    assert isinstance(task, PlaceholderActivity)
+    assert "could not be resolved" in task.comment
+
+
 def test_bash_operator_run_id_macro_defaults_to_run_id_ref():
     # run_id has no user default: threaded through a widget whose default resolves to the run id.
     p = _load(
@@ -1206,6 +1227,40 @@ def test_expand_direct_call_form():
     assert isinstance(_by_key(p)["run"], ForEachActivity)
 
 
+def test_for_each_inputs_come_from_expand_not_partial():
+    # A list-valued .partial() arg is a FIXED value; only the .expand() kwarg is fanned out. Taking the
+    # partial list would iterate the wrong values (and the wrong count).
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='d') as dag:\n"
+        "    m = BashOperator.partial(task_id='t', env=['FIXED1', 'FIXED2']).expand(\n"
+        "        bash_command=['echo a', 'echo b', 'echo c'])\n"
+    )
+    task = _by_key(p)["t"]
+    assert isinstance(task, ForEachActivity)
+    assert task.items_expression == '["echo a", "echo b", "echo c"]'
+
+
+def test_placeholder_nested_in_for_each_is_collected_as_a_gap():
+    # A mapped operator whose command isn't a literal becomes a PlaceholderActivity INSIDE the
+    # for_each; gaps.json and the inventory must see it, or the guidance is generated then dropped.
+    from flowx.sources.airflow.convert import _collect_gaps
+    from flowx.sources.airflow.discover import _classify
+
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='d') as dag:\n"
+        "    m = BashOperator.partial(task_id='t').expand(bash_command=['echo a', 'echo b'])\n"
+    )
+    outer = _by_key(p)["t"]
+    assert isinstance(outer, ForEachActivity)
+    assert isinstance(outer.inner_activities[0], PlaceholderActivity)
+    assert len(_collect_gaps([p])) == 1
+    assert [item["strategy"] for item in _classify(p)].count("agentic") == 1
+
+
 def test_mixed_shift_directions_preserve_each_operator_direction():
     p = _load(
         "from airflow import DAG\n"
diff --git a/tests/unit/test_airflow_templating.py b/tests/unit/test_airflow_templating.py
index fe4cc6a..c05d2c3 100644
--- a/tests/unit/test_airflow_templating.py
+++ b/tests/unit/test_airflow_templating.py
@@ -1,7 +1,8 @@
-"""Unit tests for Airflow Jinja -> DAB reference conversion (flowx.sources.airflow.templating)."""
+"""Unit tests for Airflow Jinja -> DAB reference conversion and cron -> Quartz translation."""
 
 from __future__ import annotations
 
+from flowx.sources.airflow.loader import _cron_to_quartz
 from flowx.sources.airflow.templating import (
     convert_shell_template,
     convert_sql_template,
@@ -71,3 +72,22 @@ def test_macro_param_default_covers_date_and_run_id_and_none():
     assert macro_param_default("run_date", {"kind": "schedule"}) == "{{job.trigger.time.iso_date}}"
     assert macro_param_default("run_id", None) == "{{job.run_id}}"
     assert macro_param_default("env", None) is None  # a user param, not macro-derived
+
+
+def test_quartz_never_restricts_both_day_of_month_and_day_of_week():
+    # Unix cron ORs a restricted dom with a restricted dow; Quartz rejects an expression that sets
+    # both, so one must become '?' or the emitted job fails to validate.
+    assert _cron_to_quartz("0 0 1 * 1") == "0 0 0 ? * 2"
+    assert _cron_to_quartz("0 0 15 * MON") == "0 0 0 ? * MON"
+    # The single-restriction cases keep their field and '?' the other.
+    assert _cron_to_quartz("0 0 1 * *") == "0 0 0 1 * ?"
+    assert _cron_to_quartz("0 6 * * 1") == "0 0 6 ? * 2"
+    assert _cron_to_quartz("0 0 * * *") == "0 0 0 ? * *"
+
+
+def test_quartz_splits_week_wrapping_weekday_ranges():
+    # Unix 5-0 (Fri-Sun) shifts to 6-1, which Quartz reads as a descending (empty) range.
+    assert _cron_to_quartz("0 0 * * 5-0") == "0 0 0 ? * 6-7,1"
+    assert _cron_to_quartz("0 0 * * 6-2") == "0 0 0 ? * 7,1-3"
+    # A non-wrapping range is untouched apart from the +1 shift.
+    assert _cron_to_quartz("0 0 * * 1-5") == "0 0 0 ? * 2-6"

From 8a72cd03f84a70bb38e15e163aa87346c7b0805d Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Wed, 5 Aug 2026 08:07:48 -0700
Subject: [PATCH 48/77] Register operators instantiated without an assignment

Airflow registers a task when the operator is instantiated inside a DAG
context, so an unassigned operator is still a task. Only visit_Assign
collected them, so bare statements (widely used in the Airflow example
DAGs) and bare Op() >> Op() chains were dropped while coverage still
reported 100%. Share the registration path and key bare operators by a
synthetic var from their task_id.

Also carry the .partial()/.expand() mapping call into the gap
raw_definition so fixed argument values reach the agentic round.
---
 src/flowx/sources/airflow/loader.py  | 81 ++++++++++++++++++++-----
 tests/unit/test_airflow_operators.py | 89 +++++++++++++++++++++++++++-
 2 files changed, 155 insertions(+), 15 deletions(-)

diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py
index 2720cae..9db6fa4 100644
--- a/src/flowx/sources/airflow/loader.py
+++ b/src/flowx/sources/airflow/loader.py
@@ -308,6 +308,8 @@ def __init__(self, module: ast.Module) -> None:
         # mapped var -> the kwarg names passed to .expand(). Only these fan out; a list-valued
         # .partial() arg is a fixed value and must not be mistaken for the mapped iterable.
         self.expand_kwargs: dict[str, list[str]] = {}
+        # Disambiguates synthetic vars for operators instantiated without an assignment.
+        self._bare_operator_counter = 0
         # TaskFlow: function name -> (FunctionDef, decorator dotted-name) for @task-decorated defs.
         # Pre-scanned so a @task def defined after the @dag body that uses it is still resolved.
         self.taskflow_defs: dict[str, tuple[ast.FunctionDef, str]] = {}
@@ -352,26 +354,65 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
     def visit_Assign(self, node: ast.Assign) -> None:
         if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name) and isinstance(node.value, ast.Call):
             var = node.targets[0].id
-            direct = _direct_operator_call(node.value)
-            mapped = None if direct is not None else _mapped_operator_call(node.value)
-            call = direct or (mapped[0] if mapped is not None else None)
-            if call is not None and isinstance(call.func, ast.Name):
-                construct = call.func.id
-                kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg}
-                task_id = ops.literal_str(kwargs.get("task_id")) or ops.literal_str(kwargs.get("group_id")) or var
-                self.operators[var] = (task_id, construct, kwargs)
-                self.calls[var] = call
-                if mapped is not None:
-                    self.mapped.add(var)
-                    self.expand_kwargs[var] = mapped[1]
-                if self._group_stack:
-                    self.groups[var] = "__".join(self._group_stack)
+            if self._register_operator_call(node.value, var):
+                pass  # a `x = SomeOperator(...)` (optionally .expand()-mapped) instantiation
             elif self._register_taskflow_call(node.value, var):
                 pass  # a `x = mytask(...)` TaskFlow invocation, captured with var as its key
             else:
                 self._register_taskgroup_call(node.value, var)  # a `x = mygroup(...)` @task_group call
         self.generic_visit(node)
 
+    def _register_operator_call(self, node: ast.Call, var: str) -> bool:
+        """Registers a classic operator/sensor instantiation under the task variable *var*.
+
+        Airflow registers a task when the operator is instantiated inside a DAG context; assigning it
+        to a name is a Python convenience, not a requirement. So this is shared by the assigned form
+        and the bare-statement / bare-chain forms, which synthesise *var* from the task_id.
+
+        Returns True when *node* was a (possibly ``.expand()``-mapped) operator call.
+        """
+        direct = _direct_operator_call(node)
+        mapped = None if direct is not None else _mapped_operator_call(node)
+        call = direct or (mapped[0] if mapped is not None else None)
+        if call is None or not isinstance(call.func, ast.Name):
+            return False
+        construct = call.func.id
+        kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg}
+        task_id = ops.literal_str(kwargs.get("task_id")) or ops.literal_str(kwargs.get("group_id")) or var
+        self.operators[var] = (task_id, construct, kwargs)
+        self.calls[var] = call
+        if mapped is not None:
+            self.mapped.add(var)
+            self.expand_kwargs[var] = mapped[1]
+        if self._group_stack:
+            self.groups[var] = "__".join(self._group_stack)
+        return True
+
+    def _register_bare_operator_call(self, node: ast.Call) -> str | None:
+        """Registers an operator instantiated without an assignment, keyed by a synthetic var.
+
+        The var is derived from the literal ``task_id`` (which is what the emitted task key comes from
+        anyway), with a counter suffix if two bare operators somehow share one.
+        """
+        if _direct_operator_call(node) is None and _mapped_operator_call(node) is None:
+            return None
+        kwargs = {kw.arg: kw.value for kw in node.keywords if kw.arg}
+        base = ops.literal_str(kwargs.get("task_id")) or ops.literal_str(kwargs.get("group_id"))
+        if base is None:
+            # `.expand()` chains carry task_id on the inner .partial(...) call, not the outer one.
+            mapped = _mapped_operator_call(node)
+            if mapped is not None:
+                inner_kwargs = {kw.arg: kw.value for kw in mapped[0].keywords if kw.arg}
+                base = ops.literal_str(inner_kwargs.get("task_id")) or ops.literal_str(inner_kwargs.get("group_id"))
+        if base is None:
+            self._bare_operator_counter += 1
+            base = f"_bare_task{self._bare_operator_counter}"
+        var = base
+        while var in self.operators:
+            self._bare_operator_counter += 1
+            var = f"{base}__{self._bare_operator_counter}"
+        return var if self._register_operator_call(node, var) else None
+
     def _taskflow_def_name(self, call: ast.Call) -> tuple[str | None, bool, str | None]:
         """Resolves a call's underlying ``@task`` def name, unwrapping the mapping/config chain.
 
@@ -609,6 +650,8 @@ def visit_Expr(self, node: ast.Expr) -> None:
                     self._taskflow_counter += 1
                     task_var = f"{def_name}__tf{self._taskflow_counter}"
                 self._register_taskflow_call(value, task_var)
+            elif self._register_bare_operator_call(value) is not None:
+                pass  # a bare `SomeOperator(task_id=...)` statement -- registered under a synthetic var
             elif not self._register_taskgroup_call(value, None):
                 self._collect_set_dependency(value)
         self.generic_visit(node)
@@ -643,6 +686,10 @@ def _shift_position_names(self, node: ast.expr) -> list[str]:
                 synthetic = f"{def_name}__tf{self._taskflow_counter}"
                 self._register_taskflow_call(node, synthetic)
                 return [synthetic]
+            # An inline classic operator (`Op(...) >> Op(...)` with no assignments) is still a task.
+            bare_var = self._register_bare_operator_call(node)
+            if bare_var is not None:
+                return [bare_var]
         return []
 
     def _collect_set_dependency(self, call: ast.Call) -> None:
@@ -1129,6 +1176,11 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
             # emitting a single-run notebook.
             reason = f"mapped parameter {tf.expand_kwarg!r}" if tf.expand_kwarg else "multiple mapped parameters"
             func = functions.get(tf.def_name)
+            # The mapping call carries the .partial(...) fixed args and the mapped iterable, neither of
+            # which appears in the callable's own source -- without it the agentic round can't
+            # reconstruct the invocation.
+            mapping_call = visitor.calls.get(var)
+            mapping_source = ast.get_source_segment(source, mapping_call) if mapping_call is not None else None
             placeholder = PlaceholderActivity(
                 name=tf.task_id,
                 task_key=task_key,
@@ -1141,6 +1193,7 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
                 raw_definition={
                     "operator": f"@{tf.decorator}.expand",
                     "source": ast.get_source_segment(source, func) if func is not None else "",
+                    "mapping": mapping_source or "",
                 },
             )
             placeholder.depends_on = depends_on
diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py
index 8e0e3c1..5e25941 100644
--- a/tests/unit/test_airflow_operators.py
+++ b/tests/unit/test_airflow_operators.py
@@ -15,7 +15,7 @@
     SparkPythonActivity,
     SqlActivity,
 )
-from flowx.sources.airflow.loader import load_airflow_dag
+from flowx.sources.airflow.loader import load_airflow_dag, load_airflow_dags
 
 
 def _load(dag_source: str):
@@ -25,6 +25,14 @@ def _load(dag_source: str):
         return load_airflow_dag(path)
 
 
+def _load_all(dag_source: str):
+    """Loads every DAG declared in one module (the multi-DAG form)."""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = Path(tmp) / "dag.py"
+        path.write_text(dag_source, encoding="utf-8")
+        return load_airflow_dags(path)
+
+
 def _by_key(pipeline):
     return {t.task_key: t for t in pipeline.tasks}
 
@@ -1197,6 +1205,64 @@ def test_trigger_rule_enum_member_maps_to_run_if_constant():
     assert _by_key(p)["cleanup"].depends_on[0].outcome == "ALL_DONE"
 
 
+# --------------------------------------------------------------------------------------
+# Bare (unassigned) operator statements
+# --------------------------------------------------------------------------------------
+
+
+def test_bare_operator_statements_are_registered():
+    # Airflow registers a task when the operator is instantiated inside a DAG context; assigning it to
+    # a name is optional. Unassigned operators must not vanish (the Airflow example DAGs use this form).
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='d') as dag:\n"
+        "    BashOperator(task_id='alpha', bash_command='echo alpha')\n"
+        "    BashOperator(task_id='beta', bash_command='echo beta')\n"
+        "    assigned = BashOperator(task_id='gamma', bash_command='echo gamma')\n"
+    )
+    assert set(_by_key(p)) == {"alpha", "beta", "gamma"}
+
+
+def test_bare_operator_chain_keeps_tasks_and_edge():
+    # `Op() >> Op()` with no assignments: both tasks register and the dependency edge survives.
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='d') as dag:\n"
+        "    BashOperator(task_id='first', bash_command='echo 1')"
+        " >> BashOperator(task_id='second', bash_command='echo 2')\n"
+    )
+    tasks = _by_key(p)
+    assert set(tasks) == {"first", "second"}
+    assert [d.task_key for d in tasks["second"].depends_on] == ["first"]
+
+
+def test_bare_operator_without_literal_task_id_gets_synthetic_key():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='d') as dag:\n"
+        "    BashOperator(bash_command='echo x')\n"
+    )
+    assert list(_by_key(p)) == ["bare_task1"]
+
+
+def test_bare_operators_stay_scoped_to_their_own_dag():
+    # Two DAGs in one module: each keeps only its own bare tasks.
+    p = _load_all(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='one') as dag1:\n"
+        "    BashOperator(task_id='a', bash_command='echo a')\n"
+        "with DAG(dag_id='two') as dag2:\n"
+        "    BashOperator(task_id='b', bash_command='echo b')\n"
+        "    BashOperator(task_id='c', bash_command='echo c')\n"
+    )
+    by_name = {pipeline.name: sorted(t.task_key for t in pipeline.tasks) for pipeline in p}
+    assert by_name == {"one": ["a"], "two": ["b", "c"]}
+
+
 # --------------------------------------------------------------------------------------
 # Dynamic mapping (.expand), TaskGroup prefixing, timezone/timedelta schedules
 # --------------------------------------------------------------------------------------
@@ -1667,6 +1733,27 @@ def test_taskflow_expand_literal_list_becomes_for_each():
     compile(inner.generated_source, "", "exec")
 
 
+def test_taskflow_partial_expand_gap_carries_the_mapping_call():
+    # .partial() fixed args can't ride on a for_each inner task, so the task becomes a placeholder --
+    # but the gap must carry the mapping call, or the fixed argument values are lost and the agentic
+    # round can't reconstruct the invocation (the callable's own source doesn't contain them).
+    p = _load(
+        "from airflow.decorators import dag, task\n"
+        "@task\n"
+        "def get_astronauts():\n    return [{'name': 'A'}]\n"
+        "@task\n"
+        "def greet(greeting, person):\n    print(greeting, person)\n"
+        "@dag(dag_id='f')\n"
+        "def pipeline():\n"
+        "    greet.partial(greeting='Hello! :)').expand(person=get_astronauts())\n"
+        "pipeline()\n"
+    )
+    placeholder = next(t for t in p.tasks if isinstance(t, PlaceholderActivity))
+    mapping = placeholder.raw_definition["mapping"]
+    assert "greeting='Hello! :)'" in mapping
+    assert "expand(person=get_astronauts())" in mapping
+
+
 def test_taskflow_expand_dict_list_becomes_for_each():
     p = _load(
         "from airflow.decorators import dag, task\n"

From bbd14e37bb2ec1423e4ae823ee08e6b076aeab4e Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Fri, 7 Aug 2026 13:42:53 -0700
Subject: [PATCH 49/77] Rebuild Airflow DAG capture around stable identities

---
 pyproject.toml                                |   2 +-
 src/flowx/sources/airflow/loader.py           | 588 ++++++++++++++++--
 .../airflow/review_repros/a1_assigned_dag.py  |  10 +
 .../review_repros/a2_task_key_collision.py    |   8 +
 .../review_repros/a8_classic_mapping.py       |   6 +
 .../airflow/review_repros/t10_loopliteral.py  |   6 +
 .../airflow/review_repros/t11_dagvar.py       |   7 +
 .../airflow/review_repros/t12_globals.py      |   6 +
 .../airflow/review_repros/t13_sqlescape.py    |   4 +
 .../airflow/review_repros/t14_retries.py      |   7 +
 .../airflow/review_repros/t15_magic.py        |  10 +
 .../airflow/review_repros/t16_sensor.py       |   9 +
 .../airflow/review_repros/t17_taskflow.py     |  14 +
 .../airflow/review_repros/t18_xcompush.py     |   9 +
 .../airflow/review_repros/t19_fncollide.py    |  14 +
 .../airflow/review_repros/t1_loop.py          |   9 +
 .../airflow/review_repros/t20_sqlesc.py       |   5 +
 .../review_repros/t21_partialexpand.py        |   4 +
 .../airflow/review_repros/t22_expandbash.py   |   8 +
 .../airflow/review_repros/t23_tr2.py          |  10 +
 .../airflow/review_repros/t24_sensorscope.py  |   8 +
 .../airflow/review_repros/t25_tr3.py          |  10 +
 .../airflow/review_repros/t26_loopedge.py     |   8 +
 .../resources/airflow/review_repros/t27_ss.py |   4 +
 .../airflow/review_repros/t28_nodash.py       |   4 +
 .../airflow/review_repros/t29_dagsem.py       |   5 +
 .../airflow/review_repros/t2_sparksubmit.py   |   7 +
 .../airflow/review_repros/t30_dagvar2.py      |  14 +
 .../airflow/review_repros/t31_inject.py       |   4 +
 .../review_repros/t32_multiassigned.py        |  10 +
 .../airflow/review_repros/t3_collide.py       |   8 +
 .../airflow/review_repros/t4_bashjinja.py     |   4 +
 .../airflow/review_repros/t5_alias.py         |   8 +
 .../airflow/review_repros/t6_chain.py         |  10 +
 .../airflow/review_repros/t7_subclass.py      |   8 +
 .../airflow/review_repros/t8_helperfn.py      |   8 +
 .../airflow/review_repros/t9_triggerrule.py   |  10 +
 tests/unit/test_airflow_operators.py          |   6 +-
 .../unit/test_airflow_production_readiness.py |  83 +++
 39 files changed, 891 insertions(+), 64 deletions(-)
 create mode 100644 tests/resources/airflow/review_repros/a1_assigned_dag.py
 create mode 100644 tests/resources/airflow/review_repros/a2_task_key_collision.py
 create mode 100644 tests/resources/airflow/review_repros/a8_classic_mapping.py
 create mode 100644 tests/resources/airflow/review_repros/t10_loopliteral.py
 create mode 100644 tests/resources/airflow/review_repros/t11_dagvar.py
 create mode 100644 tests/resources/airflow/review_repros/t12_globals.py
 create mode 100644 tests/resources/airflow/review_repros/t13_sqlescape.py
 create mode 100644 tests/resources/airflow/review_repros/t14_retries.py
 create mode 100644 tests/resources/airflow/review_repros/t15_magic.py
 create mode 100644 tests/resources/airflow/review_repros/t16_sensor.py
 create mode 100644 tests/resources/airflow/review_repros/t17_taskflow.py
 create mode 100644 tests/resources/airflow/review_repros/t18_xcompush.py
 create mode 100644 tests/resources/airflow/review_repros/t19_fncollide.py
 create mode 100644 tests/resources/airflow/review_repros/t1_loop.py
 create mode 100644 tests/resources/airflow/review_repros/t20_sqlesc.py
 create mode 100644 tests/resources/airflow/review_repros/t21_partialexpand.py
 create mode 100644 tests/resources/airflow/review_repros/t22_expandbash.py
 create mode 100644 tests/resources/airflow/review_repros/t23_tr2.py
 create mode 100644 tests/resources/airflow/review_repros/t24_sensorscope.py
 create mode 100644 tests/resources/airflow/review_repros/t25_tr3.py
 create mode 100644 tests/resources/airflow/review_repros/t26_loopedge.py
 create mode 100644 tests/resources/airflow/review_repros/t27_ss.py
 create mode 100644 tests/resources/airflow/review_repros/t28_nodash.py
 create mode 100644 tests/resources/airflow/review_repros/t29_dagsem.py
 create mode 100644 tests/resources/airflow/review_repros/t2_sparksubmit.py
 create mode 100644 tests/resources/airflow/review_repros/t30_dagvar2.py
 create mode 100644 tests/resources/airflow/review_repros/t31_inject.py
 create mode 100644 tests/resources/airflow/review_repros/t32_multiassigned.py
 create mode 100644 tests/resources/airflow/review_repros/t3_collide.py
 create mode 100644 tests/resources/airflow/review_repros/t4_bashjinja.py
 create mode 100644 tests/resources/airflow/review_repros/t5_alias.py
 create mode 100644 tests/resources/airflow/review_repros/t6_chain.py
 create mode 100644 tests/resources/airflow/review_repros/t7_subclass.py
 create mode 100644 tests/resources/airflow/review_repros/t8_helperfn.py
 create mode 100644 tests/resources/airflow/review_repros/t9_triggerrule.py
 create mode 100644 tests/unit/test_airflow_production_readiness.py

diff --git a/pyproject.toml b/pyproject.toml
index 9b0c2ec..4e50f1f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -79,7 +79,7 @@ markers = [
 cache-dir = ".venv/ruff-cache"
 target-version = "py312"
 line-length = 120
-exclude = ["templates/*"]
+exclude = ["templates/*", "tests/resources/airflow/review_repros/*"]
 
 [tool.ruff.lint]
 select = ["E", "F", "I"]
diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py
index 9db6fa4..0634491 100644
--- a/src/flowx/sources/airflow/loader.py
+++ b/src/flowx/sources/airflow/loader.py
@@ -21,6 +21,7 @@
 from __future__ import annotations
 
 import ast
+import copy
 import json
 import re
 from dataclasses import dataclass, field
@@ -68,6 +69,57 @@ class _TaskFlowTask:
     expand_items_json: str | None = None
 
 
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SourceSpan:
+    """Stable source location used to identify captured Airflow constructs."""
+
+    line: int
+    column: int
+    end_line: int
+    end_column: int
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class DagDeclaration:
+    """One statically discovered DAG declaration in a Python module."""
+
+    capture_id: str
+    variable: str | None
+    node: ast.stmt
+    span: SourceSpan
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TaskCapture:
+    """One operator or TaskFlow invocation before Databricks key allocation."""
+
+    capture_id: str
+    variable: str
+    task_id: str
+    operator: str
+    call: ast.Call
+    span: SourceSpan
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class EdgeCapture:
+    """A dependency edge expressed in capture identities rather than task keys."""
+
+    upstream_id: str
+    downstream_id: str
+    span: SourceSpan
+
+
+def _span(node: ast.AST) -> SourceSpan:
+    """Returns a complete source span for an AST node."""
+    return SourceSpan(
+        line=getattr(node, "lineno", 0),
+        column=getattr(node, "col_offset", 0),
+        end_line=getattr(node, "end_lineno", getattr(node, "lineno", 0)),
+        end_column=getattr(node, "end_col_offset", getattr(node, "col_offset", 0)),
+    )
+
+
 def _sanitize_task_key(name: str) -> str:
     """Converts an Airflow task_id into a valid Databricks task key."""
     key = re.sub(r"[^a-zA-Z0-9_-]", "_", name)
@@ -276,11 +328,194 @@ def _schedule_from_interval(
     return None
 
 
+_UNRESOLVED = object()
+
+
+def _import_aliases(module: ast.Module) -> dict[str, str]:
+    """Returns local import bindings mapped to their canonical dotted names."""
+    aliases: dict[str, str] = {}
+    for node in module.body:
+        if isinstance(node, ast.Import):
+            for item in node.names:
+                aliases[item.asname or item.name.split(".")[0]] = item.name
+        elif isinstance(node, ast.ImportFrom) and node.module:
+            for item in node.names:
+                if item.name != "*":
+                    aliases[item.asname or item.name] = f"{node.module}.{item.name}"
+    return aliases
+
+
+def _canonical_name(node: ast.expr, aliases: dict[str, str]) -> str:
+    """Resolves an imported name or attribute chain without importing its module."""
+    parts: list[str] = []
+    current: ast.expr = node
+    while isinstance(current, ast.Attribute):
+        parts.append(current.attr)
+        current = current.value
+    if not isinstance(current, ast.Name):
+        return ""
+    root = aliases.get(current.id, current.id)
+    return ".".join([root, *reversed(parts)])
+
+
+def _construct_name(node: ast.expr, aliases: dict[str, str]) -> str:
+    """Returns the canonical class/function leaf name for a call target."""
+    canonical = _canonical_name(node, aliases)
+    return canonical.rsplit(".", 1)[-1] if canonical else ""
+
+
+def _safe_static_value(node: ast.expr, constants: dict[str, Any]) -> Any:
+    """Evaluates the small literal expression subset used by static DAG factories."""
+    if isinstance(node, ast.Constant):
+        return node.value
+    if isinstance(node, ast.Name):
+        return constants.get(node.id, _UNRESOLVED)
+    if isinstance(node, (ast.List, ast.Tuple, ast.Set)):
+        values = [_safe_static_value(item, constants) for item in node.elts]
+        if any(value is _UNRESOLVED for value in values):
+            return _UNRESOLVED
+        if isinstance(node, ast.Tuple):
+            return tuple(values)
+        if isinstance(node, ast.Set):
+            return set(values)
+        return values
+    if isinstance(node, ast.Dict):
+        keys = [_safe_static_value(item, constants) for item in node.keys if item is not None]
+        values = [_safe_static_value(item, constants) for item in node.values]
+        if len(keys) != len(node.values) or any(value is _UNRESOLVED for value in [*keys, *values]):
+            return _UNRESOLVED
+        return dict(zip(keys, values))
+    if isinstance(node, ast.JoinedStr):
+        parts: list[str] = []
+        for item in node.values:
+            if isinstance(item, ast.Constant) and isinstance(item.value, str):
+                parts.append(item.value)
+                continue
+            if isinstance(item, ast.FormattedValue):
+                value = _safe_static_value(item.value, constants)
+                if value is not _UNRESOLVED:
+                    parts.append(str(value))
+                    continue
+            return _UNRESOLVED
+        return "".join(parts)
+    if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
+        left = _safe_static_value(node.left, constants)
+        right = _safe_static_value(node.right, constants)
+        if left is _UNRESOLVED or right is _UNRESOLVED:
+            return _UNRESOLVED
+        try:
+            return left + right
+        except TypeError:
+            return _UNRESOLVED
+    if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)):
+        value = _safe_static_value(node.operand, constants)
+        if value is _UNRESOLVED or not isinstance(value, (int, float)):
+            return _UNRESOLVED
+        return -value if isinstance(node.op, ast.USub) else value
+    return _UNRESOLVED
+
+
+def _value_node(value: Any) -> ast.expr:
+    """Builds an expression node for a statically evaluated Python value."""
+    return ast.parse(repr(value), mode="eval").body
+
+
+class _ConstantSubstituter(ast.NodeTransformer):
+    """Replaces known constant names and folds the supported literal subset."""
+
+    def __init__(self, constants: dict[str, Any]) -> None:
+        self.constants = constants
+
+    def visit_Name(self, node: ast.Name) -> ast.expr:
+        value = self.constants.get(node.id, _UNRESOLVED)
+        return ast.copy_location(_value_node(value), node) if value is not _UNRESOLVED else node
+
+    def generic_visit(self, node: ast.AST) -> ast.AST:
+        visited = super().generic_visit(node)
+        if isinstance(visited, ast.expr):
+            value = _safe_static_value(visited, {})
+            if value is not _UNRESOLVED:
+                return ast.copy_location(_value_node(value), visited)
+        return visited
+
+
+def _bind_constants(node: ast.AST, constants: dict[str, Any]) -> Any:
+    """Returns a deep-copied AST with known constant names substituted and folded."""
+    bound = _ConstantSubstituter(constants).visit(copy.deepcopy(node))
+    ast.fix_missing_locations(bound)
+    if isinstance(bound, ast.expr):
+        value = _safe_static_value(bound, {})
+        if value is not _UNRESOLVED:
+            return ast.copy_location(_value_node(value), bound)
+    return bound
+
+
+def _static_iteration_nodes(node: ast.expr, constants: dict[str, Any]) -> list[ast.expr] | None:
+    """Returns bounded literal/range loop values, or None for a dynamic iterable."""
+    if isinstance(node, ast.Call) and _construct_name(node.func, {}) == "range":
+        values = [_safe_static_value(argument, constants) for argument in node.args]
+        if any(value is _UNRESOLVED or not isinstance(value, int) for value in values):
+            return None
+        try:
+            result = list(range(*values))
+        except (TypeError, ValueError):
+            return None
+        return [_value_node(value) for value in result] if len(result) <= 256 else None
+    if isinstance(node, (ast.List, ast.Tuple)):
+        return [copy.deepcopy(item) for item in node.elts] if len(node.elts) <= 256 else None
+    value = _safe_static_value(node, constants)
+    if isinstance(value, (list, tuple)) and len(value) <= 256:
+        return [_value_node(item) for item in value]
+    return None
+
+
+def _expand_top_level_loops(module: ast.Module) -> ast.Module:
+    """Unrolls bounded module-level loops so generated DAG declarations stay distinct."""
+    body: list[ast.stmt] = []
+    constants: dict[str, Any] = {}
+    for statement in module.body:
+        if (
+            isinstance(statement, ast.Assign)
+            and len(statement.targets) == 1
+            and isinstance(statement.targets[0], ast.Name)
+        ):
+            value = _safe_static_value(statement.value, constants)
+            if value is not _UNRESOLVED:
+                constants[statement.targets[0].id] = value
+        if isinstance(statement, ast.For) and isinstance(statement.target, ast.Name):
+            items = _static_iteration_nodes(statement.iter, constants)
+            if items is not None:
+                for item in items:
+                    value = _safe_static_value(item, constants)
+                    if value is _UNRESOLVED:
+                        continue
+                    iteration_constants = {**constants, statement.target.id: value}
+                    body.extend(_bind_constants(child, iteration_constants) for child in statement.body)
+                continue
+        body.append(statement)
+    expanded = ast.Module(body=body, type_ignores=list(module.type_ignores))
+    ast.fix_missing_locations(expanded)
+    return expanded
+
+
 class _DagVisitor(ast.NodeVisitor):
     """Collects operator calls, dependency edges, and the DAG's schedule."""
 
-    def __init__(self, module: ast.Module) -> None:
-        self._functions: dict[str, ast.FunctionDef] = {node.name: node for node in _iter_functions(module)}
+    def __init__(self, module: ast.Module, *, target_dag_variable: str | None = None) -> None:
+        self._aliases = _import_aliases(module)
+        self._target_dag_variable = target_dag_variable
+        # Classic python_callable resolution starts at module scope. Nested functions are only visible
+        # from their lexical parent and must never overwrite a same-named module function.
+        self._functions: dict[str, ast.FunctionDef] = {
+            node.name: node for node in module.body if isinstance(node, ast.FunctionDef)
+        }
+        self._constants: dict[str, Any] = {}
+        self._task_bindings: dict[str, str | list[str]] = {}
+        self._list_bindings: dict[str, list[str]] = {}
+        self._capture_sequence = 0
+        self.task_captures: dict[str, TaskCapture] = {}
+        self.edge_captures: list[EdgeCapture] = []
+        self.unresolved_constructs: list[tuple[str, ast.AST]] = []
         # task variable name -> (task_id, operator, kwargs)
         self.operators: dict[str, tuple[str, str, dict[str, ast.expr]]] = {}
         # task variable name -> the operator's ast.Call node (for source-slicing placeholders)
@@ -334,7 +569,8 @@ def __init__(self, module: ast.Module) -> None:
         self.is_taskflow_dag: bool = False
 
     def functions(self) -> dict[str, ast.FunctionDef]:
-        return self._functions
+        taskflow = {name: definition for name, (definition, _decorator) in self.taskflow_defs.items()}
+        return {**self._functions, **taskflow}
 
     def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
         # A @task- or @task_group-decorated function defines a task / sub-pipeline from its body,
@@ -354,15 +590,47 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
     def visit_Assign(self, node: ast.Assign) -> None:
         if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name) and isinstance(node.value, ast.Call):
             var = node.targets[0].id
-            if self._register_operator_call(node.value, var):
+            if _construct_name(node.value.func, self._aliases) == "DAG":
+                self._read_dag_kwargs(node.value)
+                self.dag_id = self.dag_id or var
+                return
+            internal_var = self._new_task_var(var, node.value)
+            if self._register_operator_call(node.value, internal_var, binding=var):
                 pass  # a `x = SomeOperator(...)` (optionally .expand()-mapped) instantiation
-            elif self._register_taskflow_call(node.value, var):
+            elif self._register_helper_factory_call(node.value, internal_var, binding=var):
+                pass
+            elif self._register_taskflow_call(node.value, internal_var):
+                self._task_bindings[var] = internal_var
                 pass  # a `x = mytask(...)` TaskFlow invocation, captured with var as its key
             else:
-                self._register_taskgroup_call(node.value, var)  # a `x = mygroup(...)` @task_group call
+                if self._register_taskgroup_call(node.value, internal_var):
+                    self._task_bindings[var] = internal_var
+                    return
+        elif len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+            target = node.targets[0].id
+            if isinstance(node.value, ast.Name):
+                resolved = self._resolve_task_names(node.value)
+                if resolved:
+                    self._task_bindings[target] = resolved[0] if len(resolved) == 1 else resolved
+                    self._constants.pop(target, None)
+                    return
+            value = _safe_static_value(node.value, self._constants)
+            if value is not _UNRESOLVED:
+                self._constants[target] = value
+                self._task_bindings.pop(target, None)
+                if isinstance(value, list):
+                    self._list_bindings[target] = []
+                return
         self.generic_visit(node)
 
-    def _register_operator_call(self, node: ast.Call, var: str) -> bool:
+    def _new_task_var(self, binding: str, node: ast.AST) -> str:
+        """Allocates an internal identity while preserving Python's latest name binding."""
+        if binding not in self.operators and binding not in self.taskflow_tasks and binding not in self.taskgroup_calls:
+            return binding
+        self._capture_sequence += 1
+        return f"{binding}__L{getattr(node, 'lineno', 0)}_{self._capture_sequence}"
+
+    def _register_operator_call(self, node: ast.Call, var: str, *, binding: str | None = None) -> bool:
         """Registers a classic operator/sensor instantiation under the task variable *var*.
 
         Airflow registers a task when the operator is instantiated inside a DAG context; assigning it
@@ -371,16 +639,34 @@ def _register_operator_call(self, node: ast.Call, var: str) -> bool:
 
         Returns True when *node* was a (possibly ``.expand()``-mapped) operator call.
         """
-        direct = _direct_operator_call(node)
-        mapped = None if direct is not None else _mapped_operator_call(node)
+        direct = _direct_operator_call(node, self._aliases)
+        mapped = None if direct is not None else _mapped_operator_call(node, self._aliases)
         call = direct or (mapped[0] if mapped is not None else None)
-        if call is None or not isinstance(call.func, ast.Name):
+        if call is None:
             return False
-        construct = call.func.id
-        kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg}
+        construct = _construct_name(call.func, self._aliases)
+        kwargs = {kw.arg: _bind_constants(kw.value, self._constants) for kw in call.keywords if kw.arg}
+        dag_node = kwargs.get("dag")
+        if self._target_dag_variable is not None and not (
+            isinstance(dag_node, ast.Name) and dag_node.id == self._target_dag_variable
+        ):
+            return False
+        call = ast.Call(func=ast.Name(id=construct, ctx=ast.Load()), args=[], keywords=[
+            ast.keyword(arg=key, value=value) for key, value in kwargs.items()
+        ])
+        ast.copy_location(call, node)
         task_id = ops.literal_str(kwargs.get("task_id")) or ops.literal_str(kwargs.get("group_id")) or var
         self.operators[var] = (task_id, construct, kwargs)
         self.calls[var] = call
+        self._task_bindings[binding or var] = var
+        self.task_captures[var] = TaskCapture(
+            capture_id=var,
+            variable=binding or var,
+            task_id=task_id,
+            operator=construct,
+            call=call,
+            span=_span(node),
+        )
         if mapped is not None:
             self.mapped.add(var)
             self.expand_kwargs[var] = mapped[1]
@@ -388,19 +674,69 @@ def _register_operator_call(self, node: ast.Call, var: str) -> bool:
             self.groups[var] = "__".join(self._group_stack)
         return True
 
+    def _register_helper_factory_call(self, node: ast.Call, var: str, *, binding: str) -> bool:
+        """Expands the deliberately narrow single-return operator factory shape."""
+        if not isinstance(node.func, ast.Name):
+            return False
+        helper = self._functions.get(node.func.id)
+        if helper is None or helper.decorator_list or helper.args.vararg or helper.args.kwarg:
+            return False
+        body = list(helper.body)
+        if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant):
+            if isinstance(body[0].value.value, str):
+                body = body[1:]
+        if len(body) != 1 or not isinstance(body[0], ast.Return) or not isinstance(body[0].value, ast.Call):
+            return False
+        parameters = [*helper.args.posonlyargs, *helper.args.args, *helper.args.kwonlyargs]
+        if len(node.args) > len(parameters) or any(keyword.arg is None for keyword in node.keywords):
+            return False
+        bound: dict[str, Any] = {}
+        for parameter, argument in zip(parameters, node.args):
+            bound[parameter.arg] = _bind_constants(argument, self._constants)
+        for keyword in node.keywords:
+            if keyword.arg:
+                bound[keyword.arg] = _bind_constants(keyword.value, self._constants)
+        missing = [parameter.arg for parameter in parameters if parameter.arg not in bound]
+        positional_defaults = [None] * (len(helper.args.args) - len(helper.args.defaults)) + list(helper.args.defaults)
+        defaults = {
+            parameter.arg: default
+            for parameter, default in zip(helper.args.args, positional_defaults)
+            if default is not None
+        }
+        defaults.update(
+            {
+                parameter.arg: default
+                for parameter, default in zip(helper.args.kwonlyargs, helper.args.kw_defaults)
+                if default is not None
+            }
+        )
+        for name in missing:
+            if name not in defaults:
+                return False
+            bound[name] = defaults[name]
+        constants = dict(self._constants)
+        for name, expression in bound.items():
+            if isinstance(expression, ast.expr):
+                value = _safe_static_value(expression, constants)
+                if value is _UNRESOLVED:
+                    return False
+                constants[name] = value
+        factory_call = _bind_constants(body[0].value, constants)
+        return isinstance(factory_call, ast.Call) and self._register_operator_call(factory_call, var, binding=binding)
+
     def _register_bare_operator_call(self, node: ast.Call) -> str | None:
         """Registers an operator instantiated without an assignment, keyed by a synthetic var.
 
         The var is derived from the literal ``task_id`` (which is what the emitted task key comes from
         anyway), with a counter suffix if two bare operators somehow share one.
         """
-        if _direct_operator_call(node) is None and _mapped_operator_call(node) is None:
+        if _direct_operator_call(node, self._aliases) is None and _mapped_operator_call(node, self._aliases) is None:
             return None
         kwargs = {kw.arg: kw.value for kw in node.keywords if kw.arg}
         base = ops.literal_str(kwargs.get("task_id")) or ops.literal_str(kwargs.get("group_id"))
         if base is None:
             # `.expand()` chains carry task_id on the inner .partial(...) call, not the outer one.
-            mapped = _mapped_operator_call(node)
+            mapped = _mapped_operator_call(node, self._aliases)
             if mapped is not None:
                 inner_kwargs = {kw.arg: kw.value for kw in mapped[0].keywords if kw.arg}
                 base = ops.literal_str(inner_kwargs.get("task_id")) or ops.literal_str(inner_kwargs.get("group_id"))
@@ -411,7 +747,7 @@ def _register_bare_operator_call(self, node: ast.Call) -> str | None:
         while var in self.operators:
             self._bare_operator_counter += 1
             var = f"{base}__{self._bare_operator_counter}"
-        return var if self._register_operator_call(node, var) else None
+        return var if self._register_operator_call(node, var, binding=var) else None
 
     def _taskflow_def_name(self, call: ast.Call) -> tuple[str | None, bool, str | None]:
         """Resolves a call's underlying ``@task`` def name, unwrapping the mapping/config chain.
@@ -562,8 +898,10 @@ def _resolve_taskflow_arg(self, arg: ast.expr) -> str | None:
         is registered as its own synthetic task instance and its var returned, so the whole
         expression tree becomes a chain of task instances.
         """
-        if isinstance(arg, ast.Name) and (arg.id in self.operators or arg.id in self.taskflow_tasks):
-            return arg.id
+        if isinstance(arg, ast.Name):
+            resolved = self._resolve_task_names(arg)
+            if len(resolved) == 1:
+                return resolved[0]
         if isinstance(arg, ast.Call):
             def_name, _mapped, _override = self._taskflow_def_name(arg)
             if def_name is not None:
@@ -577,10 +915,11 @@ def visit_With(self, node: ast.With) -> None:
         pushed_group = False
         for item in node.items:
             call = item.context_expr
-            if isinstance(call, ast.Call) and isinstance(call.func, ast.Name):
-                if call.func.id == "DAG":
+            if isinstance(call, ast.Call):
+                construct = _construct_name(call.func, self._aliases)
+                if construct == "DAG":
                     self._read_dag_kwargs(call)
-                elif call.func.id == "TaskGroup":
+                elif construct == "TaskGroup":
                     # `with TaskGroup("etl") as tg:` — namespace the member tasks by group id.
                     kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg}
                     group_id = (
@@ -594,20 +933,20 @@ def visit_With(self, node: ast.With) -> None:
                     # edge on `tg` resolves to the group's member tasks.
                     if isinstance(item.optional_vars, ast.Name):
                         self.group_vars[item.optional_vars.id] = "__".join(self._group_stack)
-                elif _is_task_construct(call.func.id) and item.optional_vars is not None:
+                elif _is_task_construct(construct) and item.optional_vars is not None:
                     # `with DbtTaskGroup(...) as g:` — a cosmos group bound to a name.
                     if isinstance(item.optional_vars, ast.Name):
                         var = item.optional_vars.id
                         kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg}
                         task_id = ops.literal_str(kwargs.get("group_id")) or var
-                        self.operators[var] = (task_id, call.func.id, kwargs)
+                        self.operators[var] = (task_id, construct, kwargs)
                         self.calls[var] = call
         self.generic_visit(node)
         if pushed_group:
             self._group_stack.pop()
 
     def _read_dag_kwargs(self, call: ast.Call) -> None:
-        kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg}
+        kwargs = {kw.arg: _bind_constants(kw.value, self._constants) for kw in call.keywords if kw.arg}
         self.dag_id = ops.literal_str(kwargs.get("dag_id"))
         self._apply_dag_kwargs(kwargs)
 
@@ -641,6 +980,30 @@ def visit_Expr(self, node: ast.Expr) -> None:
         if isinstance(value, ast.BinOp) and isinstance(value.op, (ast.RShift, ast.LShift)):
             self._collect_shift_chain(value)
         elif isinstance(value, ast.Call):
+            call_name = _construct_name(value.func, self._aliases)
+            if call_name == "chain":
+                positions = [self._resolve_task_names(argument) for argument in value.args]
+                for left, right in zip(positions, positions[1:]):
+                    self._add_edges(left, right, value)
+                return
+            if call_name == "cross_downstream" and len(value.args) >= 2:
+                self._add_edges(
+                    self._resolve_task_names(value.args[0]),
+                    self._resolve_task_names(value.args[1]),
+                    value,
+                )
+                return
+            if isinstance(value.func, ast.Attribute) and value.func.attr == "append" and value.args:
+                owner = value.func.value
+                if isinstance(owner, ast.Name):
+                    appended = value.args[0]
+                    if isinstance(appended, ast.Call):
+                        internal = self._register_bare_operator_call(appended)
+                        if internal is not None:
+                            self._list_bindings.setdefault(owner.id, []).append(internal)
+                            return
+                    self._list_bindings.setdefault(owner.id, []).extend(self._resolve_task_names(appended))
+                    return
             # A bare TaskFlow call (`extract()` with no assignment) is a task instance keyed by its
             # def name; otherwise it may be a set_upstream/set_downstream dependency call.
             def_name, _mapped, _override = self._taskflow_def_name(value)
@@ -656,6 +1019,49 @@ def visit_Expr(self, node: ast.Expr) -> None:
                 self._collect_set_dependency(value)
         self.generic_visit(node)
 
+    def visit_For(self, node: ast.For) -> None:
+        """Executes bounded literal/range loops with Python name rebinding semantics."""
+        if not isinstance(node.target, ast.Name):
+            self.unresolved_constructs.append(("dynamic_loop_target", node))
+            return
+        items = _static_iteration_nodes(node.iter, self._constants)
+        if items is None:
+            # A tuple/list of task variables is also statically bounded even though the values are
+            # capture identities rather than Python literals.
+            if isinstance(node.iter, (ast.List, ast.Tuple)):
+                items = list(node.iter.elts)
+            else:
+                self.unresolved_constructs.append(("dynamic_loop_iterable", node))
+                return
+        for item in items:
+            resolved_tasks = self._resolve_task_names(item)
+            if resolved_tasks:
+                self._task_bindings[node.target.id] = resolved_tasks[0] if len(resolved_tasks) == 1 else resolved_tasks
+                self._constants.pop(node.target.id, None)
+            else:
+                value = _safe_static_value(item, self._constants)
+                if value is _UNRESOLVED:
+                    self.unresolved_constructs.append(("dynamic_loop_value", item))
+                    return
+                self._constants[node.target.id] = value
+                self._task_bindings.pop(node.target.id, None)
+            for statement in node.body:
+                self.visit(statement)
+        for statement in node.orelse:
+            self.visit(statement)
+
+    def visit_If(self, node: ast.If) -> None:
+        """Follows a statically decidable branch; records ambiguous control flow explicitly."""
+        value = _safe_static_value(node.test, self._constants)
+        if value is _UNRESOLVED and isinstance(node.test, ast.Name) and node.test.id in self._task_bindings:
+            value = True
+        if value is _UNRESOLVED:
+            self.unresolved_constructs.append(("ambiguous_condition", node))
+            return
+        branch = node.body if bool(value) else node.orelse
+        for statement in branch:
+            self.visit(statement)
+
     def _collect_shift_chain(self, binop: ast.BinOp) -> None:
         self._collect_shift_expression(binop)
 
@@ -666,19 +1072,14 @@ def _collect_shift_expression(self, node: ast.expr) -> list[str]:
         left = self._collect_shift_expression(node.left)
         right = self._collect_shift_expression(node.right)
         upstream, downstream = (left, right) if isinstance(node.op, ast.RShift) else (right, left)
-        self.edges.extend((upstream_var, downstream_var) for upstream_var in upstream for downstream_var in downstream)
+        self._add_edges(upstream, downstream, node)
         return right
 
     def _shift_position_names(self, node: ast.expr) -> list[str]:
         # A shift-chain position resolves to task vars. An inline TaskFlow call (`extract()`) is
         # registered as its own instance so `prep >> finalize()` doesn't drop finalize.
-        if isinstance(node, (ast.List, ast.Tuple)):
-            names: list[str] = []
-            for elt in node.elts:
-                names.extend(self._shift_position_names(elt))
-            return names
-        if isinstance(node, ast.Name):
-            return [node.id]
+        if isinstance(node, (ast.List, ast.Tuple, ast.Name)):
+            return self._resolve_task_names(node)
         if isinstance(node, ast.Call):
             def_name, _mapped, _override = self._taskflow_def_name(node)
             if def_name is not None:
@@ -692,17 +1093,44 @@ def _shift_position_names(self, node: ast.expr) -> list[str]:
                 return [bare_var]
         return []
 
+    def _resolve_task_names(self, node: ast.expr) -> list[str]:
+        """Resolves current Python bindings to stable task capture identities."""
+        if isinstance(node, ast.Name):
+            binding = self._task_bindings.get(node.id)
+            if isinstance(binding, str):
+                return [binding]
+            if isinstance(binding, list):
+                return list(binding)
+            if node.id in self._list_bindings:
+                return list(self._list_bindings[node.id])
+            if node.id in self.group_vars:
+                return [node.id]
+            if node.id in self.operators or node.id in self.taskflow_tasks or node.id in self.taskgroup_calls:
+                return [node.id]
+            return []
+        if isinstance(node, (ast.List, ast.Tuple)):
+            return [task for item in node.elts for task in self._resolve_task_names(item)]
+        return []
+
+    def _add_edges(self, upstreams: list[str], downstreams: list[str], node: ast.AST) -> None:
+        for upstream_var in upstreams:
+            for downstream_var in downstreams:
+                self.edges.append((upstream_var, downstream_var))
+                self.edge_captures.append(
+                    EdgeCapture(upstream_id=upstream_var, downstream_id=downstream_var, span=_span(node))
+                )
+
     def _collect_set_dependency(self, call: ast.Call) -> None:
         # `x.set_upstream(y)` / `x.set_downstream(y)` where y is a Name or a list of Names.
         func = call.func
         if not (isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name) and call.args):
             return
-        this = func.value.id
-        others = _names_in(call.args[0])
+        this_names = self._resolve_task_names(func.value)
+        others = self._resolve_task_names(call.args[0])
         if func.attr == "set_downstream":
-            self.edges.extend((this, other) for other in others)
+            self._add_edges(this_names, others, call)
         elif func.attr == "set_upstream":
-            self.edges.extend((other, this) for other in others)
+            self._add_edges(others, this_names, call)
 
 
 def _expand_group_edges(
@@ -820,14 +1248,16 @@ def _literal_argument_source(node: ast.expr) -> str | None:
         return None
 
 
-def _direct_operator_call(node: ast.Call) -> ast.Call | None:
+def _direct_operator_call(node: ast.Call, aliases: dict[str, str] | None = None) -> ast.Call | None:
     """Returns *node* if it is a direct ``SomeOperator(...)`` / ``SomeSensor(...)`` call."""
-    if isinstance(node.func, ast.Name) and _is_task_construct(node.func.id):
+    if _is_task_construct(_construct_name(node.func, aliases or {})):
         return node
     return None
 
 
-def _mapped_operator_call(node: ast.Call) -> tuple[ast.Call, list[str]] | None:
+def _mapped_operator_call(
+    node: ast.Call, aliases: dict[str, str] | None = None
+) -> tuple[ast.Call, list[str]] | None:
     """Returns the underlying operator call for a dynamic-mapping ``.expand(...)`` chain.
 
     Handles ``Op(...).expand(...)`` and ``Op.partial(...).expand(...)``. Returns
@@ -841,15 +1271,15 @@ def _mapped_operator_call(node: ast.Call) -> tuple[ast.Call, list[str]] | None:
     inner = node.func.value  # the Op(...) or Op.partial(...) call
     if not isinstance(inner, ast.Call):
         return None
-    if isinstance(inner.func, ast.Name) and _is_task_construct(inner.func.id):
-        operator_name = inner.func.id  # Op(...).expand(...)
+    alias_map = aliases or {}
+    if _is_task_construct(_construct_name(inner.func, alias_map)):
+        operator_name = _construct_name(inner.func, alias_map)  # Op(...).expand(...)
     elif (
         isinstance(inner.func, ast.Attribute)
         and inner.func.attr == "partial"
-        and isinstance(inner.func.value, ast.Name)
-        and _is_task_construct(inner.func.value.id)
+        and _is_task_construct(_construct_name(inner.func.value, alias_map))
     ):
-        operator_name = inner.func.value.id  # Op.partial(...).expand(...)
+        operator_name = _construct_name(inner.func.value, alias_map)  # Op.partial(...).expand(...)
     else:
         return None
     merged = ast.Call(
@@ -915,36 +1345,63 @@ def load_airflow_dag(dag_path: Path, *, dbt_mode: str = "static") -> Pipeline:
 def load_airflow_dags(dag_path: Path, *, dbt_mode: str = "static") -> list[Pipeline]:
     """Parses every independently declared Airflow DAG in a Python file."""
     source = Path(dag_path).read_text(encoding="utf-8")
-    module = ast.parse(source)
-    dag_nodes = _top_level_dag_nodes(module)
-    if not dag_nodes:
+    module = _expand_top_level_loops(ast.parse(source))
+    declarations = _top_level_dag_declarations(module)
+    if not declarations:
         return [_load_airflow_module(dag_path, source, module, dbt_mode=dbt_mode)]
     return [
-        _load_airflow_module(dag_path, source, _module_for_dag(module, dag_node), dbt_mode=dbt_mode)
-        for dag_node in dag_nodes
+        _load_airflow_module(
+            dag_path,
+            source,
+            _module_for_dag(module, declaration),
+            dbt_mode=dbt_mode,
+            target_dag_variable=declaration.variable,
+        )
+        for declaration in declarations
     ]
 
 
-def _top_level_dag_nodes(module: ast.Module) -> list[ast.stmt]:
-    """Returns top-level context-manager and decorated-function DAG declarations."""
-    declarations: list[ast.stmt] = []
+def _top_level_dag_declarations(module: ast.Module) -> list[DagDeclaration]:
+    """Returns context-manager, decorated, and assigned top-level DAG declarations."""
+    aliases = _import_aliases(module)
+    declarations: list[DagDeclaration] = []
     for node in module.body:
+        variable: str | None = None
+        is_dag = False
         if isinstance(node, ast.FunctionDef) and _has_decorator(node, _DAG_DECORATORS):
-            declarations.append(node)
+            is_dag = True
         elif isinstance(node, ast.With) and any(
             isinstance(item.context_expr, ast.Call)
-            and isinstance(item.context_expr.func, ast.Name)
-            and item.context_expr.func.id == "DAG"
+            and _construct_name(item.context_expr.func, aliases) == "DAG"
             for item in node.items
         ):
-            declarations.append(node)
+            is_dag = True
+        elif (
+            isinstance(node, ast.Assign)
+            and len(node.targets) == 1
+            and isinstance(node.targets[0], ast.Name)
+            and isinstance(node.value, ast.Call)
+            and _construct_name(node.value.func, aliases) == "DAG"
+        ):
+            is_dag = True
+            variable = node.targets[0].id
+        if is_dag:
+            span = _span(node)
+            declarations.append(
+                DagDeclaration(
+                    capture_id=f"dag:{span.line}:{span.column}:{len(declarations) + 1}",
+                    variable=variable,
+                    node=node,
+                    span=span,
+                )
+            )
     return declarations
 
 
-def _module_for_dag(module: ast.Module, dag_node: ast.stmt) -> ast.Module:
+def _module_for_dag(module: ast.Module, declaration: DagDeclaration) -> ast.Module:
     """Returns a module containing shared definitions and one DAG declaration."""
-    dag_nodes = set(_top_level_dag_nodes(module))
-    body = [node for node in module.body if node is dag_node or node not in dag_nodes]
+    dag_nodes = {item.node for item in _top_level_dag_declarations(module)}
+    body = [node for node in module.body if node is declaration.node or node not in dag_nodes]
     return ast.Module(body=body, type_ignores=list(module.type_ignores))
 
 
@@ -954,6 +1411,7 @@ def _load_airflow_module(
     module: ast.Module,
     *,
     dbt_mode: str = "static",
+    target_dag_variable: str | None = None,
 ) -> Pipeline:
     """Parses one isolated DAG declaration into a flowx Pipeline IR.
 
@@ -970,7 +1428,7 @@ def _load_airflow_module(
         sensors remain explicit placeholders; unmapped operators become a
         PlaceholderActivity.
     """
-    visitor = _DagVisitor(module)
+    visitor = _DagVisitor(module, target_dag_variable=target_dag_variable)
     visitor.visit(module)
     functions = visitor.functions()
 
@@ -985,7 +1443,17 @@ def _task_key(var: str, task_id: str) -> str:
     var_task_ids: dict[str, str] = {var: tid for var, (tid, _, _) in visitor.operators.items()}
     var_task_ids.update({var: tf.task_id for var, tf in visitor.taskflow_tasks.items()})
     var_task_ids.update({var: task_id for var, (task_id, _, _) in visitor.taskgroup_calls.items()})
-    var_to_task_key = {var: _task_key(var, tid) for var, tid in var_task_ids.items()}
+    var_to_task_key: dict[str, str] = {}
+    used_task_keys: set[str] = set()
+    for var, task_id in var_task_ids.items():
+        base = _task_key(var, task_id)
+        candidate = base
+        suffix = 2
+        while candidate in used_task_keys:
+            candidate = f"{base}__{suffix}"
+            suffix += 1
+        used_task_keys.add(candidate)
+        var_to_task_key[var] = candidate
 
     # Expand group-level edges (`group_a >> group_b`, `task >> group`, ...) into edges between the
     # groups' boundary tasks: leaves of the upstream group -> roots of the downstream group, matching
diff --git a/tests/resources/airflow/review_repros/a1_assigned_dag.py b/tests/resources/airflow/review_repros/a1_assigned_dag.py
new file mode 100644
index 0000000..c8140be
--- /dev/null
+++ b/tests/resources/airflow/review_repros/a1_assigned_dag.py
@@ -0,0 +1,10 @@
+from datetime import timedelta
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+
+dag = DAG(dag_id="legacy_etl", schedule_interval="0 3 * * *", catchup=True,
+          default_args={"retries": 5, "execution_timeout": timedelta(hours=2)},
+          params={"env": "prod"})
+a = BashOperator(task_id="extract", bash_command="run.sh --d {{ ds }}", dag=dag)
+b = BashOperator(task_id="load", bash_command="load.sh", dag=dag, trigger_rule="all_done")
+a >> b
diff --git a/tests/resources/airflow/review_repros/a2_task_key_collision.py b/tests/resources/airflow/review_repros/a2_task_key_collision.py
new file mode 100644
index 0000000..74121c6
--- /dev/null
+++ b/tests/resources/airflow/review_repros/a2_task_key_collision.py
@@ -0,0 +1,8 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="collide", schedule="@daily") as dag:
+    x = BashOperator(task_id="load.data", bash_command="echo 1")
+    y = BashOperator(task_id="load_data", bash_command="echo 2")
+    z = BashOperator(task_id="final", bash_command="echo 3")
+    x >> z
+    y >> z
diff --git a/tests/resources/airflow/review_repros/a8_classic_mapping.py b/tests/resources/airflow/review_repros/a8_classic_mapping.py
new file mode 100644
index 0000000..98eff41
--- /dev/null
+++ b/tests/resources/airflow/review_repros/a8_classic_mapping.py
@@ -0,0 +1,6 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="fan", schedule="@daily") as dag:
+    BashOperator.partial(task_id="fan", bash_command="echo static").expand(
+        env=[{"A": "1"}, {"A": "2"}]
+    )
diff --git a/tests/resources/airflow/review_repros/t10_loopliteral.py b/tests/resources/airflow/review_repros/t10_loopliteral.py
new file mode 100644
index 0000000..452fed4
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t10_loopliteral.py
@@ -0,0 +1,6 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="loop2", schedule_interval="0 6 * * *") as dag:
+    tasks = []
+    for r in ["us", "eu"]:
+        tasks.append(BashOperator(task_id="load_" + r, bash_command="echo x"))
diff --git a/tests/resources/airflow/review_repros/t11_dagvar.py b/tests/resources/airflow/review_repros/t11_dagvar.py
new file mode 100644
index 0000000..6988d38
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t11_dagvar.py
@@ -0,0 +1,7 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+from datetime import datetime
+dag = DAG(dag_id="assigned_dag", schedule_interval="0 3 * * *", start_date=datetime(2024,1,1))
+a = BashOperator(task_id="a", bash_command="echo a", dag=dag)
+b = BashOperator(task_id="b", bash_command="echo b", dag=dag)
+a >> b
diff --git a/tests/resources/airflow/review_repros/t12_globals.py b/tests/resources/airflow/review_repros/t12_globals.py
new file mode 100644
index 0000000..c1552b4
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t12_globals.py
@@ -0,0 +1,6 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+for team in ["alpha", "beta"]:
+    with DAG(dag_id=f"etl_{team}", schedule_interval="0 6 * * *") as d:
+        BashOperator(task_id="run", bash_command="echo x")
+    globals()[f"etl_{team}"] = d
diff --git a/tests/resources/airflow/review_repros/t13_sqlescape.py b/tests/resources/airflow/review_repros/t13_sqlescape.py
new file mode 100644
index 0000000..28526a5
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t13_sqlescape.py
@@ -0,0 +1,4 @@
+from airflow import DAG
+from airflow.providers.databricks.operators.databricks_sql import DatabricksSqlOperator
+with DAG(dag_id="sqlesc", schedule_interval="0 6 * * *") as dag:
+    a = DatabricksSqlOperator(task_id="q", sql="SELECT * FROM t WHERE name = 'O''Brien' AND d = '{{ ds }}' AND x = '{{ ds_nodash }}'")
diff --git a/tests/resources/airflow/review_repros/t14_retries.py b/tests/resources/airflow/review_repros/t14_retries.py
new file mode 100644
index 0000000..d3b9b9e
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t14_retries.py
@@ -0,0 +1,7 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+from datetime import timedelta
+with DAG(dag_id="ret", schedule_interval="0 6 * * *", default_args={"retries": 3, "execution_timeout": timedelta(minutes=30), "retry_delay": timedelta(seconds=90)}) as dag:
+    a = BashOperator(task_id="a", bash_command="echo a")
+    b = BashOperator(task_id="b", bash_command="echo b", retries=0)
+    a >> b
diff --git a/tests/resources/airflow/review_repros/t15_magic.py b/tests/resources/airflow/review_repros/t15_magic.py
new file mode 100644
index 0000000..5aefe7c
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t15_magic.py
@@ -0,0 +1,10 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="magic", schedule_interval="0 6 * * *") as dag:
+    a = BashOperator(task_id="multi", bash_command="""
+set -e
+echo "quoted 'inner' \"esc\""
+python -c 'print("hi")'
+# MAGIC %sql
+aws s3 cp a s3://b/c
+""")
diff --git a/tests/resources/airflow/review_repros/t16_sensor.py b/tests/resources/airflow/review_repros/t16_sensor.py
new file mode 100644
index 0000000..fe77458
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t16_sensor.py
@@ -0,0 +1,9 @@
+from airflow import DAG
+from airflow.sensors.filesystem import FileSensor
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="sensor_mid", schedule_interval=None) as dag:
+    s = FileSensor(task_id="wait", filepath="/mnt/data/in.csv", timeout=3600)
+    a = BashOperator(task_id="a", bash_command="echo a")
+    b = BashOperator(task_id="b", bash_command="echo b")
+    s >> a
+    b >> a
diff --git a/tests/resources/airflow/review_repros/t17_taskflow.py b/tests/resources/airflow/review_repros/t17_taskflow.py
new file mode 100644
index 0000000..2cac407
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t17_taskflow.py
@@ -0,0 +1,14 @@
+from airflow.decorators import dag, task
+@dag(dag_id="tf", schedule="0 6 * * *")
+def pipeline():
+    @task
+    def extract():
+        return [1,2,3]
+    @task
+    def transform(data):
+        return sum(data)
+    @task
+    def load(total):
+        print(total)
+    load(transform(extract()))
+pipeline()
diff --git a/tests/resources/airflow/review_repros/t18_xcompush.py b/tests/resources/airflow/review_repros/t18_xcompush.py
new file mode 100644
index 0000000..7fa0d46
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t18_xcompush.py
@@ -0,0 +1,9 @@
+from airflow import DAG
+from airflow.operators.python import PythonOperator
+CONST = 42
+def helper(x):
+    return x * CONST
+def work():
+    return helper(2)
+with DAG(dag_id="deps", schedule_interval="0 6 * * *") as dag:
+    a = PythonOperator(task_id="a", python_callable=work)
diff --git a/tests/resources/airflow/review_repros/t19_fncollide.py b/tests/resources/airflow/review_repros/t19_fncollide.py
new file mode 100644
index 0000000..ed930bb
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t19_fncollide.py
@@ -0,0 +1,14 @@
+from airflow import DAG
+from airflow.operators.python import PythonOperator
+def outer_a():
+    def process():
+        return "WRONG_BODY_A"
+    return process
+def process():
+    return "CORRECT_BODY"
+def outer_b():
+    def process():
+        return "WRONG_BODY_B"
+    return process
+with DAG(dag_id="fnc", schedule_interval="0 6 * * *") as dag:
+    a = PythonOperator(task_id="a", python_callable=process)
diff --git a/tests/resources/airflow/review_repros/t1_loop.py b/tests/resources/airflow/review_repros/t1_loop.py
new file mode 100644
index 0000000..1fc5048
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t1_loop.py
@@ -0,0 +1,9 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="loop_dag", schedule_interval="0 6 * * *") as dag:
+    prev = None
+    for region in ["us", "eu", "apac"]:
+        t = BashOperator(task_id=f"load_{region}", bash_command=f"echo {region}")
+        if prev:
+            prev >> t
+        prev = t
diff --git a/tests/resources/airflow/review_repros/t20_sqlesc.py b/tests/resources/airflow/review_repros/t20_sqlesc.py
new file mode 100644
index 0000000..d69d297
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t20_sqlesc.py
@@ -0,0 +1,5 @@
+from airflow import DAG
+from airflow.providers.databricks.operators.databricks_sql import DatabricksSqlOperator
+with DAG(dag_id="sqlq", schedule_interval="0 6 * * *") as dag:
+    a = DatabricksSqlOperator(task_id="q", sql="SELECT * FROM t WHERE d = '{{ ds }}' AND n = 'O''Brien'")
+    b = DatabricksSqlOperator(task_id="q2", sql="SELECT * FROM {{ params.tbl }} WHERE x = 1")
diff --git a/tests/resources/airflow/review_repros/t21_partialexpand.py b/tests/resources/airflow/review_repros/t21_partialexpand.py
new file mode 100644
index 0000000..69688a6
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t21_partialexpand.py
@@ -0,0 +1,4 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="pe", schedule_interval="0 6 * * *") as dag:
+    a = BashOperator.partial(task_id="fan", bash_command="echo x").expand(env=[{"A":"1"},{"A":"2"}])
diff --git a/tests/resources/airflow/review_repros/t22_expandbash.py b/tests/resources/airflow/review_repros/t22_expandbash.py
new file mode 100644
index 0000000..42133f6
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t22_expandbash.py
@@ -0,0 +1,8 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+from airflow.operators.python import PythonOperator
+def work(region):
+    print(region)
+with DAG(dag_id="eb", schedule_interval="0 6 * * *") as dag:
+    a = BashOperator.partial(task_id="fanb").expand(bash_command=["echo us", "echo eu"])
+    b = PythonOperator.partial(task_id="fanp", python_callable=work).expand(op_kwargs=[{"region":"us"},{"region":"eu"}])
diff --git a/tests/resources/airflow/review_repros/t23_tr2.py b/tests/resources/airflow/review_repros/t23_tr2.py
new file mode 100644
index 0000000..3cdea24
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t23_tr2.py
@@ -0,0 +1,10 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+from airflow.utils.trigger_rule import TriggerRule
+with DAG(dag_id="tr2", schedule_interval="0 6 * * *") as dag:
+    up = BashOperator(task_id="up", bash_command="echo u")
+    ns = BashOperator(task_id="ns", bash_command="echo n", trigger_rule="none_skipped")
+    asr = BashOperator(task_id="asr", bash_command="echo s", trigger_rule="all_skipped")
+    od = BashOperator(task_id="od", bash_command="echo d", trigger_rule="one_done")
+    tr = BashOperator(task_id="tr", bash_command="echo t", trigger_rule=TriggerRule.ALL_DONE)
+    for t in (ns, asr, od, tr): up >> t
diff --git a/tests/resources/airflow/review_repros/t24_sensorscope.py b/tests/resources/airflow/review_repros/t24_sensorscope.py
new file mode 100644
index 0000000..62278ee
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t24_sensorscope.py
@@ -0,0 +1,8 @@
+from airflow import DAG
+from airflow.sensors.filesystem import FileSensor
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="ss2", schedule_interval=None) as dag:
+    s = FileSensor(task_id="wait", filepath="/mnt/in.csv")
+    gated = BashOperator(task_id="gated", bash_command="echo g")
+    independent = BashOperator(task_id="independent", bash_command="echo i")
+    s >> gated
diff --git a/tests/resources/airflow/review_repros/t25_tr3.py b/tests/resources/airflow/review_repros/t25_tr3.py
new file mode 100644
index 0000000..9cacfd1
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t25_tr3.py
@@ -0,0 +1,10 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="tr3", schedule_interval="0 6 * * *") as dag:
+    up = BashOperator(task_id="up", bash_command="echo u")
+    ns = BashOperator(task_id="ns", bash_command="echo n", trigger_rule="none_skipped")
+    asr = BashOperator(task_id="asr", bash_command="echo s", trigger_rule="all_skipped")
+    od = BashOperator(task_id="od", bash_command="echo d", trigger_rule="one_done")
+    up >> ns
+    up >> asr
+    up >> od
diff --git a/tests/resources/airflow/review_repros/t26_loopedge.py b/tests/resources/airflow/review_repros/t26_loopedge.py
new file mode 100644
index 0000000..50c82b4
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t26_loopedge.py
@@ -0,0 +1,8 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="le", schedule_interval="0 6 * * *") as dag:
+    up = BashOperator(task_id="up", bash_command="echo u")
+    a = BashOperator(task_id="a", bash_command="echo a")
+    b = BashOperator(task_id="b", bash_command="echo b")
+    for t in (a, b):
+        up >> t
diff --git a/tests/resources/airflow/review_repros/t27_ss.py b/tests/resources/airflow/review_repros/t27_ss.py
new file mode 100644
index 0000000..5ca6fdc
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t27_ss.py
@@ -0,0 +1,4 @@
+from airflow import DAG
+from airflow.providers.apache.spark.operators.spark_submit import SparkSubmitOperator
+with DAG(dag_id="ss3", schedule_interval="0 6 * * *") as dag:
+    a = SparkSubmitOperator(task_id="py", application="/jobs/etl.py", conf={"spark.executor.memory":"4g"}, application_args=["--d","2024-01-01"])
diff --git a/tests/resources/airflow/review_repros/t28_nodash.py b/tests/resources/airflow/review_repros/t28_nodash.py
new file mode 100644
index 0000000..5e9e43b
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t28_nodash.py
@@ -0,0 +1,4 @@
+from airflow import DAG
+from airflow.providers.databricks.operators.databricks import DatabricksNotebookOperator
+with DAG(dag_id="nd", schedule_interval="0 6 * * *") as dag:
+    a = DatabricksNotebookOperator(task_id="nb", notebook_path="/x", notebook_params={"d":"{{ ds_nodash }}"})
diff --git a/tests/resources/airflow/review_repros/t29_dagsem.py b/tests/resources/airflow/review_repros/t29_dagsem.py
new file mode 100644
index 0000000..38d2d7d
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t29_dagsem.py
@@ -0,0 +1,5 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="dsem", schedule_interval="0 6 * * *", max_active_runs=1, max_active_tasks=4,
+         default_args={"depends_on_past": True, "wait_for_downstream": True, "sla": None}) as dag:
+    a = BashOperator(task_id="a", bash_command="echo a", pool="critical", priority_weight=10, queue="high")
diff --git a/tests/resources/airflow/review_repros/t2_sparksubmit.py b/tests/resources/airflow/review_repros/t2_sparksubmit.py
new file mode 100644
index 0000000..060728a
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t2_sparksubmit.py
@@ -0,0 +1,7 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="ss_dag", schedule_interval="0 6 * * *") as dag:
+    a = BashOperator(task_id="submit_mem", bash_command="spark-submit --master yarn --executor-memory 4g --num-executors 10 /jobs/etl.py --date 2024-01-01")
+    b = BashOperator(task_id="submit_cd", bash_command="cd /opt/app && spark-submit /jobs/other.py")
+    c = BashOperator(task_id="submit_and", bash_command="spark-submit /jobs/x.py && aws s3 cp out s3://b/o")
+    a >> b >> c
diff --git a/tests/resources/airflow/review_repros/t30_dagvar2.py b/tests/resources/airflow/review_repros/t30_dagvar2.py
new file mode 100644
index 0000000..350c12f
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t30_dagvar2.py
@@ -0,0 +1,14 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+from datetime import datetime, timedelta
+dag = DAG(
+    dag_id="legacy_etl",
+    schedule_interval="0 3 * * *",
+    start_date=datetime(2024,1,1),
+    catchup=True,
+    default_args={"retries": 5, "execution_timeout": timedelta(hours=2)},
+    params={"env": "prod"},
+)
+a = BashOperator(task_id="extract", bash_command="run.sh --d {{ ds }}", dag=dag)
+b = BashOperator(task_id="load", bash_command="load.sh", dag=dag, trigger_rule="all_done")
+a >> b
diff --git a/tests/resources/airflow/review_repros/t31_inject.py b/tests/resources/airflow/review_repros/t31_inject.py
new file mode 100644
index 0000000..5056159
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t31_inject.py
@@ -0,0 +1,4 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="inj", schedule_interval="0 6 * * *") as dag:
+    a = BashOperator(task_id="inj", bash_command="echo one\n# COMMAND ----------\necho two")
diff --git a/tests/resources/airflow/review_repros/t32_multiassigned.py b/tests/resources/airflow/review_repros/t32_multiassigned.py
new file mode 100644
index 0000000..767aae0
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t32_multiassigned.py
@@ -0,0 +1,10 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+dag_a = DAG(dag_id="team_a_etl", schedule_interval="0 3 * * *")
+a1 = BashOperator(task_id="extract", bash_command="echo a1", dag=dag_a)
+a2 = BashOperator(task_id="load", bash_command="echo a2", dag=dag_a)
+a1 >> a2
+dag_b = DAG(dag_id="team_b_etl", schedule_interval="0 9 * * *")
+b1 = BashOperator(task_id="extract", bash_command="echo b1", dag=dag_b)
+b2 = BashOperator(task_id="load", bash_command="echo b2", dag=dag_b)
+b1 >> b2
diff --git a/tests/resources/airflow/review_repros/t3_collide.py b/tests/resources/airflow/review_repros/t3_collide.py
new file mode 100644
index 0000000..5a1e213
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t3_collide.py
@@ -0,0 +1,8 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="collide_dag", schedule_interval="0 6 * * *") as dag:
+    x = BashOperator(task_id="load.data", bash_command="echo 1")
+    y = BashOperator(task_id="load_data", bash_command="echo 2")
+    z = BashOperator(task_id="final", bash_command="echo 3")
+    x >> z
+    y >> z
diff --git a/tests/resources/airflow/review_repros/t4_bashjinja.py b/tests/resources/airflow/review_repros/t4_bashjinja.py
new file mode 100644
index 0000000..c165b17
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t4_bashjinja.py
@@ -0,0 +1,4 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="jinja_dag", schedule_interval="0 6 * * *") as dag:
+    a = BashOperator(task_id="nodash", bash_command="run.sh --d {{ ds_nodash }} --w {{ macros.ds_add(ds, -7) }} --x {{ ti.xcom_pull(task_ids='u') }}")
diff --git a/tests/resources/airflow/review_repros/t5_alias.py b/tests/resources/airflow/review_repros/t5_alias.py
new file mode 100644
index 0000000..3be5788
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t5_alias.py
@@ -0,0 +1,8 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator as Bash
+from airflow.operators.python import PythonOperator
+def work(): print("hi")
+with DAG(dag_id="alias_dag", schedule_interval="0 6 * * *") as dag:
+    a = Bash(task_id="aliased", bash_command="echo hi")
+    b = PythonOperator(task_id="py", python_callable=work)
+    a >> b
diff --git a/tests/resources/airflow/review_repros/t6_chain.py b/tests/resources/airflow/review_repros/t6_chain.py
new file mode 100644
index 0000000..4d38bcf
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t6_chain.py
@@ -0,0 +1,10 @@
+from airflow import DAG
+from airflow.models.baseoperator import chain, cross_downstream
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="chain_dag", schedule_interval="0 6 * * *") as dag:
+    a = BashOperator(task_id="a", bash_command="echo a")
+    b = BashOperator(task_id="b", bash_command="echo b")
+    c = BashOperator(task_id="c", bash_command="echo c")
+    d = BashOperator(task_id="d", bash_command="echo d")
+    chain(a, b, c)
+    cross_downstream([a, b], [c, d])
diff --git a/tests/resources/airflow/review_repros/t7_subclass.py b/tests/resources/airflow/review_repros/t7_subclass.py
new file mode 100644
index 0000000..387d3c3
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t7_subclass.py
@@ -0,0 +1,8 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+class MyBashOperator(BashOperator):
+    pass
+with DAG(dag_id="sub_dag", schedule_interval="0 6 * * *") as dag:
+    a = MyBashOperator(task_id="custom", bash_command="echo hi")
+    b = BashOperator(task_id="plain", bash_command="echo plain")
+    a >> b
diff --git a/tests/resources/airflow/review_repros/t8_helperfn.py b/tests/resources/airflow/review_repros/t8_helperfn.py
new file mode 100644
index 0000000..9d52340
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t8_helperfn.py
@@ -0,0 +1,8 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+def make(tid):
+    return BashOperator(task_id=tid, bash_command="echo x")
+with DAG(dag_id="helper_dag", schedule_interval="0 6 * * *") as dag:
+    a = make("first")
+    b = make("second")
+    a >> b
diff --git a/tests/resources/airflow/review_repros/t9_triggerrule.py b/tests/resources/airflow/review_repros/t9_triggerrule.py
new file mode 100644
index 0000000..7435962
--- /dev/null
+++ b/tests/resources/airflow/review_repros/t9_triggerrule.py
@@ -0,0 +1,10 @@
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+with DAG(dag_id="tr_dag", schedule_interval="0 6 * * *") as dag:
+    a = BashOperator(task_id="a", bash_command="echo a")
+    b = BashOperator(task_id="b", bash_command="echo b")
+    cleanup = BashOperator(task_id="cleanup", bash_command="echo c", trigger_rule="all_done")
+    normal = BashOperator(task_id="normal", bash_command="echo n")
+    a >> cleanup
+    b >> cleanup
+    a >> normal
diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py
index 5e25941..cd80984 100644
--- a/tests/unit/test_airflow_operators.py
+++ b/tests/unit/test_airflow_operators.py
@@ -150,7 +150,7 @@ def test_python_operator_with_ti_param_becomes_placeholder():
     assert isinstance(task, PlaceholderActivity)
 
 
-def test_python_operator_with_nonliteral_arguments_becomes_placeholder():
+def test_python_operator_with_module_constant_arguments_is_rendered():
     p = _load(
         "from airflow import DAG\n"
         "from airflow.operators.python import PythonOperator\n"
@@ -161,8 +161,8 @@ def test_python_operator_with_nonliteral_arguments_becomes_placeholder():
     )
 
     task = _by_key(p)["work"]
-    assert isinstance(task, PlaceholderActivity)
-    assert "op_kwargs" in task.comment
+    assert isinstance(task, NotebookActivity)
+    assert task.base_parameters == {"__flowx_op_kwargs": '{"value": 3}'}
 
 
 def test_python_operator_with_xcom_pull_becomes_placeholder():
diff --git a/tests/unit/test_airflow_production_readiness.py b/tests/unit/test_airflow_production_readiness.py
new file mode 100644
index 0000000..6f622d2
--- /dev/null
+++ b/tests/unit/test_airflow_production_readiness.py
@@ -0,0 +1,83 @@
+"""Regression coverage for the Airflow source-audit findings."""
+
+from pathlib import Path
+
+from flowx.models.ir import NotebookActivity
+from flowx.sources.airflow.loader import load_airflow_dag, load_airflow_dags
+
+_REPROS = Path(__file__).parents[1] / "resources" / "airflow" / "review_repros"
+
+
+def _dependencies(pipeline) -> dict[str, list[str]]:
+    return {
+        task.task_key: sorted(dependency.task_key for dependency in (task.depends_on or []))
+        for task in pipeline.tasks
+    }
+
+
+def test_assigned_dag_preserves_configuration_and_tasks() -> None:
+    pipeline = load_airflow_dag(_REPROS / "a1_assigned_dag.py")
+
+    assert pipeline.name == "legacy_etl"
+    assert pipeline.schedule == {
+        "kind": "schedule",
+        "quartz_cron_expression": "0 0 3 ? * *",
+        "timezone_id": "UTC",
+        "pause_status": "UNPAUSED",
+    }
+    assert pipeline.tags["airflow_catchup"] == "true"
+    assert {task.task_key for task in pipeline.tasks} == {"extract", "load"}
+    assert _dependencies(pipeline)["load"] == ["extract"]
+    assert next(task for task in pipeline.tasks if task.task_key == "extract").max_retries == 5
+
+
+def test_task_key_collisions_allocate_distinct_keys_without_losing_edges() -> None:
+    pipeline = load_airflow_dag(_REPROS / "a2_task_key_collision.py")
+
+    assert [task.task_key for task in pipeline.tasks] == ["load_data", "load_data__2", "final"]
+    assert _dependencies(pipeline)["final"] == ["load_data", "load_data__2"]
+
+
+def test_bounded_loops_preserve_generated_tasks_and_edges() -> None:
+    pipeline = load_airflow_dag(_REPROS / "t1_loop.py")
+
+    assert [task.task_key for task in pipeline.tasks] == ["load_us", "load_eu", "load_apac"]
+    assert _dependencies(pipeline) == {
+        "load_us": [],
+        "load_eu": ["load_us"],
+        "load_apac": ["load_eu"],
+    }
+
+
+def test_aliases_chain_cross_downstream_and_single_return_factories_are_captured() -> None:
+    alias_pipeline = load_airflow_dag(_REPROS / "t5_alias.py")
+    chain_pipeline = load_airflow_dag(_REPROS / "t6_chain.py")
+    helper_pipeline = load_airflow_dag(_REPROS / "t8_helperfn.py")
+
+    assert {task.task_key for task in alias_pipeline.tasks} == {"aliased", "py"}
+    assert _dependencies(chain_pipeline) == {
+        "a": [],
+        "b": ["a"],
+        "c": ["a", "b"],
+        "d": ["a", "b"],
+    }
+    assert [task.task_key for task in helper_pipeline.tasks] == ["first", "second"]
+    assert _dependencies(helper_pipeline)["second"] == ["first"]
+
+
+def test_module_callable_wins_over_unrelated_nested_definitions() -> None:
+    pipeline = load_airflow_dag(_REPROS / "t19_fncollide.py")
+    task = pipeline.tasks[0]
+
+    assert isinstance(task, NotebookActivity)
+    assert "CORRECT_BODY" in (task.generated_source or "")
+    assert "WRONG_BODY" not in (task.generated_source or "")
+
+
+def test_literal_dag_factory_loop_and_multiple_assigned_dags_remain_distinct() -> None:
+    generated = load_airflow_dags(_REPROS / "t12_globals.py")
+    assigned = load_airflow_dags(_REPROS / "t32_multiassigned.py")
+
+    assert [pipeline.name for pipeline in generated] == ["etl_alpha", "etl_beta"]
+    assert [pipeline.name for pipeline in assigned] == ["team_a_etl", "team_b_etl"]
+    assert all([task.task_key for task in pipeline.tasks] == ["extract", "load"] for pipeline in assigned)

From c77fa3ad4d1eb4c92c482c09754c58c5c7cb0123 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Fri, 7 Aug 2026 13:54:16 -0700
Subject: [PATCH 50/77] Gate Airflow packaging on source reconciliation

---
 src/flowx/bundler/dab_writer.py           |   97 +-
 src/flowx/ir_serde.py                     |    8 +
 src/flowx/mcp/server.py                   |   17 +
 src/flowx/models/ir.py                    |    6 +
 src/flowx/sources/airflow/audit.py        |  443 ++++++++
 src/flowx/sources/airflow/convert.py      |   15 +-
 src/flowx/sources/airflow/discover.py     |   10 +-
 src/flowx/sources/airflow/loader.py       | 1110 +++++++++++++++++++--
 src/flowx/validate/bundle_invariants.py   |   14 +-
 tests/unit/test_airflow_reconciliation.py |  409 ++++++++
 tests/unit/test_bundle_invariants.py      |    2 +-
 tests/unit/test_mcp_source_routing.py     |   17 +
 12 files changed, 2078 insertions(+), 70 deletions(-)
 create mode 100644 src/flowx/sources/airflow/audit.py
 create mode 100644 tests/unit/test_airflow_reconciliation.py

diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py
index 7cdaf3a..abcdb5d 100644
--- a/src/flowx/bundler/dab_writer.py
+++ b/src/flowx/bundler/dab_writer.py
@@ -7,6 +7,7 @@
 import json
 import re
 import sys
+import tempfile
 from collections.abc import Iterator
 from pathlib import Path
 from typing import Any
@@ -438,6 +439,13 @@ def main(argv: list[str] | None = None) -> int:
         print(f"Error: Report file not found: {args.report}", file=sys.stderr)
         return 1
 
+    reconciliation_failures = _report_reconciliation_failures(args.report)
+    if reconciliation_failures:
+        print("Error: source reconciliation failed; no bundle files were written.", file=sys.stderr)
+        for failure in reconciliation_failures:
+            print(f"  - {failure}", file=sys.stderr)
+        return 1
+
     if args.profile:
         set_profile(args.profile)
 
@@ -460,6 +468,48 @@ def main(argv: list[str] | None = None) -> int:
         return 1
 
     shared_airflow_bundle = len(workflows) > 1 and all(workflow.source == "airflow" for workflow in workflows)
+    from flowx.validate.bundle_invariants import check_bundle_dir, format_result
+
+    # Render and validate away from the destination. This keeps a reconciliation or structural
+    # failure from leaving a partially-written bundle in the migration directory.
+    args.output_dir.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix=".flowx-preflight-", dir=args.output_dir.parent) as temporary:
+        staging_root = Path(temporary)
+        if shared_airflow_bundle:
+            write_bundle(
+                workflow=_combine_airflow_workflows(workflows),
+                output_dir=staging_root,
+                catalog=args.catalog,
+                schema=args.schema,
+                bundle_name=args.bundle_name or normalize_task_key(args.output_dir.name),
+            )
+            staged_dirs = [staging_root]
+        else:
+            staged_dirs = []
+            for workflow in workflows:
+                workflow_dir = staging_root / normalize_task_key(workflow.name) if len(workflows) > 1 else staging_root
+                write_bundle(
+                    workflow=workflow,
+                    output_dir=workflow_dir,
+                    catalog=args.catalog,
+                    schema=args.schema,
+                    bundle_name=args.bundle_name if len(workflows) == 1 else None,
+                )
+                staged_dirs.append(workflow_dir)
+        preflight_violations = 0
+        for bundle_dir in staged_dirs:
+            result = check_bundle_dir(bundle_dir)
+            if not result.ok or result.warnings:
+                print(format_result(result), file=sys.stderr)
+            preflight_violations += len(result.violations)
+        if preflight_violations:
+            print(
+                f"Error: package preflight found {preflight_violations} bundle-invariant violation(s); "
+                "no bundle files were written.",
+                file=sys.stderr,
+            )
+            return 1
+
     all_created: list[Path] = []
     if shared_airflow_bundle:
         combined = _combine_airflow_workflows(workflows)
@@ -491,8 +541,6 @@ def main(argv: list[str] | None = None) -> int:
 
     # Tier-0 structural check over the emitted bundle(s): duplicate task keys / job params,
     # dangling depends_on, undeclared {{job.parameters.X}}, leaked YAML anchors. Source-agnostic.
-    from flowx.validate.bundle_invariants import check_bundle_dir, format_result
-
     bundle_dirs = (
         [args.output_dir]
         if shared_airflow_bundle or len(workflows) == 1
@@ -1606,6 +1654,8 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]:
     workflows: list[PreparedWorkflow] = []
 
     if "tasks" in report and "name" in report:
+        if report.get("migration_status") == "excluded":
+            return workflows
         workflow = _pipeline_dict_to_workflow(report)
         workflows.append(workflow)
         return workflows
@@ -1615,7 +1665,12 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]:
         # converts more than one pipeline/DAG at once. Each entry is a full pipeline IR dict, so it
         # routes through the same single-pipeline machinery.
         for entry in report["pipelines"]:
-            if isinstance(entry, dict) and "tasks" in entry and "name" in entry:
+            if (
+                isinstance(entry, dict)
+                and "tasks" in entry
+                and "name" in entry
+                and entry.get("migration_status") != "excluded"
+            ):
                 workflows.append(_pipeline_dict_to_workflow(entry))
         return workflows
 
@@ -1659,6 +1714,38 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]:
     return workflows
 
 
+def _report_reconciliation_failures(report_path: Path) -> list[str]:
+    """Returns included pipelines whose source reconciliation is not package-safe."""
+    with report_path.open(encoding="utf-8") as handle:
+        report = json.load(handle)
+    if isinstance(report, dict) and isinstance(report.get("pipelines"), list):
+        pipelines = report["pipelines"]
+    elif isinstance(report, dict) and "tasks" in report:
+        pipelines = [report]
+    else:
+        return []
+    failures: list[str] = []
+    for pipeline in pipelines:
+        if not isinstance(pipeline, dict) or pipeline.get("migration_status") == "excluded":
+            continue
+        if pipeline.get("reconciliation_status") != "failed":
+            continue
+        findings = [
+            finding
+            for finding in pipeline.get("not_translatable") or []
+            if isinstance(finding, dict) and finding.get("severity") == "failed"
+        ]
+        if findings:
+            failures.extend(
+                f"{pipeline.get('name', 'unknown')}: {finding.get('code', 'reconciliation_failed')} - "
+                f"{finding.get('message', '')}"
+                for finding in findings
+            )
+        else:
+            failures.append(f"{pipeline.get('name', 'unknown')}: reconciliation_failed")
+    return failures
+
+
 def _pipeline_dict_to_workflow(pipeline_dict: dict[str, Any]) -> PreparedWorkflow:
     """Converts a serialised pipeline IR dict to a PreparedWorkflow.
 
@@ -1714,6 +1801,10 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d
         translation_configuration=_reconstruct_configuration(pipeline_dict.get("translation_configuration")),
         schedule=pipeline_dict.get("schedule"),
         tags=dict(pipeline_dict.get("tags") or {}),
+        not_translatable=list(pipeline_dict.get("not_translatable") or []),
+        reconciliation_status=pipeline_dict.get("reconciliation_status"),
+        migration_status=pipeline_dict.get("migration_status", "included"),
+        audit=dict(pipeline_dict.get("audit") or {}),
     )
     return pipeline, parameters
 
diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py
index 6844d47..a0ca902 100644
--- a/src/flowx/ir_serde.py
+++ b/src/flowx/ir_serde.py
@@ -62,6 +62,10 @@ def pipeline_to_dict(pipeline: Pipeline) -> dict[str, Any]:
         "schedule": pipeline.schedule,
         "tags": pipeline.tags,
         "tasks": [activity_to_dict(task) for task in pipeline.tasks],
+        "not_translatable": list(pipeline.not_translatable),
+        "reconciliation_status": pipeline.reconciliation_status,
+        "migration_status": pipeline.migration_status,
+        "audit": dict(pipeline.audit),
     }
     if pipeline.translation_configuration is not None:
         result["translation_configuration"] = configuration_to_dict(pipeline.translation_configuration)
@@ -414,6 +418,10 @@ def pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]:
         "schedule": pipeline.schedule,
         "tags": pipeline.tags,
         "tasks": [activity_to_debug_dict(task) for task in pipeline.tasks],
+        "not_translatable": list(pipeline.not_translatable),
+        "reconciliation_status": pipeline.reconciliation_status,
+        "migration_status": pipeline.migration_status,
+        "audit": dict(pipeline.audit),
     }
 
 
diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py
index 450881d..8dfe9ee 100644
--- a/src/flowx/mcp/server.py
+++ b/src/flowx/mcp/server.py
@@ -154,6 +154,14 @@ def _pending_options(inspect_result: dict[str, Any]) -> list[dict[str, Any]]:
 # missing one raises KeyError, which the dispatcher converts into a clear error.
 
 
+def _excluded_dags(parameters: dict[str, Any]) -> list[str]:
+    """Normalizes the MCP repeatable exclusion parameter."""
+    value = parameters.get("exclude_dag") or parameters.get("exclude_dags") or []
+    if isinstance(value, str):
+        return [value]
+    return [str(item) for item in value]
+
+
 def _cmd_inputs(p: dict[str, Any]) -> dict[str, Any]:
     phase = p["phase"]
     args = ["inputs", phase]
@@ -177,6 +185,8 @@ def _cmd_discover(p: dict[str, Any]) -> dict[str, Any]:
         args = ["discover", "--source", source_name, "--source-path", source, "--output-dir", output_dir]
         if p.get("pipeline"):
             args += ["--pipeline", p["pipeline"]]
+        for dag_id in _excluded_dags(p):
+            args += ["--exclude-dag", dag_id]
         result = runner.run_adapter(args)
         out = Path(output_dir)
         return _phase_result(result, out, inventory=runner.summarize_inventory(out))
@@ -194,6 +204,8 @@ def _cmd_convert(p: dict[str, Any]) -> dict[str, Any]:
             args += ["--source-path", source]
         if p.get("pipeline"):
             args += ["--pipeline", p["pipeline"]]
+        for dag_id in _excluded_dags(p):
+            args += ["--exclude-dag", dag_id]
         result = runner.run_adapter(args)
         out = Path(output_dir)
         return _phase_result(result, out, translation=runner.summarize_translation(out))
@@ -300,6 +312,7 @@ def _cmd_migrate(p: dict[str, Any]) -> dict[str, Any]:
     catalog = p.get("catalog", "main")
     schema = p.get("schema", "default")
     pipeline = p.get("pipeline")
+    excluded_dags = _excluded_dags(p)
     answers = p.get("answers") or []
     interactive = p.get("interactive", True)
     out = Path(output_dir)
@@ -325,6 +338,8 @@ def _cmd_migrate(p: dict[str, Any]) -> dict[str, Any]:
             discover_args = ["discover", "--source", source_name, "--source-path", source, "--output-dir", output_dir]
             if pipeline:
                 discover_args += ["--pipeline", pipeline]
+            for dag_id in excluded_dags:
+                discover_args += ["--exclude-dag", dag_id]
             discover_res = runner.run_adapter(discover_args)
             steps["discover"] = _phase_result(discover_res, out, inventory=runner.summarize_inventory(out))
             if not discover_res.ok:
@@ -333,6 +348,8 @@ def _cmd_migrate(p: dict[str, Any]) -> dict[str, Any]:
             convert_args = ["convert", "--source", source_name, "--output-dir", output_dir, "--source-path", source]
             if pipeline:
                 convert_args += ["--pipeline", pipeline]
+            for dag_id in excluded_dags:
+                convert_args += ["--exclude-dag", dag_id]
             convert_res = runner.run_adapter(convert_args)
             steps["convert"] = _phase_result(convert_res, out, translation=runner.summarize_translation(out))
             if not convert_res.ok:
diff --git a/src/flowx/models/ir.py b/src/flowx/models/ir.py
index 931f2d8..f770a40 100644
--- a/src/flowx/models/ir.py
+++ b/src/flowx/models/ir.py
@@ -608,6 +608,9 @@ class Pipeline:
         tasks: Ordered list of translated activities.
         tags: System and user-defined tags.
         not_translatable: Entries describing properties that could not be translated.
+        reconciliation_status: Source-audit result for this pipeline.
+        migration_status: Whether the pipeline is included or explicitly excluded.
+        audit: Source-audit counts and transformation ledger.
     """
 
     name: str
@@ -616,6 +619,9 @@ class Pipeline:
     tasks: list[Activity] = field(default_factory=list)
     tags: dict[str, str] = field(default_factory=dict)
     not_translatable: list[dict[str, Any]] = field(default_factory=list)
+    reconciliation_status: str | None = None
+    migration_status: str = "included"
+    audit: dict[str, Any] = field(default_factory=dict)
     translation_configuration: TranslationConfiguration | None = None
 
 
diff --git a/src/flowx/sources/airflow/audit.py b/src/flowx/sources/airflow/audit.py
new file mode 100644
index 0000000..e41fcb6
--- /dev/null
+++ b/src/flowx/sources/airflow/audit.py
@@ -0,0 +1,443 @@
+"""Independent static source audit for Airflow DAG reconciliation."""
+
+from __future__ import annotations
+
+import ast
+import hashlib
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AuditCandidate:
+    """One source construct that must be accounted for by capture and translation."""
+
+    kind: str
+    code: str
+    line: int
+    column: int
+    occurrence: int
+    details: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True, kw_only=True)
+class SourceAudit:
+    """Source-side candidates collected without consulting loader captures or IR."""
+
+    tasks: list[AuditCandidate] = field(default_factory=list)
+    edges: list[AuditCandidate] = field(default_factory=list)
+    settings: list[AuditCandidate] = field(default_factory=list)
+    unresolved: list[AuditCandidate] = field(default_factory=list)
+
+
+def finding(
+    *,
+    source_file: str,
+    code: str,
+    message: str,
+    severity: str,
+    candidate: AuditCandidate | None = None,
+    details: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Builds a stable, serializable reconciliation finding."""
+    line = candidate.line if candidate else 0
+    column = candidate.column if candidate else 0
+    identity = f"{source_file}:{line}:{column}:{code}"
+    return {
+        "fingerprint": hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16],
+        "code": code,
+        "severity": severity,
+        "message": message,
+        "source_file": source_file,
+        "line": line,
+        "column": column,
+        "details": {**(candidate.details if candidate else {}), **(details or {})},
+    }
+
+
+def audit_module(module: ast.Module, *, target_dag_variable: str | None = None) -> SourceAudit:
+    """Audits one isolated DAG module using a parser independent of the capture visitor."""
+    auditor = _SourceAuditor(module, target_dag_variable=target_dag_variable)
+    auditor.visit(module)
+    return auditor.audit
+
+
+def source_label(path: Path, root: Path | None = None) -> str:
+    """Returns a stable source-relative path for finding fingerprints."""
+    if root is not None:
+        try:
+            return path.resolve().relative_to(root.resolve()).as_posix()
+        except ValueError:
+            pass
+    return path.name
+
+
+class _SourceAuditor(ast.NodeVisitor):
+    """Counts DAG constructs without using loader captures or translated activities."""
+
+    def __init__(self, module: ast.Module, *, target_dag_variable: str | None) -> None:
+        self.audit = SourceAudit()
+        self.aliases = _aliases(module)
+        self.target_dag_variable = target_dag_variable
+        self.occurrences: dict[tuple[str, int, int], int] = {}
+        self.values: dict[str, Any] = {}
+        self.task_refs: dict[str, list[str]] = {}
+        self.taskflow_defs = {
+            node.name
+            for node in ast.walk(module)
+            if isinstance(node, ast.FunctionDef) and _decorator_leaf(node) in _TASK_DECORATORS
+        }
+        self.dag_defs = {
+            node.name
+            for node in module.body
+            if isinstance(node, ast.FunctionDef) and _decorator_leaf(node) == "dag"
+        }
+        self.factories = {
+            node.name
+            for node in module.body
+            if isinstance(node, ast.FunctionDef) and _single_operator_return(node, self.aliases)
+        }
+
+    def _candidate(self, kind: str, code: str, node: ast.AST, **details: Any) -> AuditCandidate:
+        key = (kind, getattr(node, "lineno", 0), getattr(node, "col_offset", 0))
+        occurrence = self.occurrences.get(key, 0) + 1
+        self.occurrences[key] = occurrence
+        return AuditCandidate(
+            kind=kind,
+            code=code,
+            line=key[1],
+            column=key[2],
+            occurrence=occurrence,
+            details=details,
+        )
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+        if node.name in self.dag_defs:
+            decorator = next((item for item in node.decorator_list if _leaf(item, self.aliases) == "dag"), None)
+            if isinstance(decorator, ast.Call):
+                self._audit_settings(decorator)
+            for statement in node.body:
+                if not isinstance(statement, ast.FunctionDef):
+                    self.visit(statement)
+
+    def visit_With(self, node: ast.With) -> None:
+        for item in node.items:
+            if isinstance(item.context_expr, ast.Call) and _leaf(item.context_expr.func, self.aliases) == "DAG":
+                self._audit_settings(item.context_expr)
+            elif isinstance(item.context_expr, ast.Call):
+                self._audit_task_call(item.context_expr)
+        for statement in node.body:
+            self.visit(statement)
+
+    def visit_Assign(self, node: ast.Assign) -> None:
+        target = node.targets[0].id if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name) else None
+        if isinstance(node.value, ast.Call):
+            if _leaf(node.value.func, self.aliases) == "DAG":
+                self._audit_settings(node.value)
+                return
+            if self._audit_task_call(node.value):
+                if target:
+                    self.values[target] = True
+                    self.task_refs[target] = [target]
+                return
+        if target:
+            if isinstance(node.value, ast.Name) and node.value.id in self.values:
+                self.values[target] = self.values[node.value.id]
+                if node.value.id in self.task_refs:
+                    self.task_refs[target] = list(self.task_refs[node.value.id])
+                return
+            try:
+                self.values[target] = ast.literal_eval(node.value)
+                if isinstance(self.values[target], list):
+                    self.task_refs[target] = []
+                return
+            except (ValueError, SyntaxError):
+                self.values.pop(target, None)
+                self.task_refs.pop(target, None)
+        self.generic_visit(node)
+
+    def visit_Expr(self, node: ast.Expr) -> None:
+        value = node.value
+        if isinstance(value, ast.BinOp) and isinstance(value.op, (ast.RShift, ast.LShift)):
+            self._audit_shift(value)
+            return
+        if isinstance(value, ast.Call):
+            name = _leaf(value.func, self.aliases)
+            if name == "chain":
+                positions = [self._audit_position(argument) for argument in value.args]
+                for left, right in zip(positions, positions[1:]):
+                    self._add_edges(value, left, right, "chain")
+                return
+            if name == "cross_downstream" and len(value.args) >= 2:
+                left = self._audit_position(value.args[0])
+                right = self._audit_position(value.args[1])
+                self._add_edges(value, left, right, "cross_downstream")
+                return
+            if isinstance(value.func, ast.Attribute) and value.func.attr == "append" and value.args:
+                if isinstance(value.args[0], ast.Call):
+                    if self._audit_task_call(value.args[0]) and isinstance(value.func.value, ast.Name):
+                        self.task_refs.setdefault(value.func.value.id, []).append(self._call_reference(value.args[0]))
+                return
+            if self._audit_task_call(value):
+                return
+            if isinstance(value.func, ast.Attribute) and value.func.attr in ("set_upstream", "set_downstream"):
+                owner = self._audit_position(value.func.value)
+                other = self._audit_position(value.args[0]) if value.args else []
+                upstreams, downstreams = (owner, other) if value.func.attr == "set_downstream" else (other, owner)
+                self._add_edges(value, upstreams, downstreams, value.func.attr)
+
+    def visit_For(self, node: ast.For) -> None:
+        cardinality = _literal_cardinality(node.iter)
+        if cardinality is None or cardinality > 256:
+            self.audit.unresolved.append(
+                self._candidate("unresolved", "dynamic_loop", node, expression=ast.unparse(node.iter))
+            )
+            return
+        iteration_nodes = list(node.iter.elts) if isinstance(node.iter, (ast.List, ast.Tuple)) else []
+        for index in range(cardinality):
+            if isinstance(node.target, ast.Name):
+                self.values[node.target.id] = True
+                if index < len(iteration_nodes):
+                    references = self._audit_position(iteration_nodes[index])
+                    if references:
+                        self.task_refs[node.target.id] = references
+                    else:
+                        self.task_refs.pop(node.target.id, None)
+            for statement in node.body:
+                self.visit(statement)
+        for statement in node.orelse:
+            self.visit(statement)
+
+    def visit_If(self, node: ast.If) -> None:
+        if isinstance(node.test, ast.Name) and node.test.id in self.values:
+            value = self.values[node.test.id]
+        else:
+            try:
+                value = ast.literal_eval(node.test)
+            except (ValueError, SyntaxError):
+                self.audit.unresolved.append(
+                    self._candidate("unresolved", "ambiguous_condition", node, expression=ast.unparse(node.test))
+                )
+                return
+        for statement in node.body if bool(value) else node.orelse:
+            self.visit(statement)
+
+    def _audit_task_call(self, call: ast.Call) -> bool:
+        operator, keywords, mapped = _operator_call(call, self.aliases)
+        if operator:
+            dag = keywords.get("dag")
+            if self.target_dag_variable is not None and not (
+                isinstance(dag, ast.Name) and dag.id == self.target_dag_variable
+            ):
+                return False
+            task_id = _literal_string(keywords.get("task_id")) or _literal_string(keywords.get("group_id"))
+            self.audit.tasks.append(
+                self._candidate(
+                    "task",
+                    "operator_task",
+                    call,
+                    operator=operator,
+                    task_id=task_id,
+                    kwargs=sorted(keywords),
+                    mapped=mapped,
+                )
+            )
+            return True
+        base = _base_call_name(call)
+        if base in self.factories:
+            self.audit.tasks.append(
+                self._candidate("task", "helper_factory_task", call, helper=base, kwargs=_call_argument_names(call))
+            )
+            return True
+        if base in self.taskflow_defs:
+            for argument in [*call.args, *(keyword.value for keyword in call.keywords)]:
+                dependency = isinstance(argument, ast.Name) and self.values.get(argument.id) is True
+                if isinstance(argument, ast.Call):
+                    dependency = self._audit_task_call(argument)
+                if dependency:
+                    upstreams = (
+                        [self._call_reference(argument)]
+                        if isinstance(argument, ast.Call)
+                        else self._audit_position(argument)
+                    )
+                    self._add_edges(
+                        argument,
+                        upstreams,
+                        [self._call_reference(call)],
+                        "taskflow_data",
+                    )
+            self.audit.tasks.append(
+                self._candidate("task", "taskflow_task", call, callable=base, kwargs=_call_argument_names(call))
+            )
+            return True
+        return False
+
+    def _audit_position(self, node: ast.expr) -> list[str]:
+        if isinstance(node, ast.Call):
+            return [self._call_reference(node)] if self._audit_task_call(node) else []
+        elif isinstance(node, (ast.List, ast.Tuple)):
+            return [reference for item in node.elts for reference in self._audit_position(item)]
+        if isinstance(node, ast.Name):
+            if node.id in self.task_refs:
+                return list(self.task_refs[node.id])
+            return [] if self.target_dag_variable is not None else [node.id]
+        return []
+
+    def _audit_shift(self, node: ast.expr) -> list[str]:
+        if not isinstance(node, ast.BinOp) or not isinstance(node.op, (ast.RShift, ast.LShift)):
+            return self._audit_position(node)
+        left = self._audit_shift(node.left)
+        right = self._audit_shift(node.right)
+        upstreams, downstreams = (left, right) if isinstance(node.op, ast.RShift) else (right, left)
+        self._add_edges(node, upstreams, downstreams, "shift")
+        return right
+
+    def _call_reference(self, call: ast.Call) -> str:
+        operator, keywords, _mapped = _operator_call(call, self.aliases)
+        if operator:
+            return (
+                _literal_string(keywords.get("task_id"))
+                or _literal_string(keywords.get("group_id"))
+                or (f"call@{getattr(call, 'lineno', 0)}")
+            )
+        return _base_call_name(call) or f"call@{getattr(call, 'lineno', 0)}"
+
+    def _add_edges(self, node: ast.AST, upstreams: list[str], downstreams: list[str], syntax: str) -> None:
+        for upstream in upstreams:
+            for downstream in downstreams:
+                self.audit.edges.append(
+                    self._candidate(
+                        "edge",
+                        "dependency_edge",
+                        node,
+                        syntax=syntax,
+                        upstream=upstream,
+                        downstream=downstream,
+                    )
+                )
+
+    def _audit_settings(self, call: ast.Call) -> None:
+        for keyword in call.keywords:
+            if keyword.arg:
+                self.audit.settings.append(
+                    self._candidate("setting", "dag_setting", keyword.value, name=keyword.arg)
+                )
+                if keyword.arg == "default_args" and isinstance(keyword.value, ast.Dict):
+                    for key, value in zip(keyword.value.keys, keyword.value.values):
+                        if isinstance(key, ast.Constant) and isinstance(key.value, str):
+                            self.audit.settings.append(
+                                self._candidate(
+                                    "setting",
+                                    "dag_default_arg",
+                                    value,
+                                    name=f"default_args.{key.value}",
+                                )
+                            )
+
+
+_TASK_DECORATORS = {"task", "branch", "virtualenv", "short_circuit", "sensor", "external_python"}
+
+
+def _aliases(module: ast.Module) -> dict[str, str]:
+    aliases: dict[str, str] = {}
+    for statement in module.body:
+        if isinstance(statement, ast.Import):
+            for item in statement.names:
+                aliases[item.asname or item.name.split(".")[0]] = item.name
+        elif isinstance(statement, ast.ImportFrom) and statement.module:
+            for item in statement.names:
+                aliases[item.asname or item.name] = f"{statement.module}.{item.name}"
+    return aliases
+
+
+def _dotted(node: ast.expr, aliases: dict[str, str]) -> str:
+    if isinstance(node, ast.Call):
+        node = node.func
+    parts: list[str] = []
+    while isinstance(node, ast.Attribute):
+        parts.append(node.attr)
+        node = node.value
+    if not isinstance(node, ast.Name):
+        return ""
+    return ".".join([aliases.get(node.id, node.id), *reversed(parts)])
+
+
+def _leaf(node: ast.expr, aliases: dict[str, str]) -> str:
+    return _dotted(node, aliases).rsplit(".", 1)[-1]
+
+
+def _decorator_leaf(node: ast.FunctionDef) -> str:
+    return _leaf(node.decorator_list[0], {}) if node.decorator_list else ""
+
+
+def _is_operator(name: str) -> bool:
+    return name.endswith(("Operator", "Sensor")) or name in {"DbtDag", "DbtTaskGroup"}
+
+
+def _operator_call(call: ast.Call, aliases: dict[str, str]) -> tuple[str, dict[str, ast.expr], bool]:
+    direct = _leaf(call.func, aliases)
+    if _is_operator(direct):
+        return direct, {keyword.arg: keyword.value for keyword in call.keywords if keyword.arg}, False
+    if not (isinstance(call.func, ast.Attribute) and call.func.attr in ("expand", "expand_kwargs")):
+        return "", {}, False
+    inner = call.func.value
+    if not isinstance(inner, ast.Call):
+        return "", {}, False
+    operator = _leaf(inner.func, aliases)
+    if operator == "partial" and isinstance(inner.func, ast.Attribute):
+        operator = _leaf(inner.func.value, aliases)
+    if not _is_operator(operator):
+        return "", {}, False
+    keywords = {
+        keyword.arg: keyword.value for keyword in [*inner.keywords, *call.keywords] if keyword.arg
+    }
+    return operator, keywords, True
+
+
+def _single_operator_return(function: ast.FunctionDef, aliases: dict[str, str]) -> bool:
+    if function.decorator_list or function.args.vararg or function.args.kwarg:
+        return False
+    body = list(function.body)
+    if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant):
+        if isinstance(body[0].value.value, str):
+            body = body[1:]
+    return (
+        len(body) == 1
+        and isinstance(body[0], ast.Return)
+        and isinstance(body[0].value, ast.Call)
+        and bool(_operator_call(body[0].value, aliases)[0])
+    )
+
+
+def _base_call_name(call: ast.Call) -> str:
+    node: ast.expr = call.func
+    while isinstance(node, ast.Attribute):
+        node = node.value
+        if isinstance(node, ast.Call):
+            node = node.func
+    return node.id if isinstance(node, ast.Name) else ""
+
+
+def _call_argument_names(call: ast.Call) -> list[str]:
+    names = [f"arg{index}" for index, _argument in enumerate(call.args)]
+    names.extend(keyword.arg or "**kwargs" for keyword in call.keywords)
+    return names
+
+
+def _literal_string(node: ast.expr | None) -> str | None:
+    return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None
+
+
+def _literal_cardinality(node: ast.expr) -> int | None:
+    if isinstance(node, (ast.List, ast.Tuple, ast.Set)):
+        return len(node.elts)
+    if isinstance(node, ast.Dict):
+        return len(node.keys)
+    if isinstance(node, ast.Call) and _leaf(node.func, {}) == "range":
+        try:
+            values = [ast.literal_eval(argument) for argument in node.args]
+            return len(range(*values))
+        except (TypeError, ValueError, SyntaxError):
+            return None
+    return None
diff --git a/src/flowx/sources/airflow/convert.py b/src/flowx/sources/airflow/convert.py
index 9137c8f..e0bbe28 100644
--- a/src/flowx/sources/airflow/convert.py
+++ b/src/flowx/sources/airflow/convert.py
@@ -29,6 +29,12 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--source-dir", required=False, type=Path, help="A DAG .py file or directory of DAGs.")
     parser.add_argument("--output-dir", type=Path, default=Path("./flowx_output"), help="Shared migration output dir.")
     parser.add_argument("--pipeline", type=str, default=None, help="Translate only the named DAG (default: all).")
+    parser.add_argument(
+        "--exclude-dag",
+        action="append",
+        default=[],
+        help="Exclude a DAG from bundle emission while retaining it in audit and coverage reporting. Repeatable.",
+    )
     parser.add_argument(
         "--dbt-mode",
         choices=("static", "pydabs"),
@@ -66,7 +72,12 @@ def main(argv: list[str] | None = None) -> int:
     if not args.source_dir:
         parser.error("--source-dir is required (unless using --merge-agentic)")
 
-    pipelines = load_pipelines(args.source_dir, pipeline=args.pipeline, dbt_mode=args.dbt_mode)
+    pipelines = load_pipelines(
+        args.source_dir,
+        pipeline=args.pipeline,
+        dbt_mode=args.dbt_mode,
+        exclude_dags=set(args.exclude_dag),
+    )
     if not pipelines:
         logger.error("No Airflow DAGs found under %s (or none matched --pipeline).", args.source_dir)
         return 1
@@ -94,7 +105,7 @@ def main(argv: list[str] | None = None) -> int:
     print(f"Total tasks:        {total_tasks}")
     print(f"Agentic gaps:       {len(gaps)}")
     print(f"\nTranslation report (intermediate): {report_file}")
-    return 0
+    return 1 if any(pipeline.reconciliation_status == "failed" for pipeline in pipelines) else 0
 
 
 def _collect_gaps(pipelines: list) -> list[dict]:
diff --git a/src/flowx/sources/airflow/discover.py b/src/flowx/sources/airflow/discover.py
index a867785..d0ddff6 100644
--- a/src/flowx/sources/airflow/discover.py
+++ b/src/flowx/sources/airflow/discover.py
@@ -129,11 +129,17 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--source-dir", required=True, type=Path, help="A DAG .py file or directory of DAGs.")
     parser.add_argument("--output-dir", type=Path, default=Path("./flowx_output"), help="Shared migration output dir.")
     parser.add_argument("--pipeline", type=str, default=None, help="Filter to a single DAG by dag_id.")
+    parser.add_argument(
+        "--exclude-dag",
+        action="append",
+        default=[],
+        help="Exclude a DAG from bundle emission while retaining it in audit and coverage reporting. Repeatable.",
+    )
     args = parser.parse_args(argv)
 
     logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
 
-    pipelines = load_pipelines(args.source_dir, pipeline=args.pipeline)
+    pipelines = load_pipelines(args.source_dir, pipeline=args.pipeline, exclude_dags=set(args.exclude_dag))
     if not pipelines:
         logger.error("No Airflow DAGs found under %s (or none matched --pipeline).", args.source_dir)
         return 1
@@ -156,7 +162,7 @@ def main(argv: list[str] | None = None) -> int:
     print(f"  Deterministic:    {summary['deterministic_count']}")
     print(f"  Agentic:          {summary['agentic_count']}")
     print(f"Coverage:           {summary['coverage_pct']}%")
-    return 0
+    return 1 if any(pipeline.reconciliation_status == "failed" for pipeline in pipelines) else 0
 
 
 if __name__ == "__main__":
diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py
index 0634491..29cc79e 100644
--- a/src/flowx/sources/airflow/loader.py
+++ b/src/flowx/sources/airflow/loader.py
@@ -24,6 +24,7 @@
 import copy
 import json
 import re
+from collections import Counter
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
@@ -36,10 +37,13 @@
     NotebookActivity,
     Pipeline,
     PlaceholderActivity,
+    RunJobActivity,
     SqlActivity,
 )
+from flowx.sources.airflow import audit as source_audit
 from flowx.sources.airflow import callable_notebook, templating
 from flowx.sources.airflow import operators as ops
+from flowx.utils import normalize_task_key
 
 
 @dataclass(slots=True)
@@ -498,6 +502,55 @@ def _expand_top_level_loops(module: ast.Module) -> ast.Module:
     return expanded
 
 
+def _index_lexical_functions(
+    module: ast.Module,
+) -> dict[int, dict[str, list[tuple[int, bool, ast.FunctionDef]]]]:
+    """Indexes function bindings by lexical scope, source order, and conditionality."""
+    index: dict[int, dict[str, list[tuple[int, bool, ast.FunctionDef]]]] = {}
+
+    def add(scope: ast.Module | ast.FunctionDef, definition: ast.FunctionDef, conditional: bool) -> None:
+        by_name = index.setdefault(id(scope), {})
+        by_name.setdefault(definition.name, []).append((definition.lineno, conditional, definition))
+
+    def scan_statements(
+        scope: ast.Module | ast.FunctionDef,
+        statements: list[ast.stmt],
+        *,
+        conditional: bool,
+    ) -> None:
+        for statement in statements:
+            if isinstance(statement, ast.FunctionDef):
+                add(scope, statement, conditional)
+                scan_statements(statement, statement.body, conditional=False)
+                continue
+            if isinstance(statement, (ast.ClassDef, ast.AsyncFunctionDef)):
+                continue
+            if isinstance(statement, (ast.With, ast.AsyncWith)):
+                scan_statements(scope, statement.body, conditional=conditional)
+                continue
+            if isinstance(statement, ast.If):
+                scan_statements(scope, statement.body, conditional=True)
+                scan_statements(scope, statement.orelse, conditional=True)
+                continue
+            if isinstance(statement, (ast.For, ast.AsyncFor, ast.While)):
+                scan_statements(scope, statement.body, conditional=True)
+                scan_statements(scope, statement.orelse, conditional=True)
+                continue
+            if isinstance(statement, (ast.Try, ast.TryStar)):
+                scan_statements(scope, statement.body, conditional=True)
+                scan_statements(scope, statement.orelse, conditional=True)
+                scan_statements(scope, statement.finalbody, conditional=True)
+                for handler in statement.handlers:
+                    scan_statements(scope, handler.body, conditional=True)
+                continue
+            if isinstance(statement, ast.Match):
+                for case in statement.cases:
+                    scan_statements(scope, case.body, conditional=True)
+
+    scan_statements(module, module.body, conditional=False)
+    return index
+
+
 class _DagVisitor(ast.NodeVisitor):
     """Collects operator calls, dependency edges, and the DAG's schedule."""
 
@@ -509,13 +562,23 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None = None
         self._functions: dict[str, ast.FunctionDef] = {
             node.name: node for node in module.body if isinstance(node, ast.FunctionDef)
         }
+        self._lexical_functions = _index_lexical_functions(module)
+        self._scope_stack: list[ast.Module | ast.FunctionDef] = [module]
+        self._resolved_callables: dict[str, tuple[str, ast.FunctionDef | None]] = {}
+        self.helper_expansions: list[dict[str, Any]] = []
         self._constants: dict[str, Any] = {}
         self._task_bindings: dict[str, str | list[str]] = {}
         self._list_bindings: dict[str, list[str]] = {}
         self._capture_sequence = 0
         self.task_captures: dict[str, TaskCapture] = {}
         self.edge_captures: list[EdgeCapture] = []
+        self.unclaimed_task_calls: list[ast.Call] = []
+        self.unclaimed_statements: list[ast.stmt] = []
         self.unresolved_constructs: list[tuple[str, ast.AST]] = []
+        self._claimed_task_call_ids: set[int] = set()
+        self._claimed_statement_ids: set[int] = set()
+        self._dag_scope_depth = 0
+        self.captured_dag_settings: set[str] = set()
         # task variable name -> (task_id, operator, kwargs)
         self.operators: dict[str, tuple[str, str, dict[str, ast.expr]]] = {}
         # task variable name -> the operator's ast.Call node (for source-slicing placeholders)
@@ -543,6 +606,7 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None = None
         # mapped var -> the kwarg names passed to .expand(). Only these fan out; a list-valued
         # .partial() arg is a fixed value and must not be mistaken for the mapped iterable.
         self.expand_kwargs: dict[str, list[str]] = {}
+        self.partial_mapped: set[str] = set()
         # Disambiguates synthetic vars for operators instantiated without an assignment.
         self._bare_operator_counter = 0
         # TaskFlow: function name -> (FunctionDef, decorator dotted-name) for @task-decorated defs.
@@ -572,20 +636,51 @@ def functions(self) -> dict[str, ast.FunctionDef]:
         taskflow = {name: definition for name, (definition, _decorator) in self.taskflow_defs.items()}
         return {**self._functions, **taskflow}
 
+    def functions_for(self, task_var: str) -> dict[str, ast.FunctionDef]:
+        """Returns module functions with a task's lexically resolved callable overlaid."""
+        functions = self.functions()
+        resolved = self._resolved_callables.get(task_var)
+        if resolved is None:
+            return functions
+        name, definition = resolved
+        functions.pop(name, None)
+        if definition is not None:
+            functions[name] = definition
+        return functions
+
     def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
         # A @task- or @task_group-decorated function defines a task / sub-pipeline from its body,
         # which is internal logic rather than DAG structure, so don't descend. @dag marks the
         # DAG-defining function: read its config off the decorator, then descend so the body's task
         # instances / edges are collected.
         if _has_decorator(node, _TASK_DECORATORS) or _has_decorator(node, _TASK_GROUP_DECORATORS):
+            if self._dag_scope_depth:
+                self._claimed_statement_ids.add(id(node))
             return
-        if _has_decorator(node, _DAG_DECORATORS):
+        is_dag_definition = _has_decorator(node, _DAG_DECORATORS)
+        if not is_dag_definition:
+            if self._dag_scope_depth:
+                self._claimed_statement_ids.add(id(node))
+            return
+        if is_dag_definition:
             self.is_taskflow_dag = True
             dag_kwargs = _decorator_kwargs(node.decorator_list, _DAG_DECORATORS)
             self._apply_dag_kwargs(dag_kwargs)
             if self.dag_id is None:
                 self.dag_id = ops.literal_str(dag_kwargs.get("dag_id")) or node.name
-        self.generic_visit(node)
+        self._scope_stack.append(node)
+        if is_dag_definition:
+            self._dag_scope_depth += 1
+        try:
+            for statement in node.body:
+                if is_dag_definition:
+                    self._visit_dag_statement(statement)
+                else:
+                    self.visit(statement)
+        finally:
+            if is_dag_definition:
+                self._dag_scope_depth -= 1
+            self._scope_stack.pop()
 
     def visit_Assign(self, node: ast.Assign) -> None:
         if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name) and isinstance(node.value, ast.Call):
@@ -593,18 +688,23 @@ def visit_Assign(self, node: ast.Assign) -> None:
             if _construct_name(node.value.func, self._aliases) == "DAG":
                 self._read_dag_kwargs(node.value)
                 self.dag_id = self.dag_id or var
+                self._claimed_statement_ids.add(id(node))
                 return
             internal_var = self._new_task_var(var, node.value)
             if self._register_operator_call(node.value, internal_var, binding=var):
+                self._claimed_statement_ids.add(id(node))
                 pass  # a `x = SomeOperator(...)` (optionally .expand()-mapped) instantiation
             elif self._register_helper_factory_call(node.value, internal_var, binding=var):
+                self._claimed_statement_ids.add(id(node))
                 pass
             elif self._register_taskflow_call(node.value, internal_var):
                 self._task_bindings[var] = internal_var
+                self._claimed_statement_ids.add(id(node))
                 pass  # a `x = mytask(...)` TaskFlow invocation, captured with var as its key
             else:
                 if self._register_taskgroup_call(node.value, internal_var):
                     self._task_bindings[var] = internal_var
+                    self._claimed_statement_ids.add(id(node))
                     return
         elif len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
             target = node.targets[0].id
@@ -613,6 +713,7 @@ def visit_Assign(self, node: ast.Assign) -> None:
                 if resolved:
                     self._task_bindings[target] = resolved[0] if len(resolved) == 1 else resolved
                     self._constants.pop(target, None)
+                    self._claimed_statement_ids.add(id(node))
                     return
             value = _safe_static_value(node.value, self._constants)
             if value is not _UNRESOLVED:
@@ -620,6 +721,7 @@ def visit_Assign(self, node: ast.Assign) -> None:
                 self._task_bindings.pop(target, None)
                 if isinstance(value, list):
                     self._list_bindings[target] = []
+                self._claimed_statement_ids.add(id(node))
                 return
         self.generic_visit(node)
 
@@ -651,9 +753,11 @@ def _register_operator_call(self, node: ast.Call, var: str, *, binding: str | No
             isinstance(dag_node, ast.Name) and dag_node.id == self._target_dag_variable
         ):
             return False
-        call = ast.Call(func=ast.Name(id=construct, ctx=ast.Load()), args=[], keywords=[
-            ast.keyword(arg=key, value=value) for key, value in kwargs.items()
-        ])
+        call = ast.Call(
+            func=ast.Name(id=construct, ctx=ast.Load()),
+            args=[],
+            keywords=[ast.keyword(arg=key, value=value) for key, value in kwargs.items()],
+        )
         ast.copy_location(call, node)
         task_id = ops.literal_str(kwargs.get("task_id")) or ops.literal_str(kwargs.get("group_id")) or var
         self.operators[var] = (task_id, construct, kwargs)
@@ -667,26 +771,28 @@ def _register_operator_call(self, node: ast.Call, var: str, *, binding: str | No
             call=call,
             span=_span(node),
         )
+        self._claimed_task_call_ids.add(id(node))
+        callable_node = kwargs.get("python_callable")
+        if isinstance(callable_node, ast.Name):
+            self._resolved_callables[var] = (
+                callable_node.id,
+                self._resolve_lexical_function(callable_node.id, node),
+            )
         if mapped is not None:
             self.mapped.add(var)
             self.expand_kwargs[var] = mapped[1]
+            if mapped[2]:
+                self.partial_mapped.add(var)
         if self._group_stack:
             self.groups[var] = "__".join(self._group_stack)
         return True
 
     def _register_helper_factory_call(self, node: ast.Call, var: str, *, binding: str) -> bool:
         """Expands the deliberately narrow single-return operator factory shape."""
-        if not isinstance(node.func, ast.Name):
-            return False
-        helper = self._functions.get(node.func.id)
-        if helper is None or helper.decorator_list or helper.args.vararg or helper.args.kwarg:
-            return False
-        body = list(helper.body)
-        if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant):
-            if isinstance(body[0].value.value, str):
-                body = body[1:]
-        if len(body) != 1 or not isinstance(body[0], ast.Return) or not isinstance(body[0].value, ast.Call):
+        helper_return = self._helper_factory_return(node)
+        if helper_return is None:
             return False
+        helper, return_call = helper_return
         parameters = [*helper.args.posonlyargs, *helper.args.args, *helper.args.kwonlyargs]
         if len(node.args) > len(parameters) or any(keyword.arg is None for keyword in node.keywords):
             return False
@@ -721,8 +827,52 @@ def _register_helper_factory_call(self, node: ast.Call, var: str, *, binding: st
                 if value is _UNRESOLVED:
                     return False
                 constants[name] = value
-        factory_call = _bind_constants(body[0].value, constants)
-        return isinstance(factory_call, ast.Call) and self._register_operator_call(factory_call, var, binding=binding)
+        factory_call = _bind_constants(return_call, constants)
+        registered = isinstance(factory_call, ast.Call) and self._register_operator_call(
+            factory_call, var, binding=binding
+        )
+        if registered:
+            self._claimed_task_call_ids.add(id(node))
+            self.helper_expansions.append(
+                {
+                    "code": "helper_factory_expanded",
+                    "capture_id": var,
+                    "helper": helper.name,
+                    "helper_line": helper.lineno,
+                    "invocation_line": getattr(node, "lineno", 0),
+                }
+            )
+        return registered
+
+    def _helper_factory_return(self, node: ast.Call) -> tuple[ast.FunctionDef, ast.Call] | None:
+        """Returns the operator call from a supported single-return helper invocation."""
+        if not isinstance(node.func, ast.Name):
+            return None
+        helper = self._resolve_lexical_function(node.func.id, node)
+        if helper is None or helper.decorator_list or helper.args.vararg or helper.args.kwarg:
+            return None
+        body = list(helper.body)
+        if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant):
+            if isinstance(body[0].value.value, str):
+                body = body[1:]
+        if len(body) != 1 or not isinstance(body[0], ast.Return) or not isinstance(body[0].value, ast.Call):
+            return None
+        if _direct_operator_call(body[0].value, self._aliases) is None:
+            return None
+        return helper, body[0].value
+
+    def _resolve_lexical_function(self, name: str, reference: ast.AST) -> ast.FunctionDef | None:
+        """Resolves a function name by lexical scope and source-order binding semantics."""
+        line = getattr(reference, "lineno", 0)
+        for scope in reversed(self._scope_stack):
+            events = self._lexical_functions.get(id(scope), {}).get(name, [])
+            visible = [event for event in events if event[0] <= line]
+            if visible:
+                _event_line, conditional, definition = visible[-1]
+                return None if conditional else definition
+            if events and isinstance(scope, ast.FunctionDef):
+                return None
+        return None
 
     def _register_bare_operator_call(self, node: ast.Call) -> str | None:
         """Registers an operator instantiated without an assignment, keyed by a synthetic var.
@@ -795,6 +945,7 @@ def _register_taskflow_call(self, call: ast.Call, var: str) -> bool:
         task = _TaskFlowTask(task_id=override_id or var, def_name=def_name, decorator=decorator)
         self.taskflow_tasks[var] = task
         self.calls[var] = call
+        self._claimed_task_call_ids.add(id(call))
         if mapped:
             self.mapped.add(var)
             # ``.expand(param=)`` args live on the outer call; capture the single mapped
@@ -807,7 +958,7 @@ def _register_taskflow_call(self, call: ast.Call, var: str) -> bool:
             for mapped_arg in _mapping_chain_args(call):
                 dep = self._resolve_taskflow_arg(mapped_arg)
                 if dep is not None and dep != var:
-                    self.edges.append((dep, var))
+                    self._add_edges([dep], [var], call)
         if self._group_stack:
             self.groups[var] = "__".join(self._group_stack)
         if mapped:
@@ -817,7 +968,7 @@ def _register_taskflow_call(self, call: ast.Call, var: str) -> bool:
             dep = self._resolve_taskflow_arg(arg)
             if dep is not None:
                 task.positional_deps[index] = dep
-                self.edges.append((dep, var))
+                self._add_edges([dep], [var], call)
             else:
                 value = _literal_argument_source(arg)
                 if value is None:
@@ -831,7 +982,7 @@ def _register_taskflow_call(self, call: ast.Call, var: str) -> bool:
             dep = self._resolve_taskflow_arg(kw.value)
             if dep is not None:
                 task.keyword_deps[kw.arg] = dep
-                self.edges.append((dep, var))
+                self._add_edges([dep], [var], call)
             else:
                 value = _literal_argument_source(kw.value)
                 if value is None:
@@ -887,6 +1038,7 @@ def _register_taskgroup_call(self, call: ast.Call, var: str | None) -> bool:
             self._taskgroup_counter += 1
             var = f"{def_name}__tg{self._taskgroup_counter}"
         self.taskgroup_calls[var] = (var, def_name, mapped)
+        self._claimed_task_call_ids.add(id(call))
         if self._group_stack:
             self.groups[var] = "__".join(self._group_stack)
         return True
@@ -913,12 +1065,14 @@ def _resolve_taskflow_arg(self, arg: ast.expr) -> str | None:
 
     def visit_With(self, node: ast.With) -> None:
         pushed_group = False
+        opens_dag_scope = False
         for item in node.items:
             call = item.context_expr
             if isinstance(call, ast.Call):
                 construct = _construct_name(call.func, self._aliases)
                 if construct == "DAG":
                     self._read_dag_kwargs(call)
+                    opens_dag_scope = True
                 elif construct == "TaskGroup":
                     # `with TaskGroup("etl") as tg:` — namespace the member tasks by group id.
                     kwargs = {kw.arg: kw.value for kw in call.keywords if kw.arg}
@@ -941,9 +1095,103 @@ def visit_With(self, node: ast.With) -> None:
                         task_id = ops.literal_str(kwargs.get("group_id")) or var
                         self.operators[var] = (task_id, construct, kwargs)
                         self.calls[var] = call
-        self.generic_visit(node)
+        if opens_dag_scope:
+            self._dag_scope_depth += 1
+            try:
+                for statement in node.body:
+                    self._visit_dag_statement(statement)
+            finally:
+                self._dag_scope_depth -= 1
+        elif self._dag_scope_depth:
+            for statement in node.body:
+                self._visit_dag_statement(statement)
+        else:
+            self.generic_visit(node)
         if pushed_group:
             self._group_stack.pop()
+        self._claimed_statement_ids.add(id(node))
+
+    def _visit_dag_statement(self, statement: ast.stmt) -> None:
+        """Visits one DAG-body statement and records any unclaimed structural source."""
+        unclaimed_calls_before = len(self.unclaimed_task_calls)
+        unresolved_before = len(self.unresolved_constructs)
+        self.visit(statement)
+        if id(statement) in self._claimed_statement_ids:
+            return
+        if (
+            len(self.unclaimed_task_calls) > unclaimed_calls_before
+            or len(self.unresolved_constructs) > unresolved_before
+        ):
+            return
+        if isinstance(statement, (ast.Import, ast.ImportFrom, ast.Pass, ast.Return)):
+            self._claimed_statement_ids.add(id(statement))
+            return
+        if isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Constant):
+            self._claimed_statement_ids.add(id(statement))
+            return
+        self.unclaimed_statements.append(statement)
+
+    def visit_Call(self, node: ast.Call) -> None:
+        """Fails closed when a task-producing call in a DAG scope was not captured."""
+        in_selected_assigned_dag = False
+        is_assigned_task_factory = False
+        if self._target_dag_variable is not None:
+            direct = _direct_operator_call(node, self._aliases)
+            mapped = None if direct is not None else _mapped_operator_call(node, self._aliases)
+            operator_call = direct or (mapped[0] if mapped is not None else None)
+            if operator_call is not None:
+                dag_argument = next((kw.value for kw in operator_call.keywords if kw.arg == "dag"), None)
+                in_selected_assigned_dag = (
+                    isinstance(dag_argument, ast.Name) and dag_argument.id == self._target_dag_variable
+                )
+            is_assigned_task_factory = self._helper_targets_assigned_dag(node)
+            in_selected_assigned_dag = in_selected_assigned_dag or is_assigned_task_factory
+        if (self._dag_scope_depth or in_selected_assigned_dag) and id(node) not in self._claimed_task_call_ids:
+            is_operator = _direct_operator_call(node, self._aliases) is not None
+            is_mapped_operator = _mapped_operator_call(node, self._aliases) is not None
+            is_taskflow = self._taskflow_def_name(node)[0] is not None
+            is_taskgroup = any(
+                isinstance(candidate, ast.Name) and candidate.id in self.taskgroup_defs
+                for candidate in ast.walk(node.func)
+            )
+            if (
+                is_operator
+                or is_mapped_operator
+                or is_taskflow
+                or is_taskgroup
+                or is_assigned_task_factory
+                or self._helper_factory_return(node)
+            ):
+                self.unclaimed_task_calls.append(node)
+        self.generic_visit(node)
+
+    def _helper_targets_assigned_dag(self, call: ast.Call) -> bool:
+        """Returns whether a local helper can construct a task for the selected assigned DAG."""
+        if self._target_dag_variable is None or not isinstance(call.func, ast.Name):
+            return False
+        helper = self._resolve_lexical_function(call.func.id, call)
+        if helper is None:
+            return False
+        parameters = [*helper.args.posonlyargs, *helper.args.args, *helper.args.kwonlyargs]
+        bound: dict[str, ast.expr] = {parameter.arg: argument for parameter, argument in zip(parameters, call.args)}
+        bound.update({keyword.arg: keyword.value for keyword in call.keywords if keyword.arg})
+        for candidate in ast.walk(helper):
+            if not isinstance(candidate, ast.Call):
+                continue
+            direct = _direct_operator_call(candidate, self._aliases)
+            mapped = None if direct is not None else _mapped_operator_call(candidate, self._aliases)
+            operator_call = direct or (mapped[0] if mapped is not None else None)
+            if operator_call is None:
+                continue
+            dag_argument = next((keyword.value for keyword in operator_call.keywords if keyword.arg == "dag"), None)
+            if not isinstance(dag_argument, ast.Name):
+                continue
+            if dag_argument.id == self._target_dag_variable:
+                return True
+            bound_argument = bound.get(dag_argument.id)
+            if isinstance(bound_argument, ast.Name) and bound_argument.id == self._target_dag_variable:
+                return True
+        return False
 
     def _read_dag_kwargs(self, call: ast.Call) -> None:
         kwargs = {kw.arg: _bind_constants(kw.value, self._constants) for kw in call.keywords if kw.arg}
@@ -951,6 +1199,7 @@ def _read_dag_kwargs(self, call: ast.Call) -> None:
         self._apply_dag_kwargs(kwargs)
 
     def _apply_dag_kwargs(self, kwargs: dict[str, ast.expr]) -> None:
+        self.captured_dag_settings.update(kwargs)
         self.schedule_node = kwargs.get("schedule_interval") or kwargs.get("schedule")
         self.schedule_interval = ops.literal_str(kwargs.get("schedule_interval")) or ops.literal_str(
             kwargs.get("schedule")
@@ -965,6 +1214,7 @@ def _apply_dag_kwargs(self, kwargs: dict[str, ast.expr]) -> None:
                 for key, val in zip(default_args.keys, default_args.values)
                 if isinstance(key, ast.Constant) and isinstance(key.value, str)
             }
+            self.captured_dag_settings.update(f"default_args.{name}" for name in self.default_args)
         # params={...} supplies DAG parameter defaults; each value is a literal or a Param(default=...).
         params = kwargs.get("params")
         if isinstance(params, ast.Dict):
@@ -978,13 +1228,17 @@ def visit_Expr(self, node: ast.Expr) -> None:
         #   - method calls: `a.set_upstream(b)` / `a.set_downstream([b, c])`
         value = node.value
         if isinstance(value, ast.BinOp) and isinstance(value.op, (ast.RShift, ast.LShift)):
+            before = len(self.edge_captures)
             self._collect_shift_chain(value)
+            if len(self.edge_captures) > before:
+                self._claimed_statement_ids.add(id(node))
         elif isinstance(value, ast.Call):
             call_name = _construct_name(value.func, self._aliases)
             if call_name == "chain":
                 positions = [self._resolve_task_names(argument) for argument in value.args]
                 for left, right in zip(positions, positions[1:]):
                     self._add_edges(left, right, value)
+                self._claimed_statement_ids.add(id(node))
                 return
             if call_name == "cross_downstream" and len(value.args) >= 2:
                 self._add_edges(
@@ -992,6 +1246,7 @@ def visit_Expr(self, node: ast.Expr) -> None:
                     self._resolve_task_names(value.args[1]),
                     value,
                 )
+                self._claimed_statement_ids.add(id(node))
                 return
             if isinstance(value.func, ast.Attribute) and value.func.attr == "append" and value.args:
                 owner = value.func.value
@@ -1001,9 +1256,13 @@ def visit_Expr(self, node: ast.Expr) -> None:
                         internal = self._register_bare_operator_call(appended)
                         if internal is not None:
                             self._list_bindings.setdefault(owner.id, []).append(internal)
+                            self._claimed_statement_ids.add(id(node))
                             return
-                    self._list_bindings.setdefault(owner.id, []).extend(self._resolve_task_names(appended))
-                    return
+                    resolved = self._resolve_task_names(appended)
+                    if resolved:
+                        self._list_bindings.setdefault(owner.id, []).extend(resolved)
+                        self._claimed_statement_ids.add(id(node))
+                        return
             # A bare TaskFlow call (`extract()` with no assignment) is a task instance keyed by its
             # def name; otherwise it may be a set_upstream/set_downstream dependency call.
             def_name, _mapped, _override = self._taskflow_def_name(value)
@@ -1013,16 +1272,24 @@ def visit_Expr(self, node: ast.Expr) -> None:
                     self._taskflow_counter += 1
                     task_var = f"{def_name}__tf{self._taskflow_counter}"
                 self._register_taskflow_call(value, task_var)
+                self._claimed_statement_ids.add(id(node))
             elif self._register_bare_operator_call(value) is not None:
+                self._claimed_statement_ids.add(id(node))
                 pass  # a bare `SomeOperator(task_id=...)` statement -- registered under a synthetic var
-            elif not self._register_taskgroup_call(value, None):
+            elif self._register_taskgroup_call(value, None):
+                self._claimed_statement_ids.add(id(node))
+            else:
+                before = len(self.edge_captures)
                 self._collect_set_dependency(value)
+                if len(self.edge_captures) > before:
+                    self._claimed_statement_ids.add(id(node))
         self.generic_visit(node)
 
     def visit_For(self, node: ast.For) -> None:
         """Executes bounded literal/range loops with Python name rebinding semantics."""
         if not isinstance(node.target, ast.Name):
             self.unresolved_constructs.append(("dynamic_loop_target", node))
+            self._claimed_statement_ids.add(id(node))
             return
         items = _static_iteration_nodes(node.iter, self._constants)
         if items is None:
@@ -1032,6 +1299,7 @@ def visit_For(self, node: ast.For) -> None:
                 items = list(node.iter.elts)
             else:
                 self.unresolved_constructs.append(("dynamic_loop_iterable", node))
+                self._claimed_statement_ids.add(id(node))
                 return
         for item in items:
             resolved_tasks = self._resolve_task_names(item)
@@ -1042,13 +1310,15 @@ def visit_For(self, node: ast.For) -> None:
                 value = _safe_static_value(item, self._constants)
                 if value is _UNRESOLVED:
                     self.unresolved_constructs.append(("dynamic_loop_value", item))
+                    self._claimed_statement_ids.add(id(node))
                     return
                 self._constants[node.target.id] = value
                 self._task_bindings.pop(node.target.id, None)
             for statement in node.body:
-                self.visit(statement)
+                self._visit_dag_statement(statement) if self._dag_scope_depth else self.visit(statement)
         for statement in node.orelse:
-            self.visit(statement)
+            self._visit_dag_statement(statement) if self._dag_scope_depth else self.visit(statement)
+        self._claimed_statement_ids.add(id(node))
 
     def visit_If(self, node: ast.If) -> None:
         """Follows a statically decidable branch; records ambiguous control flow explicitly."""
@@ -1057,10 +1327,12 @@ def visit_If(self, node: ast.If) -> None:
             value = True
         if value is _UNRESOLVED:
             self.unresolved_constructs.append(("ambiguous_condition", node))
+            self._claimed_statement_ids.add(id(node))
             return
         branch = node.body if bool(value) else node.orelse
         for statement in branch:
-            self.visit(statement)
+            self._visit_dag_statement(statement) if self._dag_scope_depth else self.visit(statement)
+        self._claimed_statement_ids.add(id(node))
 
     def _collect_shift_chain(self, binop: ast.BinOp) -> None:
         self._collect_shift_expression(binop)
@@ -1257,7 +1529,7 @@ def _direct_operator_call(node: ast.Call, aliases: dict[str, str] | None = None)
 
 def _mapped_operator_call(
     node: ast.Call, aliases: dict[str, str] | None = None
-) -> tuple[ast.Call, list[str]] | None:
+) -> tuple[ast.Call, list[str], bool] | None:
     """Returns the underlying operator call for a dynamic-mapping ``.expand(...)`` chain.
 
     Handles ``Op(...).expand(...)`` and ``Op.partial(...).expand(...)``. Returns
@@ -1287,7 +1559,7 @@ def _mapped_operator_call(
         args=[],
         keywords=list(inner.keywords) + list(node.keywords),
     )
-    return merged, [kw.arg for kw in node.keywords if kw.arg]
+    return merged, [kw.arg for kw in node.keywords if kw.arg], isinstance(inner.func, ast.Attribute)
 
 
 def _is_task_construct(name: str) -> bool:
@@ -1342,13 +1614,26 @@ def load_airflow_dag(dag_path: Path, *, dbt_mode: str = "static") -> Pipeline:
     return pipelines[0]
 
 
-def load_airflow_dags(dag_path: Path, *, dbt_mode: str = "static") -> list[Pipeline]:
+def load_airflow_dags(
+    dag_path: Path,
+    *,
+    dbt_mode: str = "static",
+    source_file: str | None = None,
+) -> list[Pipeline]:
     """Parses every independently declared Airflow DAG in a Python file."""
     source = Path(dag_path).read_text(encoding="utf-8")
     module = _expand_top_level_loops(ast.parse(source))
     declarations = _top_level_dag_declarations(module)
     if not declarations:
-        return [_load_airflow_module(dag_path, source, module, dbt_mode=dbt_mode)]
+        return [
+            _load_airflow_module(
+                dag_path,
+                source,
+                module,
+                dbt_mode=dbt_mode,
+                source_file=source_file or dag_path.name,
+            )
+        ]
     return [
         _load_airflow_module(
             dag_path,
@@ -1356,6 +1641,7 @@ def load_airflow_dags(dag_path: Path, *, dbt_mode: str = "static") -> list[Pipel
             _module_for_dag(module, declaration),
             dbt_mode=dbt_mode,
             target_dag_variable=declaration.variable,
+            source_file=source_file or dag_path.name,
         )
         for declaration in declarations
     ]
@@ -1371,8 +1657,7 @@ def _top_level_dag_declarations(module: ast.Module) -> list[DagDeclaration]:
         if isinstance(node, ast.FunctionDef) and _has_decorator(node, _DAG_DECORATORS):
             is_dag = True
         elif isinstance(node, ast.With) and any(
-            isinstance(item.context_expr, ast.Call)
-            and _construct_name(item.context_expr.func, aliases) == "DAG"
+            isinstance(item.context_expr, ast.Call) and _construct_name(item.context_expr.func, aliases) == "DAG"
             for item in node.items
         ):
             is_dag = True
@@ -1412,6 +1697,7 @@ def _load_airflow_module(
     *,
     dbt_mode: str = "static",
     target_dag_variable: str | None = None,
+    source_file: str | None = None,
 ) -> Pipeline:
     """Parses one isolated DAG declaration into a flowx Pipeline IR.
 
@@ -1428,6 +1714,7 @@ def _load_airflow_module(
         sensors remain explicit placeholders; unmapped operators become a
         PlaceholderActivity.
     """
+    audit = source_audit.audit_module(module, target_dag_variable=target_dag_variable)
     visitor = _DagVisitor(module, target_dag_variable=target_dag_variable)
     visitor.visit(module)
     functions = visitor.functions()
@@ -1478,13 +1765,21 @@ def _task_key(var: str, task_id: str) -> str:
 
     # Dummy/Empty operators are structural and can be removed after dependency rewiring.
     dropped = {var for var, (_, op, _) in visitor.operators.items() if op in ops.DUMMY_OPERATORS}
+    sensor_lift_proof: dict[str, Any] | None = None
     if not has_schedule:
-        trigger_var = _root_trigger_sensor(visitor.operators, upstreams)
-        if trigger_var is not None:
+        trigger_candidate = _root_trigger_sensor(visitor.operators, upstreams, set(var_task_ids))
+        if trigger_candidate is not None:
+            trigger_var, covered_tasks = trigger_candidate
             trigger = _trigger_from_sensor(*visitor.operators[trigger_var][1:])
             if trigger is not None:
                 schedule = trigger
                 dropped.add(trigger_var)
+                sensor_lift_proof = {
+                    "code": "sensor_lift_dominates_dag",
+                    "capture_id": trigger_var,
+                    "task_key": var_to_task_key[trigger_var],
+                    "covered_capture_ids": sorted(covered_tasks),
+                }
     upstreams = _rewire_dropped(upstreams, dropped)
 
     # Collapse all dbt CLI operators over the one project into a single DbtFactoryActivity emitted at
@@ -1543,13 +1838,25 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
         return dbt_key_remap.get(key, key)
 
     tasks: list[Activity] = []
+    semantic_findings: list[dict[str, Any]] = []
+    argument_proofs = [
+        {
+            "code": "operator_arguments_classified",
+            "capture_id": var,
+            "task_key": var_to_task_key[var],
+            "operator": operator,
+            "arguments": ops.argument_classification(operator, kwargs),
+        }
+        for var, (_task_id, operator, kwargs) in visitor.operators.items()
+    ]
     referenced_params: set[str] = set()
     emitted_dbt = False
     for var, (task_id, operator, kwargs) in visitor.operators.items():
         if var in dropped:
             continue
         task_key = var_to_task_key[var]
-        outcome = templating.trigger_rule_outcome(kwargs)
+        trigger_mapping = templating.trigger_rule_mapping(kwargs)
+        outcome = trigger_mapping.outcome
         # Remap dbt-chain upstreams to the single factory key and drop self-edges (a dbt op
         # depending on another dbt op in the same collapsed chain).
         dep_keys = {_dep(u, outcome) for u in upstreams[var]}
@@ -1573,9 +1880,7 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
             emitted_dbt = True
             # The factory gates on every dbt op's external upstreams (not just the first op's), minus
             # any that are downstream of the factory itself (a sandwiched task, which would cycle).
-            factory_depends_on = [
-                Dependency(task_key=k, outcome=outcome) for k in sorted(factory_dep_keys)
-            ] or None
+            factory_depends_on = [Dependency(task_key=k, outcome=outcome) for k in sorted(factory_dep_keys)] or None
             dbt_kwargs = [visitor.operators[v][2] for v in dbt_vars]
             tasks.append(
                 _build_dbt_factory(
@@ -1596,7 +1901,7 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
             task_key=task_key,
             operator=operator,
             kwargs=kwargs,
-            functions=functions,
+            functions=visitor.functions_for(var),
             source=source,
             call_source=call_source,
             default_args=visitor.default_args,
@@ -1604,6 +1909,53 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
         builder = ops.OPERATOR_REGISTRY.get(operator, ops.build_placeholder)
         activity = builder(ctx)
         activity.depends_on = depends_on
+        if trigger_mapping.status == "unsupported":
+            activity = ops.build_placeholder_with_comment(
+                ctx,
+                f"Airflow trigger_rule {trigger_mapping.rule!r} is unsupported. {trigger_mapping.message}",
+            )
+            activity.depends_on = depends_on
+            semantic_findings.append(
+                _semantic_finding(
+                    source_file or dag_path.name,
+                    visitor.calls.get(var),
+                    code="unsupported_trigger_rule",
+                    message=(f"Task {task_id!r} uses trigger_rule {trigger_mapping.rule!r}; {trigger_mapping.message}"),
+                    task_key=task_key,
+                )
+            )
+        elif trigger_mapping.status == "approximate":
+            semantic_findings.append(
+                _semantic_finding(
+                    source_file or dag_path.name,
+                    visitor.calls.get(var),
+                    code="approximated_trigger_rule",
+                    message=(
+                        f"Task {task_id!r} maps trigger_rule {trigger_mapping.rule!r} to "
+                        f"{trigger_mapping.outcome}. {trigger_mapping.message}"
+                    ),
+                    task_key=task_key,
+                )
+            )
+        unconsumed = ops.unconsumed_kwargs(operator, kwargs)
+        if unconsumed:
+            names = ", ".join(sorted(unconsumed))
+            activity = ops.build_placeholder_with_comment(
+                ctx,
+                f"Airflow {operator} argument(s) {names} are not represented by the Databricks task; "
+                "translate them explicitly.",
+            )
+            activity.depends_on = depends_on
+            semantic_findings.append(
+                _semantic_finding(
+                    source_file or dag_path.name,
+                    visitor.calls.get(var),
+                    code="unconsumed_operator_arguments",
+                    message=f"Task {task_id!r} has unconsumed operator argument(s): {names}.",
+                    task_key=task_key,
+                    arguments=sorted(unconsumed),
+                )
+            )
         # Stamp DAG/task retry + timeout policy (per-task kwargs override default_args).
         policy = templating.retry_policy(visitor.default_args, kwargs)
         activity.max_retries = policy.get("max_retries")
@@ -1620,14 +1972,43 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
                 "translate the value manually.",
             )
             activity.depends_on = depends_on
+            semantic_findings.append(
+                _semantic_finding(
+                    source_file or dag_path.name,
+                    visitor.calls.get(var),
+                    code="unresolved_airflow_template",
+                    message=f"Task {task_id!r} contains unresolved Airflow template expression(s): {expressions}.",
+                    task_key=task_key,
+                    expressions=sorted(unresolved_templates),
+                )
+            )
 
         if var in visitor.mapped:
-            # Dynamic mapping (.expand()) -> a for_each_task iterating the mapped operator.
-            tasks.append(
-                _wrap_in_for_each(
-                    activity, task_id, task_key, depends_on, kwargs, visitor.expand_kwargs.get(var) or []
+            mapped_names = visitor.expand_kwargs.get(var) or []
+            partial_note = (
+                " The mapping also contains .partial() fixed arguments." if var in visitor.partial_mapped else ""
+            )
+            activity = ops.build_placeholder_with_comment(
+                ctx,
+                "Classic Airflow dynamic mapping cannot be emitted until every mapped argument is "
+                f"bound into the inner task ({', '.join(mapped_names) or 'unknown mapping'}).{partial_note}",
+            )
+            activity.depends_on = depends_on
+            semantic_findings.append(
+                _semantic_finding(
+                    source_file or dag_path.name,
+                    visitor.calls.get(var),
+                    code="classic_mapping_arguments_unbound",
+                    message=(
+                        f"Task {task_id!r} maps argument(s) {', '.join(mapped_names) or ''}, "
+                        "but the generated inner task cannot bind them safely."
+                    ),
+                    task_key=task_key,
+                    arguments=mapped_names,
+                    has_partial=var in visitor.partial_mapped,
                 )
             )
+            tasks.append(_wrap_in_for_each(activity, task_id, task_key, depends_on, kwargs, mapped_names))
         else:
             tasks.append(activity)
 
@@ -1718,13 +2099,535 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
         # Airflow catchup=True has no DABs schedule setting; it maps to running a native Databricks
         # backfill, which overrides the run_date job parameter with {{backfill.iso_date}} per window.
         tags["airflow_catchup"] = "true"
-    return Pipeline(
+    expected_ir_edges = {(dependency.task_key, task.task_key) for task in tasks for dependency in task.depends_on or []}
+    pipeline = Pipeline(
         name=visitor.dag_id or Path(dag_path).stem,
         tasks=tasks,
         parameters=parameters,
         schedule=schedule,
         tags=tags,
     )
+    return _reconcile_pipeline(
+        pipeline,
+        audit=audit,
+        visitor=visitor,
+        source_file=source_file or dag_path.name,
+        var_to_task_key=var_to_task_key,
+        dropped=dropped,
+        dbt_vars=dbt_vars,
+        semantic_findings=semantic_findings,
+        sensor_lift_proof=sensor_lift_proof,
+        argument_proofs=argument_proofs,
+        expected_ir_edges=expected_ir_edges,
+    )
+
+
+_SUPPORTED_DAG_SETTINGS = frozenset(
+    {
+        "dag_id",
+        "schedule",
+        "schedule_interval",
+        "start_date",
+        "timezone",
+        "catchup",
+        "default_args",
+        "params",
+        "default_args.retries",
+        "default_args.retry_delay",
+        "default_args.execution_timeout",
+    }
+)
+
+
+def _semantic_finding(
+    source_file: str,
+    node: ast.AST | None,
+    *,
+    code: str,
+    message: str,
+    task_key: str,
+    **details: Any,
+) -> dict[str, Any]:
+    """Builds a stable gap finding for a captured task-level semantic limitation."""
+    candidate = source_audit.AuditCandidate(
+        kind="task_semantics",
+        code=code,
+        line=getattr(node, "lineno", 0),
+        column=getattr(node, "col_offset", 0),
+        occurrence=1,
+        end_line=getattr(node, "end_lineno", 0),
+        end_column=getattr(node, "end_col_offset", 0),
+        details={"task_key": task_key, **details},
+    )
+    return source_audit.finding(
+        source_file=source_file,
+        code=code,
+        severity="gap",
+        message=message,
+        candidate=candidate,
+    )
+
+
+def _iter_placeholders(tasks: list[Activity]) -> list[PlaceholderActivity]:
+    """Returns placeholders in top-level and Airflow-generated for_each tasks."""
+    placeholders: list[PlaceholderActivity] = []
+    for task in tasks:
+        if isinstance(task, PlaceholderActivity):
+            placeholders.append(task)
+        if isinstance(task, ForEachActivity):
+            placeholders.extend(_iter_placeholders(task.inner_activities))
+    return placeholders
+
+
+def _reconcile_pipeline(
+    pipeline: Pipeline,
+    *,
+    audit: source_audit.SourceAudit,
+    visitor: _DagVisitor,
+    source_file: str,
+    var_to_task_key: dict[str, str],
+    dropped: set[str],
+    dbt_vars: list[str],
+    semantic_findings: list[dict[str, Any]],
+    sensor_lift_proof: dict[str, Any] | None,
+    argument_proofs: list[dict[str, Any]],
+    expected_ir_edges: set[tuple[str, str]],
+) -> Pipeline:
+    """Reconciles an independent source audit with captured graph and emitted IR."""
+    findings: list[dict[str, Any]] = list(semantic_findings)
+    transformations: list[dict[str, Any]] = list(argument_proofs)
+    transformations.extend(visitor.helper_expansions)
+    transformations.extend(
+        {
+            "code": "edge_captured",
+            "upstream_capture_id": edge.upstream_id,
+            "downstream_capture_id": edge.downstream_id,
+            "upstream_task_key": var_to_task_key.get(edge.upstream_id),
+            "downstream_task_key": var_to_task_key.get(edge.downstream_id),
+            "source_span": {
+                "line": edge.span.line,
+                "column": edge.span.column,
+                "end_line": edge.span.end_line,
+                "end_column": edge.span.end_column,
+            },
+        }
+        for edge in visitor.edge_captures
+    )
+    if sensor_lift_proof is not None:
+        transformations.append(sensor_lift_proof)
+    captured_task_count = len(visitor.operators) + len(visitor.taskflow_tasks) + len(visitor.taskgroup_calls)
+
+    unresolved = list(audit.unresolved)
+    for code, node in visitor.unresolved_constructs:
+        if not any(
+            candidate.line == getattr(node, "lineno", 0) and candidate.column == getattr(node, "col_offset", 0)
+            for candidate in unresolved
+        ):
+            unresolved.append(
+                source_audit.AuditCandidate(
+                    kind="unresolved",
+                    code=code,
+                    line=getattr(node, "lineno", 0),
+                    column=getattr(node, "col_offset", 0),
+                    occurrence=1,
+                    end_line=getattr(node, "end_lineno", 0),
+                    end_column=getattr(node, "end_col_offset", 0),
+                    details={"expression": ast.unparse(node)},
+                )
+            )
+
+    helper_claims = {
+        ("helper_factory_task", int(item["invocation_line"]), str(item["helper"])): 1
+        for item in visitor.helper_expansions
+    }
+    helper_capture_ids = {str(item["capture_id"]) for item in visitor.helper_expansions}
+    capture_claims: Counter[tuple[str, int, str]] = Counter(helper_claims)
+    for capture in visitor.task_captures.values():
+        if capture.capture_id not in helper_capture_ids:
+            capture_claims[("operator_task", capture.span.line, capture.operator)] += 1
+    for var, taskflow_task in visitor.taskflow_tasks.items():
+        call = visitor.calls.get(var)
+        capture_claims[("taskflow_task", getattr(call, "lineno", 0), taskflow_task.def_name)] += 1
+
+    unmatched_audit_tasks: list[source_audit.AuditCandidate] = []
+    for candidate in audit.tasks:
+        discriminator = str(
+            candidate.details.get("operator")
+            or candidate.details.get("helper")
+            or candidate.details.get("callable")
+            or ""
+        )
+        key = (candidate.code, candidate.line, discriminator)
+        if capture_claims[key]:
+            capture_claims[key] -= 1
+        else:
+            unmatched_audit_tasks.append(candidate)
+
+    for candidate in unmatched_audit_tasks:
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="task_capture_mismatch",
+                severity="failed",
+                message="An independently audited Airflow task candidate was not claimed by the capture pass.",
+                candidate=candidate,
+            )
+        )
+    for call in visitor.unclaimed_task_calls:
+        candidate = source_audit.AuditCandidate(
+            kind="task",
+            code="unclaimed_dag_task",
+            line=getattr(call, "lineno", 0),
+            column=getattr(call, "col_offset", 0),
+            occurrence=1,
+            end_line=getattr(call, "end_lineno", 0),
+            end_column=getattr(call, "end_col_offset", 0),
+            details={"expression": ast.unparse(call)},
+        )
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="unclaimed_dag_task",
+                severity="failed",
+                message="A task-producing call in the DAG body was not claimed by the capture pass.",
+                candidate=candidate,
+            )
+        )
+    for statement in visitor.unclaimed_statements:
+        candidate = source_audit.AuditCandidate(
+            kind="statement",
+            code="unclaimed_dag_statement",
+            line=getattr(statement, "lineno", 0),
+            column=getattr(statement, "col_offset", 0),
+            occurrence=1,
+            end_line=getattr(statement, "end_lineno", 0),
+            end_column=getattr(statement, "end_col_offset", 0),
+            details={"expression": ast.unparse(statement)},
+        )
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="unclaimed_dag_statement",
+                severity="failed",
+                message="A DAG-body statement was not classified by the static capture pass.",
+                candidate=candidate,
+            )
+        )
+
+    if len(audit.edges) != len(visitor.edge_captures):
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="edge_capture_mismatch",
+                severity="failed",
+                message=(
+                    f"Source audit found {len(audit.edges)} dependency edge(s), but capture produced "
+                    f"{len(visitor.edge_captures)}."
+                ),
+                details={"audited": len(audit.edges), "captured": len(visitor.edge_captures)},
+            )
+        )
+    comparable_audit_edges = [
+        candidate
+        for candidate in audit.edges
+        if candidate.details.get("syntax") != "taskflow_data"
+        and candidate.details.get("upstream")
+        and candidate.details.get("downstream")
+    ]
+    comparable_spans = {
+        (candidate.line, candidate.column, candidate.end_line, candidate.end_column)
+        for candidate in comparable_audit_edges
+    }
+
+    def source_reference(capture_id: str) -> str:
+        capture = visitor.task_captures.get(capture_id)
+        return capture.variable if capture is not None else capture_id.split("__L", 1)[0]
+
+    audited_edge_identities = sorted(
+        (str(candidate.details["upstream"]), str(candidate.details["downstream"]))
+        for candidate in comparable_audit_edges
+    )
+    captured_edge_identities = sorted(
+        (source_reference(edge.upstream_id), source_reference(edge.downstream_id))
+        for edge in visitor.edge_captures
+        if (edge.span.line, edge.span.column, edge.span.end_line, edge.span.end_column) in comparable_spans
+    )
+    if audited_edge_identities != captured_edge_identities:
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="edge_identity_mismatch",
+                severity="failed",
+                message="Captured Airflow dependency endpoints do not match the audited source endpoints.",
+                details={
+                    "audited_edges": [list(edge) for edge in audited_edge_identities],
+                    "captured_edges": [list(edge) for edge in captured_edge_identities],
+                },
+            )
+        )
+
+    emitted_ir_edges = {
+        (dependency.task_key, task.task_key) for task in pipeline.tasks for dependency in task.depends_on or []
+    }
+    missing_ir_edges = sorted(expected_ir_edges - emitted_ir_edges)
+    unexpected_ir_edges = sorted(emitted_ir_edges - expected_ir_edges)
+    if missing_ir_edges:
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="captured_edge_not_emitted",
+                severity="failed",
+                message="Captured dependency edge(s) were not emitted to Pipeline IR.",
+                details={"missing_edges": [list(edge) for edge in missing_ir_edges]},
+            )
+        )
+    if unexpected_ir_edges:
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="unexplained_emitted_edge",
+                severity="failed",
+                message="Pipeline IR contains dependency edge(s) absent from the transformation ledger.",
+                details={"unexpected_edges": [list(edge) for edge in unexpected_ir_edges]},
+            )
+        )
+
+    capture_by_location: dict[tuple[int, str], list[TaskCapture]] = {}
+    for capture in visitor.task_captures.values():
+        capture_by_location.setdefault((capture.span.line, capture.operator), []).append(capture)
+    argument_failure_keys: set[str] = set()
+    audit_candidate_by_capture: dict[str, source_audit.AuditCandidate] = {}
+    for candidate in audit.tasks:
+        if candidate.code != "operator_task":
+            continue
+        operator = str(candidate.details.get("operator", ""))
+        matches = capture_by_location.get((candidate.line, operator), [])
+        if not matches:
+            continue
+        capture = matches.pop(0)
+        audit_candidate_by_capture[capture.capture_id] = candidate
+        expected = set(candidate.details.get("kwargs", []))
+        actual = set(visitor.operators[capture.capture_id][2])
+        if expected == actual:
+            continue
+        task_key = var_to_task_key.get(capture.capture_id, capture.capture_id)
+        argument_failure_keys.add(task_key)
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="operator_argument_capture_mismatch",
+                severity="failed",
+                message=(
+                    f"Airflow task {capture.task_id!r} audited argument(s) {sorted(expected)}, "
+                    f"but capture retained {sorted(actual)}."
+                ),
+                candidate=candidate,
+                details={
+                    "task_key": task_key,
+                    "missing": sorted(expected - actual),
+                    "unexpected": sorted(actual - expected),
+                },
+            )
+        )
+
+    dbt_factory_var = dbt_vars[0] if dbt_vars else None
+    expected_key_by_capture: dict[str, str] = {}
+    for var, task_key in var_to_task_key.items():
+        if var in dropped:
+            continue
+        expected_key_by_capture[var] = (
+            var_to_task_key[dbt_factory_var] if var in dbt_vars and dbt_factory_var is not None else task_key
+        )
+    expected_task_keys = set(expected_key_by_capture.values())
+    emitted_task_keys = {task.task_key for task in pipeline.tasks}
+    missing_task_keys = sorted(expected_task_keys - emitted_task_keys)
+    unexpected_task_keys = sorted(emitted_task_keys - expected_task_keys)
+    if missing_task_keys:
+        missing_capture = next(
+            (var for var, task_key in expected_key_by_capture.items() if task_key in missing_task_keys),
+            None,
+        )
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="captured_task_not_emitted",
+                severity="failed",
+                message=f"Captured Airflow task key(s) were not emitted to Pipeline IR: {missing_task_keys}.",
+                candidate=audit_candidate_by_capture.get(missing_capture or ""),
+                details={"task_keys": missing_task_keys},
+            )
+        )
+    if unexpected_task_keys:
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="unexplained_emitted_task",
+                severity="failed",
+                message=f"Pipeline IR contains task key(s) with no captured Airflow task: {unexpected_task_keys}.",
+                details={"task_keys": unexpected_task_keys},
+            )
+        )
+
+    unsupported_settings = [
+        candidate for candidate in audit.settings if candidate.details.get("name") not in _SUPPORTED_DAG_SETTINGS
+    ]
+    missing_supported_settings = [
+        candidate
+        for candidate in audit.settings
+        if candidate.details.get("name") in _SUPPORTED_DAG_SETTINGS
+        and candidate.details.get("name") not in visitor.captured_dag_settings
+    ]
+    for candidate in missing_supported_settings:
+        name = str(candidate.details.get("name"))
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="dag_setting_capture_mismatch",
+                severity="failed",
+                message=f"Audited DAG setting {name!r} was not captured by the Airflow loader.",
+                candidate=candidate,
+            )
+        )
+    for candidate in unsupported_settings:
+        name = str(candidate.details.get("name"))
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="unsupported_dag_setting",
+                severity="gap",
+                message=f"Airflow DAG setting {name!r} has no deterministic Databricks Jobs mapping.",
+                candidate=candidate,
+            )
+        )
+
+    for candidate in unresolved:
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code=candidate.code,
+                severity="gap",
+                message="Dynamic Airflow control flow could not be expanded safely by the static parser.",
+                candidate=candidate,
+            )
+        )
+
+    for var, task_key in var_to_task_key.items():
+        task_id = (
+            visitor.operators[var][0]
+            if var in visitor.operators
+            else visitor.taskflow_tasks[var].task_id
+            if var in visitor.taskflow_tasks
+            else visitor.taskgroup_calls[var][0]
+        )
+        base = _sanitize_task_key(task_id)
+        if var in visitor.groups:
+            base = f"{visitor.groups[var]}__{base}"
+        transformations.append(
+            {
+                "code": "task_key_allocated",
+                "capture_id": var,
+                "source_task_id": task_id,
+                "task_key": task_key,
+                "emitted_task_key": expected_key_by_capture.get(var),
+            }
+        )
+        if task_key != base:
+            transformations.append(
+                {
+                    "code": "task_key_collision_resolved",
+                    "capture_id": var,
+                    "source_task_id": task_id,
+                    "task_key": task_key,
+                }
+            )
+    for var in sorted(dropped):
+        transformations.append(
+            {
+                "code": "structural_task_rewired",
+                "capture_id": var,
+                "task_key": var_to_task_key.get(var, var),
+            }
+        )
+    if len(dbt_vars) > 1:
+        transformations.append(
+            {
+                "code": "dbt_chain_collapsed",
+                "capture_ids": list(dbt_vars),
+                "task_key": var_to_task_key.get(dbt_vars[0], ""),
+            }
+        )
+
+    placeholder_by_key = {
+        placeholder.task_key: placeholder
+        for placeholder in _iter_placeholders(pipeline.tasks)
+        if not placeholder.task_key.startswith("__flowx_")
+    }
+    for index, placeholder in enumerate(placeholder_by_key.values()):
+        placeholder_candidate = audit.tasks[index] if index < len(audit.tasks) else None
+        findings.append(
+            source_audit.finding(
+                source_file=source_file,
+                code="operator_placeholder",
+                severity="gap",
+                message=(
+                    f"Airflow task {placeholder.name!r} ({placeholder.original_type}) requires explicit migration."
+                ),
+                candidate=placeholder_candidate,
+                details={"task_key": placeholder.task_key, "operator": placeholder.original_type},
+            )
+        )
+
+    blocking_gaps = [*unsupported_settings, *unresolved]
+    if blocking_gaps:
+        placeholder_key = "__flowx_source_gaps"
+        gap_task = PlaceholderActivity(
+            name="Airflow source semantics requiring migration",
+            task_key=placeholder_key,
+            original_type="AirflowSourceSemantics",
+            comment="Resolve the source-audit findings before enabling this DAG.",
+            raw_definition={"findings": [item for item in findings if item["severity"] == "gap"]},
+        )
+        for task in pipeline.tasks:
+            if not task.depends_on:
+                task.depends_on = [Dependency(task_key=placeholder_key)]
+        pipeline.tasks.insert(0, gap_task)
+
+    failed_findings = [item for item in findings if item["severity"] == "failed"]
+    gap_findings = [item for item in findings if item["severity"] == "gap"]
+    status = "failed" if failed_findings else "verified_with_gaps" if gap_findings else "verified"
+    failed_capture_keys = argument_failure_keys | set(missing_task_keys)
+    agentic_captured_count = len(placeholder_by_key)
+    deterministic_count = captured_task_count - len(failed_capture_keys) - agentic_captured_count
+    agentic_count = agentic_captured_count + len(unresolved)
+    failed_count = (
+        len(failed_capture_keys)
+        + len(unmatched_audit_tasks)
+        + len(visitor.unclaimed_task_calls)
+        + len(visitor.unclaimed_statements)
+    )
+    audited_task_count = (
+        captured_task_count
+        + len(unresolved)
+        + len(unmatched_audit_tasks)
+        + len(visitor.unclaimed_task_calls)
+        + len(visitor.unclaimed_statements)
+    )
+
+    pipeline.not_translatable = findings
+    pipeline.reconciliation_status = status
+    pipeline.audit = {
+        "source_file": source_file,
+        "audited_activity_count": audited_task_count,
+        "captured_task_count": captured_task_count,
+        "audited_edge_count": len(audit.edges),
+        "captured_edge_count": len(visitor.edge_captures),
+        "deterministic_count": deterministic_count,
+        "agentic_count": agentic_count,
+        "failed_count": failed_count,
+        "excluded_count": 0,
+        "transformations": transformations,
+    }
+    return pipeline
 
 
 def _wrap_in_for_each(
@@ -1963,7 +2866,7 @@ def _convert_activity_templates(activity: Activity) -> set[str]:
 def _unresolved_activity_templates(activity: Activity) -> set[str]:
     """Returns residual Airflow Jinja expressions in task parameter fields."""
     unresolved: set[str] = set()
-    for attribute in ("base_parameters", "job_parameters", "parameters", "sql"):
+    for attribute in ("base_parameters", "job_parameters", "parameters", "sql", "generated_source"):
         unresolved |= templating.unresolved_jinja_expressions(getattr(activity, attribute, None))
     return unresolved
 
@@ -1993,21 +2896,50 @@ def resolve(var: str, seen: set[str]) -> list[str]:
 def _root_trigger_sensor(
     operators: dict[str, tuple[str, str, dict[str, ast.expr]]],
     upstreams: dict[str, list[str]],
-) -> str | None:
-    """Returns the var of a root sensor eligible to become a job-level trigger, else None.
+    all_task_vars: set[str],
+) -> tuple[str, set[str]] | None:
+    """Returns a root sensor and its proven descendant set, or None.
 
     Only a sensor with no upstreams (the DAG's entry gate) can lift to a file_arrival /
     table_update trigger: mid-DAG sensors are ordering gates within the run and must stay
-    as tasks. File sensors win over table sensors when both sit at the root (Databricks jobs
-    take a single trigger). A table/SQL sensor lifts only when it names a literal table; one
-    without a ``table_name`` is an arbitrary-condition sensor kept as a polling task.
+    as tasks. The sensor must reach every non-sensor task; otherwise lifting it would gate
+    independent work that Airflow did not gate. File sensors win over table sensors when both
+    qualify. A table/SQL sensor lifts only when it names a literal table.
     """
-    file_roots = [var for var, (_id, op, _kw) in operators.items() if op in ops.FILE_SENSORS and not upstreams.get(var)]
-    if file_roots:
-        return file_roots[0]
-    for var, (_id, op, kw) in operators.items():
-        if op in ops.TABLE_SENSORS and not upstreams.get(var) and ops.literal_str(kw.get("table_name")) is not None:
-            return var
+    adjacency: dict[str, set[str]] = {var: set() for var in all_task_vars}
+    for downstream, dependencies in upstreams.items():
+        for upstream in dependencies:
+            adjacency.setdefault(upstream, set()).add(downstream)
+
+    def _descendants(root: str) -> set[str]:
+        descendants: set[str] = set()
+        stack = list(adjacency.get(root, ()))
+        while stack:
+            current = stack.pop()
+            if current in descendants:
+                continue
+            descendants.add(current)
+            stack.extend(adjacency.get(current, ()))
+        return descendants
+
+    sensor_vars = {var for var, (_id, operator, _kwargs) in operators.items() if operator.endswith("Sensor")}
+    required = all_task_vars - sensor_vars
+    candidates = [
+        var
+        for var, (_id, operator, _kwargs) in operators.items()
+        if operator in ops.FILE_SENSORS and not upstreams.get(var)
+    ]
+    candidates.extend(
+        var
+        for var, (_id, operator, kwargs) in operators.items()
+        if operator in ops.TABLE_SENSORS
+        and not upstreams.get(var)
+        and ops.literal_str(kwargs.get("table_name")) is not None
+    )
+    for candidate in candidates:
+        descendants = _descendants(candidate)
+        if required <= descendants:
+            return candidate, descendants
     return None
 
 
@@ -2167,7 +3099,13 @@ def discover_dags(source_path: Path) -> list[Path]:
     return dags
 
 
-def load_pipelines(source_path: Path, pipeline: str | None = None, *, dbt_mode: str = "static") -> list[Pipeline]:
+def load_pipelines(
+    source_path: Path,
+    pipeline: str | None = None,
+    *,
+    dbt_mode: str = "static",
+    exclude_dags: set[str] | None = None,
+) -> list[Pipeline]:
     """Loads every DAG under *source_path* into Pipeline IR.
 
     Args:
@@ -2179,14 +3117,68 @@ def load_pipelines(source_path: Path, pipeline: str | None = None, *, dbt_mode:
         One :class:`~flowx.models.ir.Pipeline` per discovered DAG, filtered to
         *pipeline* when provided.
     """
+    root = source_path if source_path.is_dir() else source_path.parent
     pipelines = [
-        loaded for dag_path in discover_dags(source_path) for loaded in load_airflow_dags(dag_path, dbt_mode=dbt_mode)
+        loaded
+        for dag_path in discover_dags(source_path)
+        for loaded in load_airflow_dags(
+            dag_path,
+            dbt_mode=dbt_mode,
+            source_file=source_audit.source_label(dag_path, root),
+        )
     ]
     if pipeline is not None:
         pipelines = [p for p in pipelines if p.name == pipeline]
+    excluded = set(exclude_dags or ())
+    for loaded in pipelines:
+        if loaded.name in excluded:
+            loaded.migration_status = "excluded"
+            count = int(loaded.audit.get("audited_activity_count", 0))
+            loaded.audit.update(
+                {
+                    "deterministic_count": 0,
+                    "agentic_count": 0,
+                    "failed_count": 0,
+                    "excluded_count": count,
+                }
+            )
+    if excluded:
+        _replace_excluded_dag_references(pipelines, excluded)
     return pipelines
 
 
+def _replace_excluded_dag_references(pipelines: list[Pipeline], excluded: set[str]) -> None:
+    """Replaces included-to-excluded run-job references with explicit placeholders."""
+    excluded_by_key = {normalize_task_key(name): name for name in excluded}
+    for pipeline in pipelines:
+        if pipeline.migration_status == "excluded":
+            continue
+        for index, task in enumerate(pipeline.tasks):
+            if isinstance(task, RunJobActivity) and task.job_name in excluded_by_key:
+                excluded_name = excluded_by_key[task.job_name]
+                placeholder = PlaceholderActivity(
+                    name=task.name,
+                    task_key=task.task_key,
+                    depends_on=task.depends_on,
+                    original_type="ExcludedDagReference",
+                    comment=f"Referenced Airflow DAG {excluded_name!r} was excluded from this migration.",
+                    raw_definition={"excluded_dag": excluded_name},
+                )
+                pipeline.tasks[index] = placeholder
+                entry = source_audit.finding(
+                    source_file=str(pipeline.audit.get("source_file", "")),
+                    code="excluded_dag_reference",
+                    severity="gap",
+                    message=f"Task {task.task_key!r} references excluded DAG {excluded_name!r}.",
+                    details={"task_key": task.task_key, "excluded_dag": excluded_name},
+                )
+                pipeline.not_translatable.append(entry)
+                if pipeline.reconciliation_status != "failed":
+                    pipeline.reconciliation_status = "verified_with_gaps"
+                pipeline.audit["agentic_count"] = int(pipeline.audit.get("agentic_count", 0)) + 1
+                pipeline.audit["deterministic_count"] = max(0, int(pipeline.audit.get("deterministic_count", 0)) - 1)
+
+
 _HOST_PATTERN = re.compile(r"https://([A-Za-z0-9._-]*(?:azuredatabricks\.net|databricks\.com|cloud\.databricks\.com))")
 
 
diff --git a/src/flowx/validate/bundle_invariants.py b/src/flowx/validate/bundle_invariants.py
index 5a88d44..01b09dc 100644
--- a/src/flowx/validate/bundle_invariants.py
+++ b/src/flowx/validate/bundle_invariants.py
@@ -25,6 +25,7 @@
 _ANCHOR_RE = re.compile(r"[&*]id\d+\b")
 _JOB_PARAM_REF_RE = re.compile(r"\{\{\s*job\.parameters\.([A-Za-z0-9_]+)\s*\}\}")
 _JOB_RESOURCE_ID_RE = re.compile(r"\$\{resources\.jobs\.([^.}]+)\.id\}")
+_PYDABS_JOB_RE = re.compile(r"resources\.add_job\(\s*['\"]([^'\"]+)['\"]")
 
 
 @dataclass(slots=True, kw_only=True)
@@ -240,6 +241,15 @@ def check_bundle_dir(bundle_dir: Path) -> BundleInvariantResult:
         jobs = (document.get("resources") or {}).get("jobs") or {}
         if isinstance(jobs, dict):
             known_jobs.update(str(job_key) for job_key in jobs)
+        python_resources = (document.get("python") or {}).get("resources") or []
+        for resource in python_resources:
+            if not isinstance(resource, str):
+                continue
+            module = resource.split(":", 1)[0]
+            if module.startswith("resources."):
+                known_jobs.add(module.rsplit(".", 1)[-1])
+    for hook_path in sorted(resources_dir.glob("*.py")) if resources_dir.exists() else []:
+        known_jobs.update(_PYDABS_JOB_RE.findall(hook_path.read_text(encoding="utf-8")))
 
     for path, document in documents:
         jobs = (document.get("resources") or {}).get("jobs") or {}
@@ -257,12 +267,10 @@ def check_bundle_dir(bundle_dir: Path) -> BundleInvariantResult:
                 findings.append(
                     BundleFinding(
                         code="dangling_run_job_reference",
-                        severity="warning",
                         location=f"{path.name}, job '{job_key}', task '{task.get('task_key', '')}'",
                         message=(
                             f"run_job_task references bundle job '{match.group(1)}', which is not declared "
-                            "in static resource YAML. Confirm it is supplied by a Python resource or replace "
-                            "the reference with a declared bundle variable containing the external job ID."
+                            "in static resource YAML or registered as a Python resource."
                         ),
                     )
                 )
diff --git a/tests/unit/test_airflow_reconciliation.py b/tests/unit/test_airflow_reconciliation.py
new file mode 100644
index 0000000..283ae2e
--- /dev/null
+++ b/tests/unit/test_airflow_reconciliation.py
@@ -0,0 +1,409 @@
+"""Tests for Airflow source auditing, exclusions, and package preflight."""
+
+from __future__ import annotations
+
+import ast
+import json
+from pathlib import Path
+
+import pytest
+
+from flowx import ir_serde
+from flowx.bundler import dab_writer
+from flowx.models.ir import Dependency, Pipeline, PlaceholderActivity
+from flowx.sources.airflow import audit
+from flowx.sources.airflow import loader as airflow_loader
+
+_SIMPLE_DAG = (
+    "from airflow import DAG\n"
+    "from airflow.operators.bash import BashOperator\n"
+    "with DAG(dag_id='audited', schedule='@daily') as dag:\n"
+    "    first = BashOperator(task_id='first', bash_command='echo first')\n"
+    "    second = BashOperator(task_id='second', bash_command='echo second')\n"
+    "    first >> second\n"
+)
+
+
+def _candidate(kind: str, code: str) -> audit.AuditCandidate:
+    return audit.AuditCandidate(kind=kind, code=code, line=1, column=0, occurrence=1)
+
+
+def test_finding_fingerprint_uses_relative_file_full_span_and_code() -> None:
+    first = audit.finding(
+        source_file="dags/example.py",
+        code="argument_loss",
+        severity="failed",
+        message="lost",
+        candidate=audit.AuditCandidate(
+            kind="argument",
+            code="argument_loss",
+            line=4,
+            column=2,
+            end_line=4,
+            end_column=12,
+            occurrence=1,
+        ),
+    )
+    second = audit.finding(
+        source_file="dags/example.py",
+        code="argument_loss",
+        severity="failed",
+        message="lost",
+        candidate=audit.AuditCandidate(
+            kind="argument",
+            code="argument_loss",
+            line=4,
+            column=2,
+            end_line=5,
+            end_column=12,
+            occurrence=1,
+        ),
+    )
+
+    assert first["source_file"] == "dags/example.py"
+    assert first["end_line"] == 4
+    assert first["end_column"] == 12
+    assert first["fingerprint"] != second["fingerprint"]
+
+
+@pytest.mark.parametrize("mutation", ["task", "edge", "setting", "argument"])
+def test_source_capture_mutations_fail_reconciliation(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mutation: str
+) -> None:
+    dag_path = tmp_path / "audited.py"
+    dag_path.write_text(_SIMPLE_DAG, encoding="utf-8")
+
+    if mutation in {"task", "edge"}:
+        original_audit = audit.audit_module
+
+        def mutate(module: ast.Module, *, target_dag_variable: str | None = None) -> audit.SourceAudit:
+            result = original_audit(module, target_dag_variable=target_dag_variable)
+            if mutation == "task":
+                result.tasks.append(_candidate("task", "removed_capture_task"))
+            else:
+                result.edges.append(_candidate("edge", "removed_capture_edge"))
+            return result
+
+        monkeypatch.setattr(audit, "audit_module", mutate)
+    elif mutation == "setting":
+        original_apply = airflow_loader._DagVisitor._apply_dag_kwargs
+
+        def drop_setting(self, kwargs):
+            original_apply(self, kwargs)
+            self.captured_dag_settings.discard("schedule")
+
+        monkeypatch.setattr(airflow_loader._DagVisitor, "_apply_dag_kwargs", drop_setting)
+    else:
+        original_register = airflow_loader._DagVisitor._register_operator_call
+
+        def drop_argument(self, node, var, *, binding=None):
+            registered = original_register(self, node, var, binding=binding)
+            if registered and self.operators[var][0] == "first":
+                self.operators[var][2].pop("bash_command", None)
+            return registered
+
+        monkeypatch.setattr(airflow_loader._DagVisitor, "_register_operator_call", drop_argument)
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+
+    assert pipeline.reconciliation_status == "failed"
+    assert any(finding["severity"] == "failed" for finding in pipeline.not_translatable)
+    assert pipeline.audit["failed_count"] == (1 if mutation in {"task", "argument"} else 0)
+
+
+def test_failed_report_blocks_package_before_bundle_writes(tmp_path: Path) -> None:
+    report = tmp_path / "failed.json"
+    output = tmp_path / "bundle"
+    pipeline = Pipeline(
+        name="failed",
+        reconciliation_status="failed",
+        not_translatable=[
+            {
+                "code": "task_capture_mismatch",
+                "severity": "failed",
+                "message": "one source task was not captured",
+            }
+        ],
+    )
+    report.write_text(json.dumps(ir_serde.pipeline_to_dict(pipeline)), encoding="utf-8")
+
+    exit_code = dab_writer.main(
+        [
+            "--report",
+            str(report),
+            "--output-dir",
+            str(output),
+            "--no-download-workspace-files",
+            "--keep-intermediates",
+        ]
+    )
+
+    assert exit_code == 1
+    assert not (output / "databricks.yml").exists()
+    assert not (output / "resources").exists()
+    assert not (output / "src").exists()
+
+
+def test_captured_task_removed_from_ir_fails_reconciliation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    dag_path = tmp_path / "audited.py"
+    dag_path.write_text(_SIMPLE_DAG, encoding="utf-8")
+    original_reconcile = airflow_loader._reconcile_pipeline
+
+    def remove_emitted_task(pipeline, **kwargs):
+        pipeline.tasks.pop()
+        return original_reconcile(pipeline, **kwargs)
+
+    monkeypatch.setattr(airflow_loader, "_reconcile_pipeline", remove_emitted_task)
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+
+    assert pipeline.reconciliation_status == "failed"
+    finding = next(item for item in pipeline.not_translatable if item["code"] == "captured_task_not_emitted")
+    assert finding["details"]["task_keys"] == ["second"]
+
+
+@pytest.mark.parametrize(
+    ("body", "expected_failed"),
+    [
+        (
+            "    head = BashOperator(task_id='head', bash_command='echo head')\n"
+            "    fanout = [BashOperator(task_id=f'work_{i}', bash_command='echo work') for i in range(3)]\n",
+            1,
+        ),
+        (
+            "    first, second = (\n"
+            "        BashOperator(task_id='first', bash_command='echo first'),\n"
+            "        BashOperator(task_id='second', bash_command='echo second'),\n"
+            "    )\n",
+            2,
+        ),
+    ],
+)
+def test_unclaimed_dag_task_construction_fails_closed(tmp_path: Path, body: str, expected_failed: int) -> None:
+    dag_path = tmp_path / "unclaimed.py"
+    dag_path.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='unclaimed') as dag:\n" + body,
+        encoding="utf-8",
+    )
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+
+    assert pipeline.reconciliation_status == "failed"
+    assert pipeline.audit["failed_count"] == expected_failed
+    assert pipeline.audit["audited_activity_count"] >= expected_failed
+    assert any(item["code"] == "unclaimed_dag_task" for item in pipeline.not_translatable)
+
+
+def test_source_edge_identity_mismatch_fails_reconciliation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    dag_path = tmp_path / "rewired.py"
+    dag_path.write_text(_SIMPLE_DAG, encoding="utf-8")
+    original_add_edges = airflow_loader._DagVisitor._add_edges
+
+    def reverse_edge(self, upstreams, downstreams, node):
+        return original_add_edges(self, downstreams, upstreams, node)
+
+    monkeypatch.setattr(airflow_loader._DagVisitor, "_add_edges", reverse_edge)
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+
+    assert pipeline.reconciliation_status == "failed"
+    finding = next(item for item in pipeline.not_translatable if item["code"] == "edge_identity_mismatch")
+    assert finding["details"]["audited_edges"] == [["first", "second"]]
+    assert finding["details"]["captured_edges"] == [["second", "first"]]
+
+
+def test_captured_edge_removed_from_ir_fails_reconciliation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    dag_path = tmp_path / "missing_ir_edge.py"
+    dag_path.write_text(_SIMPLE_DAG, encoding="utf-8")
+    original_reconcile = airflow_loader._reconcile_pipeline
+
+    def remove_emitted_edge(pipeline, **kwargs):
+        pipeline.tasks[-1].depends_on = None
+        return original_reconcile(pipeline, **kwargs)
+
+    monkeypatch.setattr(airflow_loader, "_reconcile_pipeline", remove_emitted_edge)
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+
+    assert pipeline.reconciliation_status == "failed"
+    finding = next(item for item in pipeline.not_translatable if item["code"] == "captured_edge_not_emitted")
+    assert finding["details"]["missing_edges"] == [["first", "second"]]
+
+
+def test_helper_capture_does_not_depend_on_independent_auditor_classification(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    repro = Path(__file__).parents[1] / "resources" / "airflow" / "review_repros" / "t8_helperfn.py"
+    original_audit = audit.audit_module
+
+    def omit_helper_candidates(module: ast.Module, *, target_dag_variable: str | None = None) -> audit.SourceAudit:
+        result = original_audit(module, target_dag_variable=target_dag_variable)
+        result.tasks = [candidate for candidate in result.tasks if candidate.code != "helper_factory_task"]
+        return result
+
+    monkeypatch.setattr(audit, "audit_module", omit_helper_candidates)
+
+    pipeline = airflow_loader.load_airflow_dag(repro)
+
+    assert pipeline.reconciliation_status == "verified"
+    assert pipeline.audit["audited_activity_count"] == 2
+
+
+def test_removing_real_helper_capture_is_detected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    dag_path = tmp_path / "helper.py"
+    dag_path.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "def make(task_id):\n"
+        "    return BashOperator(task_id=task_id, bash_command='echo work')\n"
+        "with DAG(dag_id='helper') as dag:\n"
+        "    work = make('work')\n",
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(airflow_loader._DagVisitor, "_register_helper_factory_call", lambda *args, **kwargs: False)
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+
+    assert pipeline.reconciliation_status == "failed"
+    assert any(item["code"] == "unclaimed_dag_task" for item in pipeline.not_translatable)
+
+
+def test_dynamic_helper_statement_cannot_escape_both_capture_passes(tmp_path: Path) -> None:
+    dag_path = tmp_path / "dynamic_helper.py"
+    dag_path.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "def make(task_id):\n"
+        "    command = 'echo work'\n"
+        "    return BashOperator(task_id=task_id, bash_command=command)\n"
+        "with DAG(dag_id='dynamic_helper') as dag:\n"
+        "    work = make('work')\n",
+        encoding="utf-8",
+    )
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+
+    assert pipeline.reconciliation_status == "failed"
+    assert pipeline.audit["failed_count"] == 1
+    assert any(item["code"] == "unclaimed_dag_statement" for item in pipeline.not_translatable)
+
+
+def test_assigned_dag_dynamic_helper_call_fails_closed(tmp_path: Path) -> None:
+    dag_path = tmp_path / "assigned_dynamic_helper.py"
+    dag_path.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "dag = DAG(dag_id='assigned_dynamic_helper')\n"
+        "def make(task_id):\n"
+        "    command = 'echo work'\n"
+        "    return BashOperator(task_id=task_id, bash_command=command, dag=dag)\n"
+        "work = make('work')\n",
+        encoding="utf-8",
+    )
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+
+    assert pipeline.reconciliation_status == "failed"
+    assert pipeline.audit["failed_count"] == 1
+    assert any(item["code"] == "unclaimed_dag_task" for item in pipeline.not_translatable)
+
+
+def test_uninvoked_module_helper_body_does_not_create_a_task(tmp_path: Path) -> None:
+    dag_path = tmp_path / "dormant_helper.py"
+    dag_path.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "dag = DAG(dag_id='dormant_helper')\n"
+        "def dormant():\n"
+        "    task = BashOperator(task_id='dormant', bash_command='echo dormant', dag=dag)\n"
+        "    return task\n",
+        encoding="utf-8",
+    )
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+
+    assert pipeline.reconciliation_status == "verified"
+    assert pipeline.tasks == []
+    assert pipeline.audit["audited_activity_count"] == 0
+
+
+def test_unresolved_construct_is_classified_in_coverage(tmp_path: Path) -> None:
+    dag_path = tmp_path / "dynamic.py"
+    dag_path.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='dynamic') as dag:\n"
+        "    stable = BashOperator(task_id='stable', bash_command='echo stable')\n"
+        "    for item in runtime_values:\n"
+        "        BashOperator(task_id=f'work_{item}', bash_command='echo work')\n",
+        encoding="utf-8",
+    )
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+
+    assert pipeline.reconciliation_status == "verified_with_gaps"
+    assert pipeline.audit["audited_activity_count"] == 2
+    assert pipeline.audit["deterministic_count"] == 1
+    assert pipeline.audit["agentic_count"] == 1
+
+
+def test_bundle_invariant_failure_is_preflighted_before_destination_writes(tmp_path: Path) -> None:
+    report = tmp_path / "dangling.json"
+    output = tmp_path / "bundle"
+    pipeline = Pipeline(
+        name="parent",
+        tags={"source": "airflow"},
+        reconciliation_status="verified",
+        tasks=[
+            PlaceholderActivity(
+                name="dangling",
+                task_key="dangling",
+                original_type="Test",
+                depends_on=[Dependency(task_key="missing")],
+            )
+        ],
+    )
+    report.write_text(json.dumps(ir_serde.pipeline_to_dict(pipeline)), encoding="utf-8")
+
+    exit_code = dab_writer.main(
+        [
+            "--report",
+            str(report),
+            "--output-dir",
+            str(output),
+            "--no-download-workspace-files",
+            "--keep-intermediates",
+        ]
+    )
+
+    assert exit_code == 1
+    assert not (output / "databricks.yml").exists()
+
+
+def test_excluded_dag_stays_audited_and_included_reference_becomes_placeholder(tmp_path: Path) -> None:
+    (tmp_path / "caller.py").write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.trigger_dagrun import TriggerDagRunOperator\n"
+        "with DAG(dag_id='caller') as dag:\n"
+        "    trigger = TriggerDagRunOperator(task_id='trigger', trigger_dag_id='target')\n",
+        encoding="utf-8",
+    )
+    (tmp_path / "target.py").write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='target') as dag:\n"
+        "    work = BashOperator(task_id='work', bash_command='echo work')\n",
+        encoding="utf-8",
+    )
+
+    pipelines = airflow_loader.load_pipelines(tmp_path, exclude_dags={"target"})
+    by_name = {pipeline.name: pipeline for pipeline in pipelines}
+
+    assert by_name["target"].migration_status == "excluded"
+    assert by_name["target"].audit["audited_activity_count"] == 1
+    assert by_name["target"].audit["excluded_count"] == 1
+    assert isinstance(by_name["caller"].tasks[0], PlaceholderActivity)
+    assert by_name["caller"].tasks[0].raw_definition == {"excluded_dag": "target"}
+    assert by_name["caller"].reconciliation_status == "verified_with_gaps"
diff --git a/tests/unit/test_bundle_invariants.py b/tests/unit/test_bundle_invariants.py
index 8c2f8a1..92f4606 100644
--- a/tests/unit/test_bundle_invariants.py
+++ b/tests/unit/test_bundle_invariants.py
@@ -122,7 +122,7 @@ def test_bundle_job_reference_to_unknown_resource_is_flagged(tmp_path):
     result = check_bundle_dir(tmp_path)
     finding = next(finding for finding in result.findings if finding.code == "dangling_run_job_reference")
 
-    assert finding.severity == "warning"
+    assert finding.severity == "violation"
     assert "parent.yml" in finding.location
     assert "call_missing" in finding.location
     assert "dangling_run_job_reference" in format_result(result)
diff --git a/tests/unit/test_mcp_source_routing.py b/tests/unit/test_mcp_source_routing.py
index 1dd870a..491f641 100644
--- a/tests/unit/test_mcp_source_routing.py
+++ b/tests/unit/test_mcp_source_routing.py
@@ -74,6 +74,23 @@ def test_discover_routes_airflow_source(captured, tmp_path: Path):
     assert argv[argv.index("--source-path") + 1] == str(tmp_path)
 
 
+def test_airflow_exclusions_are_forwarded_as_repeatable_flags(captured, tmp_path: Path):
+    parameters = {
+        "source": "airflow",
+        "airflow_source_path": str(tmp_path),
+        "output_dir": str(tmp_path / "o"),
+        "exclude_dag": ["legacy", "experimental"],
+    }
+
+    server._cmd_discover(parameters)
+    server._cmd_convert(parameters)
+
+    for command in ("discover", "convert"):
+        argv = _argv(captured, command)
+        exclusions = [argv[index + 1] for index, value in enumerate(argv) if value == "--exclude-dag"]
+        assert exclusions == ["legacy", "experimental"]
+
+
 def test_convert_threads_source(captured, tmp_path: Path):
     server._cmd_convert({"source": "airflow", "airflow_source_path": str(tmp_path), "output_dir": str(tmp_path)})
     argv = _argv(captured, "convert")

From c29b685db98b5e8da80169edd9dda2ca16ec5a47 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Fri, 7 Aug 2026 14:03:32 -0700
Subject: [PATCH 51/77] Harden Airflow task semantics

---
 src/flowx/sources/airflow/operators.py        | 141 +++++++++++++++--
 src/flowx/sources/airflow/templating.py       |  65 ++++++--
 .../unit/test_airflow_production_readiness.py | 147 +++++++++++++++++-
 3 files changed, 320 insertions(+), 33 deletions(-)

diff --git a/src/flowx/sources/airflow/operators.py b/src/flowx/sources/airflow/operators.py
index 64de9a8..033cc56 100644
--- a/src/flowx/sources/airflow/operators.py
+++ b/src/flowx/sources/airflow/operators.py
@@ -16,6 +16,7 @@
 
 import ast
 import json as _json
+import re
 import shlex
 from dataclasses import dataclass, field
 from typing import Any, Callable
@@ -176,10 +177,19 @@ def _sh_notebook(task_id: str, command: str, env_widgets: dict[str, str] | None
             export.append(f"dbutils.widgets.text({name!r}, '')")
             export.append(f"os.environ[{name!r}] = dbutils.widgets.get({name!r})")
         header += "\n".join(export) + "\n\n# COMMAND ----------\n\n"
-    lines = "".join(f"# MAGIC {line}\n" for line in command.splitlines())
+    lines = "".join(f"# MAGIC {_sanitize_sh_line(line)}\n" for line in command.splitlines())
     return header + "# MAGIC %sh\n" + lines
 
 
+def _sanitize_sh_line(line: str) -> str:
+    """Keeps notebook source directives inert inside the generated shell cell."""
+    stripped = line.lstrip()
+    if stripped.startswith("# MAGIC") or stripped.startswith("# COMMAND ----------"):
+        indentation = line[: len(line) - len(stripped)]
+        return f"{indentation}#{stripped}"
+    return line
+
+
 # Airflow sensor defaults (seconds): poke every 60s, give up after 7 days.
 _DEFAULT_POKE_INTERVAL = 60
 _DEFAULT_SENSOR_TIMEOUT = 604800
@@ -420,41 +430,94 @@ class _SparkSubmit:
     app_args: list[str]
 
 
+_SPARK_VALUE_OPTIONS = frozenset(
+    {
+        "--master",
+        "--deploy-mode",
+        "--class",
+        "--name",
+        "--jars",
+        "--packages",
+        "--exclude-packages",
+        "--repositories",
+        "--py-files",
+        "--files",
+        "--archives",
+        "--conf",
+        "--properties-file",
+        "--driver-memory",
+        "--driver-java-options",
+        "--driver-library-path",
+        "--driver-class-path",
+        "--executor-memory",
+        "--proxy-user",
+        "--driver-cores",
+        "--queue",
+        "--num-executors",
+        "--total-executor-cores",
+        "--executor-cores",
+        "--principal",
+        "--keytab",
+        "--resourceProfile",
+    }
+)
+_SPARK_BOOLEAN_OPTIONS = frozenset({"--supervise", "--verbose", "--load-spark-defaults"})
+_SHELL_CONTROL = re.compile(r"(?:&&|\|\||[|;]|(?:^|\s)cd(?:\s|$))")
+
+
 def parse_spark_submit(command: str) -> _SparkSubmit | None:
     """Parses a ``spark-submit ...`` command line into its application + args.
 
     Returns ``None`` when the command is not a spark-submit invocation.
     """
+    if "\n" in command or _SHELL_CONTROL.search(command):
+        return None
     try:
         tokens = shlex.split(command)
     except ValueError:
         return None
-    if "spark-submit" not in tokens:
+    if not tokens or tokens[0].rsplit("/", 1)[-1] != "spark-submit":
         return None
-    tokens = tokens[tokens.index("spark-submit") + 1 :]
+    tokens = tokens[1:]
 
     java_class: str | None = None
     application: str | None = None
     app_args: list[str] = []
     index = 0
-    # spark-submit flags that take a value we skip over (cluster-side config, not app args).
-    valued_flags = {"--master", "--deploy-mode", "--conf", "--name", "--jars", "--packages", "--files", "--py-files"}
     while index < len(tokens):
         token = tokens[index]
-        if token == "--class" and index + 1 < len(tokens):
-            java_class = tokens[index + 1]
-            index += 2
+        option, separator, inline_value = token.partition("=")
+        if option in _SPARK_VALUE_OPTIONS:
+            if separator:
+                value = inline_value
+                if not value or value.startswith("-"):
+                    return None
+            elif index + 1 < len(tokens):
+                value = tokens[index + 1]
+                if value.startswith("-"):
+                    return None
+            else:
+                return None
+            if option == "--class":
+                java_class = value
+            index += 1 if separator else 2
             continue
-        if token in valued_flags and index + 1 < len(tokens):
-            index += 2
-            continue
-        if token.startswith("--"):
+        if option in _SPARK_BOOLEAN_OPTIONS and not separator:
             index += 1
             continue
-        # First bare token is the application; the rest are application args.
+        if token.startswith("--"):
+            return None
+        if token.startswith("-"):
+            return None
+        if token == "spark-submit":
+            return None
+        if not token:
+            return None
         application = token
         app_args = tokens[index + 1 :]
         break
+    if application is None:
+        return None
     return _SparkSubmit(application=application, java_class=java_class, app_args=app_args)
 
 
@@ -751,6 +814,58 @@ def build_placeholder_with_comment(ctx: OperatorContext, comment: str) -> Activi
     return _placeholder(ctx, comment)
 
 
+_LOADER_CONSUMED_KWARGS = frozenset({"task_id", "dag", "trigger_rule", "retries", "retry_delay", "execution_timeout"})
+_OPERATOR_CONSUMED_KWARGS: dict[str, frozenset[str]] = {
+    "PythonOperator": frozenset({"python_callable", "op_args", "op_kwargs"}),
+    "BranchPythonOperator": frozenset({"python_callable", "op_args", "op_kwargs"}),
+    "ShortCircuitOperator": frozenset({"python_callable", "op_args", "op_kwargs"}),
+    "BashOperator": frozenset({"bash_command"}),
+    "SSHOperator": frozenset({"command"}),
+    "SparkSubmitOperator": frozenset({"application", "java_class", "application_args"}),
+    "DatabricksSubmitRunOperator": frozenset({"json"}),
+    "DatabricksSubmitRunDeferrableOperator": frozenset({"json"}),
+    "DatabricksRunNowOperator": frozenset({"job_id", "notebook_params", "python_params", "jar_params"}),
+    "DatabricksRunNowDeferrableOperator": frozenset({"job_id", "notebook_params", "python_params", "jar_params"}),
+    "DatabricksNotebookOperator": frozenset({"notebook_path", "notebook_params"}),
+    "DatabricksSqlOperator": frozenset({"sql"}),
+    "DatabricksSQLStatementsOperator": frozenset({"sql"}),
+    "DatabricksCopyIntoOperator": frozenset({"file_location", "table_name", "file_format"}),
+    "SQLExecuteQueryOperator": frozenset({"sql"}),
+    "PostgresOperator": frozenset({"sql"}),
+    "MySqlOperator": frozenset({"sql"}),
+    "HiveOperator": frozenset({"hql"}),
+    "TriggerDagRunOperator": frozenset({"trigger_dag_id", "conf"}),
+    "PythonVirtualenvOperator": frozenset({"python_callable", "requirements"}),
+    "ExternalPythonOperator": frozenset({"python_callable", "requirements"}),
+    "EmailOperator": frozenset(),
+    "ExternalTaskSensor": frozenset({"external_dag_id", "external_task_id", "poke_interval", "timeout"}),
+    "ExternalTaskSensorAsync": frozenset({"external_dag_id", "external_task_id", "poke_interval", "timeout"}),
+    "HttpSensor": frozenset({"endpoint", "poke_interval", "timeout"}),
+    "HttpSensorAsync": frozenset({"endpoint", "poke_interval", "timeout"}),
+    "PythonSensor": frozenset({"python_callable", "op_args", "op_kwargs", "poke_interval", "timeout"}),
+    "DateTimeSensor": frozenset({"target_time", "timeout"}),
+    "DateTimeSensorAsync": frozenset({"target_time", "timeout"}),
+}
+_FILE_SENSOR_KWARGS = frozenset(
+    {"bucket_key", "bucket_name", "object", "bucket", "filepath", "filepath_", "poke_interval", "timeout"}
+)
+_TABLE_SENSOR_KWARGS = frozenset({"sql", "table_name", "poke_interval", "timeout"})
+
+
+def unconsumed_kwargs(operator: str, kwargs: dict[str, ast.expr]) -> set[str]:
+    """Returns supplied arguments with no declared loader or adapter semantics."""
+    consumed: frozenset[str] | None
+    if operator in FILE_SENSORS:
+        consumed = _FILE_SENSOR_KWARGS
+    elif operator in TABLE_SENSORS:
+        consumed = _TABLE_SENSOR_KWARGS
+    else:
+        consumed = _OPERATOR_CONSUMED_KWARGS.get(operator)
+    if consumed is None:
+        return set()
+    return set(kwargs) - _LOADER_CONSUMED_KWARGS - consumed
+
+
 # --------------------------------------------------------------------------------------
 # Registry: operator name -> builder
 # --------------------------------------------------------------------------------------
diff --git a/src/flowx/sources/airflow/templating.py b/src/flowx/sources/airflow/templating.py
index e54dcdc..49776cd 100644
--- a/src/flowx/sources/airflow/templating.py
+++ b/src/flowx/sources/airflow/templating.py
@@ -12,6 +12,7 @@
 import ast
 import math
 import re
+from dataclasses import dataclass
 from typing import Any
 
 # Airflow date/time macros carrying the run's *logical date* -> a named job parameter (not an inline
@@ -306,11 +307,18 @@ def email_on_failure(dag_default_args: dict[str, ast.expr], task_kwargs: dict[st
 # trigger_rule -> dependency outcome
 # --------------------------------------------------------------------------------------
 
-# Map Airflow trigger_rule -> the DAB job ``run_if`` constant carried as a dependency outcome. The
-# preparer's reducer passes these through unchanged. Rules with no exact DAB equivalent fall back to
-# the closest safe constant: none_failed_min_one_success -> NONE_FAILED so an upstream failure
-# never permits downstream execution; none_failed_or_skipped -> NONE_FAILED (skips are non-failures).
-# Airflow's default all_success maps to None (no run_if key -> Databricks default ALL_SUCCESS).
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TriggerRuleMapping:
+    """Databricks run-if mapping and its semantic confidence."""
+
+    rule: str
+    outcome: str | None
+    status: str
+    message: str | None = None
+
+
+# Exact mappings only. Rules outside this table must not collapse to ALL_SUCCESS.
 _TRIGGER_RULE_TO_RUN_IF: dict[str, str | None] = {
     "all_success": None,
     "all_done": "ALL_DONE",
@@ -318,24 +326,47 @@ def email_on_failure(dag_default_args: dict[str, ast.expr], task_kwargs: dict[st
     "one_failed": "AT_LEAST_ONE_FAILED",
     "one_success": "AT_LEAST_ONE_SUCCESS",
     "none_failed": "NONE_FAILED",
-    "none_failed_min_one_success": "NONE_FAILED",
     "none_failed_or_skipped": "NONE_FAILED",
-    "always": "ALL_DONE",
 }
 
+_UNSUPPORTED_TRIGGER_RULES = frozenset({"always", "dummy", "none_skipped", "all_skipped", "one_done"})
 
-def trigger_rule_outcome(task_kwargs: dict[str, ast.expr]) -> str | None:
-    """Maps a task's ``trigger_rule`` kwarg to a DAB ``run_if`` constant, or None (ALL_SUCCESS)."""
+
+def _trigger_rule_name(task_kwargs: dict[str, ast.expr]) -> str | None:
     node = task_kwargs.get("trigger_rule")
     if isinstance(node, ast.Constant) and isinstance(node.value, str):
-        rule = node.value
-    elif isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id == "TriggerRule":
-        rule = node.attr.lower()
-    else:
-        rule = None
-    if rule is None:
-        return None
-    return _TRIGGER_RULE_TO_RUN_IF.get(rule)
+        return node.value.lower()
+    if isinstance(node, ast.Attribute):
+        return node.attr.lower()
+    return None
+
+
+def trigger_rule_mapping(task_kwargs: dict[str, ast.expr]) -> TriggerRuleMapping:
+    """Classifies an Airflow trigger rule as exact, approximate, or unsupported."""
+    rule = _trigger_rule_name(task_kwargs) or "all_success"
+    if rule == "none_failed_min_one_success":
+        return TriggerRuleMapping(
+            rule=rule,
+            outcome="NONE_FAILED",
+            status="approximate",
+            message=(
+                "Databricks NONE_FAILED preserves the no-upstream-failure requirement but may run "
+                "when every upstream task was skipped or excluded."
+            ),
+        )
+    if rule in _TRIGGER_RULE_TO_RUN_IF:
+        return TriggerRuleMapping(rule=rule, outcome=_TRIGGER_RULE_TO_RUN_IF[rule], status="exact")
+    detail = (
+        "Databricks has no run_if predicate with equivalent skipped/not-run behavior."
+        if rule in _UNSUPPORTED_TRIGGER_RULES
+        else "The trigger rule is not recognized by the static Airflow translator."
+    )
+    return TriggerRuleMapping(rule=rule, outcome=None, status="unsupported", message=detail)
+
+
+def trigger_rule_outcome(task_kwargs: dict[str, ast.expr]) -> str | None:
+    """Maps a task's ``trigger_rule`` kwarg to a DAB ``run_if`` constant, or None (ALL_SUCCESS)."""
+    return trigger_rule_mapping(task_kwargs).outcome
 
 
 # --------------------------------------------------------------------------------------
diff --git a/tests/unit/test_airflow_production_readiness.py b/tests/unit/test_airflow_production_readiness.py
index 6f622d2..c6f0d1f 100644
--- a/tests/unit/test_airflow_production_readiness.py
+++ b/tests/unit/test_airflow_production_readiness.py
@@ -2,7 +2,8 @@
 
 from pathlib import Path
 
-from flowx.models.ir import NotebookActivity
+from flowx.models.ir import ForEachActivity, NotebookActivity, PlaceholderActivity, SparkPythonActivity
+from flowx.preparer.workflow_preparer import prepare_workflow
 from flowx.sources.airflow.loader import load_airflow_dag, load_airflow_dags
 
 _REPROS = Path(__file__).parents[1] / "resources" / "airflow" / "review_repros"
@@ -10,8 +11,7 @@
 
 def _dependencies(pipeline) -> dict[str, list[str]]:
     return {
-        task.task_key: sorted(dependency.task_key for dependency in (task.depends_on or []))
-        for task in pipeline.tasks
+        task.task_key: sorted(dependency.task_key for dependency in (task.depends_on or [])) for task in pipeline.tasks
     }
 
 
@@ -81,3 +81,144 @@ def test_literal_dag_factory_loop_and_multiple_assigned_dags_remain_distinct() -
     assert [pipeline.name for pipeline in generated] == ["etl_alpha", "etl_beta"]
     assert [pipeline.name for pipeline in assigned] == ["team_a_etl", "team_b_etl"]
     assert all([task.task_key for task in pipeline.tasks] == ["extract", "load"] for pipeline in assigned)
+
+
+def test_spark_submit_requires_a_single_invocation_and_known_option_arities() -> None:
+    pipeline = load_airflow_dag(_REPROS / "t2_sparksubmit.py")
+    first, second, third = pipeline.tasks
+
+    assert isinstance(first, SparkPythonActivity)
+    assert first.python_file == "/jobs/etl.py"
+    assert first.parameters == ["--date", "2024-01-01"]
+    assert isinstance(second, NotebookActivity)
+    assert "cd /opt/app && spark-submit" in (second.generated_source or "")
+    assert isinstance(third, NotebookActivity)
+    assert "spark-submit /jobs/x.py && aws" in (third.generated_source or "")
+
+
+def test_unknown_spark_submit_option_falls_back_to_bash(tmp_path: Path) -> None:
+    dag = tmp_path / "spark.py"
+    dag.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='spark') as dag:\n"
+        "    run = BashOperator(task_id='run', bash_command='spark-submit --future-option value app.py')\n",
+        encoding="utf-8",
+    )
+
+    task = load_airflow_dag(dag).tasks[0]
+
+    assert isinstance(task, NotebookActivity)
+    assert "--future-option value app.py" in (task.generated_source or "")
+
+
+def test_spark_submit_option_cannot_consume_another_option_as_its_value(tmp_path: Path) -> None:
+    dag = tmp_path / "spark_option_value.py"
+    dag.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='spark_option_value') as dag:\n"
+        "    run = BashOperator(\n"
+        "        task_id='run',\n"
+        "        bash_command='spark-submit --conf --driver-memory=4g app.py',\n"
+        "    )\n",
+        encoding="utf-8",
+    )
+
+    task = load_airflow_dag(dag).tasks[0]
+
+    assert isinstance(task, NotebookActivity)
+    assert "spark-submit --conf --driver-memory=4g app.py" in (task.generated_source or "")
+
+
+def test_unresolved_jinja_in_generated_source_becomes_placeholder() -> None:
+    pipeline = load_airflow_dag(_REPROS / "t4_bashjinja.py")
+
+    assert isinstance(pipeline.tasks[0], PlaceholderActivity)
+    assert pipeline.reconciliation_status == "verified_with_gaps"
+    assert any(finding["code"] == "unresolved_airflow_template" for finding in pipeline.not_translatable)
+
+
+def test_unsupported_and_approximate_trigger_rules_are_explicit(tmp_path: Path) -> None:
+    unsupported = load_airflow_dag(_REPROS / "t23_tr2.py")
+    assert all(isinstance(unsupported.tasks[index], PlaceholderActivity) for index in (1, 2, 3))
+
+    dag = tmp_path / "approximate.py"
+    dag.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='rules') as dag:\n"
+        "    up = BashOperator(task_id='up', bash_command='echo up')\n"
+        "    down = BashOperator(task_id='down', bash_command='echo down', "
+        "trigger_rule='none_failed_min_one_success')\n"
+        "    up >> down\n",
+        encoding="utf-8",
+    )
+    approximate = load_airflow_dag(dag)
+
+    assert approximate.tasks[1].depends_on[0].outcome == "NONE_FAILED"
+    finding = next(item for item in approximate.not_translatable if item["code"] == "approximated_trigger_rule")
+    assert "every upstream task was skipped or excluded" in finding["message"]
+
+
+def test_sensor_lift_requires_full_non_sensor_reachability(tmp_path: Path) -> None:
+    guarded = load_airflow_dag(_REPROS / "t24_sensorscope.py")
+    assert guarded.schedule is None
+    assert {task.task_key for task in guarded.tasks} == {"wait", "gated", "independent"}
+
+    dag = tmp_path / "dominating_sensor.py"
+    dag.write_text(
+        "from airflow import DAG\n"
+        "from airflow.sensors.filesystem import FileSensor\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='dominating', schedule=None) as dag:\n"
+        "    wait = FileSensor(task_id='wait', filepath='/mnt/input')\n"
+        "    work = BashOperator(task_id='work', bash_command='echo work')\n"
+        "    wait >> work\n",
+        encoding="utf-8",
+    )
+    dominating = load_airflow_dag(dag)
+
+    assert (dominating.schedule or {})["kind"] == "file_arrival"
+    proof = next(item for item in dominating.audit["transformations"] if item["code"].startswith("sensor_lift"))
+    assert proof["covered_capture_ids"] == ["work"]
+
+
+def test_classic_mapping_with_unbound_args_links_a_failing_placeholder() -> None:
+    pipeline = load_airflow_dag(_REPROS / "a8_classic_mapping.py")
+    outer = pipeline.tasks[0]
+
+    assert isinstance(outer, ForEachActivity)
+    assert isinstance(outer.inner_activities[0], PlaceholderActivity)
+    assert any(finding["code"] == "classic_mapping_arguments_unbound" for finding in pipeline.not_translatable)
+
+    prepared = prepare_workflow(pipeline)
+    inner_task = prepared.tasks[0]["for_each_task"]["task"]
+    notebook_path = inner_task["notebook_task"]["notebook_path"]
+    notebook = next(item for item in prepared.notebooks if notebook_path.endswith(item.relative_path))
+    assert "raise NotImplementedError" in notebook.content
+
+
+def test_shell_notebook_directives_remain_inert() -> None:
+    magic = load_airflow_dag(_REPROS / "t15_magic.py").tasks[0]
+    boundary = load_airflow_dag(_REPROS / "t31_inject.py").tasks[0]
+
+    assert isinstance(magic, NotebookActivity)
+    assert isinstance(boundary, NotebookActivity)
+    magic_source = magic.generated_source or ""
+    boundary_source = boundary.generated_source or ""
+    assert magic_source.count("# MAGIC %sh") == 1
+    assert "# MAGIC ## MAGIC %sql" in magic_source
+    assert boundary_source.count("# MAGIC %sh") == 1
+    assert "# MAGIC ## COMMAND ----------" in boundary_source
+    assert "# MAGIC echo one" in boundary_source
+    assert "# MAGIC echo two" in boundary_source
+
+
+def test_unconsumed_operator_arguments_become_placeholder() -> None:
+    pipeline = load_airflow_dag(_REPROS / "t29_dagsem.py")
+    task = next(task for task in pipeline.tasks if task.task_key == "a")
+
+    assert isinstance(task, PlaceholderActivity)
+    finding = next(item for item in pipeline.not_translatable if item["code"] == "unconsumed_operator_arguments")
+    assert finding["details"]["arguments"] == ["pool", "priority_weight", "queue"]

From 126afbaf79077d6fcd067afe88b2eca637ca5de6 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Fri, 7 Aug 2026 14:24:46 -0700
Subject: [PATCH 52/77] Report audited Airflow migration coverage

---
 README.md                                     |  15 +-
 docs/content/docs/architecture.mdx            |   2 +-
 docs/content/docs/guide.mdx                   |   4 +
 docs/content/docs/options.mdx                 |   6 +-
 .../flowx-convert/sources/airflow-coverage.md |  23 +--
 src/flowx/adapter/__main__.py                 |   1 +
 src/flowx/mcp/server.py                       |   6 +-
 src/flowx/reporting/coverage.py               |  42 +++++-
 src/flowx/reporting/dashboard_template.json   |  76 +++++++---
 src/flowx/reporting/results.py                |  39 ++++-
 src/flowx/sources/airflow/audit.py            |  31 ++--
 src/flowx/sources/airflow/discover.py         |  80 +++++++++--
 src/flowx/sources/airflow/operators.py        |  45 ++++--
 tests/unit/test_airflow_adapter_reporting.py  |  70 +++++++++
 tests/unit/test_airflow_operators.py          |   2 +-
 .../unit/test_airflow_production_readiness.py | 134 ++++++++++++++++++
 tests/unit/test_reporting_coverage.py         |  81 +++++++++++
 tests/unit/test_reporting_dashboard.py        |   5 +
 tests/unit/test_reporting_results.py          |  92 +++++++++++-
 tests/unit/test_source_router.py              |   4 +-
 20 files changed, 665 insertions(+), 93 deletions(-)

diff --git a/README.md b/README.md
index 1dfb647..f4f22ab 100644
--- a/README.md
+++ b/README.md
@@ -157,7 +157,7 @@ execution) and maps ~35 operator/sensor families to the shared IR. Highlights:
   …) → `sql_task`.
 - **TaskFlow API** — `@dag` / `@task`; implicit XCom data flow lowers to `dbutils.jobs.taskValues`.
   `@task.expand([literal])` → `for_each_task`; non-literal / `.partial().expand()` / `@task_group` →
-  placeholder + gap (dependencies preserved — never a silent drop).
+  a linked placeholder notebook that raises `NotImplementedError`.
 - **Sensors** — file/table/time sensors → job triggers or polling notebooks; `ExternalTaskSensor` →
   cross-DAG wait; Http/Python/DateTime → polling tasks.
 - **dbt** — dbt CLI operators and astronomer-cosmos `DbtDag` / `DbtTaskGroup` → a dbt-factory job
@@ -168,10 +168,19 @@ execution) and maps ~35 operator/sensor families to the shared IR. Highlights:
 Operators without a deterministic mapping become a placeholder recorded in `gaps.json` for the
 agentic round. Full matrix: [`skills/flowx-convert/sources/airflow-coverage.md`](skills/flowx-convert/sources/airflow-coverage.md).
 
+Airflow discovery independently audits DAG declarations, task candidates, dependency declarations,
+DAG settings, mapped calls, and operator arguments before comparing them with captured IR. An
+included DAG is `verified` when every audited construct has a proven translation,
+`verified_with_gaps` when every unsupported construct is linked to a runnable-failure placeholder,
+or `failed` when reconciliation finds unexplained loss. Failed reconciliation exits nonzero and
+blocks package writes. `--exclude-dag ` is repeatable; excluded DAGs emit no Job but remain
+visible with zero translated activities in inventory and coverage reporting. This guarantee applies
+to the supported static subset; flowx never imports or executes DAG modules.
+
 ## How It Works
 
 ### Phase 1: Discover
-Parses the source into typed nodes and classifies each activity/operator as deterministic, agentic, or unsupported — ADF JSON from Unity Catalog volumes (or a `/Workspace` Git folder, normalizing ARM template format), or Airflow DAG `.py` modules read statically with `ast`. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`.
+Parses the source into typed nodes and classifies each activity/operator as deterministic, agentic, or unsupported — ADF JSON from Unity Catalog volumes (or a `/Workspace` Git folder, normalizing ARM template format), or Airflow DAG `.py` modules read statically with `ast`. Airflow inventory includes audited/deterministic/agentic/failed/excluded counts, reconciliation status, stable finding fingerprints, translation-path coverage, and deterministic coverage. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`.
 
 ### Phase 2: Convert
 Applies deterministic translators (ADF activity registry / Airflow operator mapping), resolves dependencies, and flags agentic gaps for LLM-assisted translation. Produces the shared Pipeline IR consumed unchanged by the package phase.
@@ -188,7 +197,7 @@ flowx_output/
   databricks.yml              # Bundle configuration (package)
   resources/
     jobs/
-      .yml     # One job per ADF pipeline
+      .yml     # One Job per included ADF pipeline or Airflow DAG
   src/
     notebooks/
       /
diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx
index fbdb0f8..99d6b9b 100644
--- a/docs/content/docs/architecture.mdx
+++ b/docs/content/docs/architecture.mdx
@@ -28,7 +28,7 @@ Each activity is classified with a `TranslationStrategy`:
 * `AGENTIC` (LLM-assisted gaps)
 * `UNSUPPORTED`
 
-The reporting layer (`reporting/`) can write per-run coverage to a Unity Catalog table and publish an AI/BI dashboard.
+The reporting layer (`reporting/`) can write per-run coverage to a Unity Catalog table and publish an AI/BI dashboard. Airflow rows use independently audited candidates as the denominator and persist reconciliation status, failed/excluded counts, stable finding fingerprints, translation-path coverage, and deterministic coverage.
 
 ## Two surfaces over one core
 
diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx
index d758899..e263f02 100644
--- a/docs/content/docs/guide.mdx
+++ b/docs/content/docs/guide.mdx
@@ -82,6 +82,10 @@ When running with workspace auth (e.g. Genie Code), `package` can optionally per
 coverage to a Unity Catalog table — one row per pipeline stamped with a UUID `run_id`, `run_date`,
 and `run_by` (`record-results`) — and install a published AI/BI coverage dashboard over that table
 (`install-dashboard`). See [Configuration options](/docs/options) for details.
+
+For Airflow, `activities` is the independent source-audit count rather than the number of tasks the
+translator happened to emit. Reporting distinguishes deterministic, agentic, failed, and excluded
+candidates and carries reconciliation status plus both translation-path and deterministic coverage.
 
 
 
diff --git a/docs/content/docs/options.mdx b/docs/content/docs/options.mdx
index bbfa4f0..cc4f1e7 100644
--- a/docs/content/docs/options.mdx
+++ b/docs/content/docs/options.mdx
@@ -114,11 +114,13 @@ phase surfaces three optional inputs — `results_table`, `results_warehouse_id`
 
 - **`record-results`** writes one row **per pipeline per run** to the supplied Unity Catalog
   table (`catalog.schema.table`), combining the complexity columns above with the
-  deterministic/agentic/unsupported coverage breakdown. Every row is stamped with a shared
+  audited/deterministic/agentic/failed/excluded coverage breakdown, reconciliation and migration
+  status, finding fingerprints, translation-path coverage, and deterministic coverage. Airflow's
+  audited count remains the denominator even for failed or excluded candidates. Every row is stamped with a shared
   **`run_id`** (UUID), **`run_date`** (`CURRENT_TIMESTAMP()`), and **`run_by`**
   (`CURRENT_USER()`), so coverage is trackable across runs and users.
 - **`install-dashboard`** creates and publishes an AI/BI (Lakeview) dashboard over that table —
-  KPI counters (pipelines, coverage %, deterministic/agentic/unsupported activity totals), a
+  KPI counters (pipelines, audited activities, and coverage), failed/excluded totals, a
   pipelines-by-complexity bar chart, a coverage-over-runs line, and a per-pipeline coverage
   table.
 
diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md
index acee272..876218a 100644
--- a/skills/flowx-convert/sources/airflow-coverage.md
+++ b/skills/flowx-convert/sources/airflow-coverage.md
@@ -14,7 +14,7 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't
 | `PythonOperator` (classic) | Notebook task; callable `def` preserved, transitive helpers/constants/non-Airflow imports carried, `op_args`/`op_kwargs` passed as JSON widgets, return value via `dbutils.jobs.taskValues.set`. |
 | `PythonVirtualenvOperator` / `ExternalPythonOperator` | Notebook task with a `%pip install` cell for `requirements`. |
 | `BranchPythonOperator` / `ShortCircuitOperator` | Placeholder routed to the agentic-gap round (runtime branch selection can't be lowered statically). |
-| `BashOperator` / `SSHOperator` | `%sh` notebook; a wrapped `spark-submit` is lifted to a Spark JAR/Python task. |
+| `BashOperator` / `SSHOperator` | `%sh` notebook; a single unchained `spark-submit` invocation is lifted only when every option arity is known. |
 | `SparkSubmitOperator` | Spark JAR or Python task. |
 | Databricks provider operators (`DatabricksSubmitRun*`, `DatabricksRunNow*`, `DatabricksNotebookOperator`) | Notebook / run-job tasks. |
 | SQL operators (`DatabricksSql*`, `SQLExecuteQueryOperator`, `PostgresOperator`, `MySqlOperator`, `HiveOperator`, `DatabricksCopyIntoOperator`) | `sql_task` (SqlActivity); Jinja values → `:name`, identifier positions → `IDENTIFIER(:name)`, with `sql_task.parameters`. |
@@ -22,18 +22,19 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't
 | `EmailOperator` | Placeholder recommending job-level email notifications. |
 | dbt CLI operators (`DbtRun/Test/Seed/Snapshot/Build/Deps`) and Cosmos `DbtDag` / `DbtTaskGroup` | Single `DbtFactoryActivity`, **static explosion** (default) or **PyDABs** (`--dbt-mode pydabs`); see [dbt factory](#dbt-factory-mode). |
 | **TaskFlow API** (`@dag`, `@task`, `@task.virtualenv`) | Each `@task` invocation → a task; implicit XCom data flow (`transform(extract())`) → a notebook that reads upstream return values via `dbutils.jobs.taskValues.get`, calls the function, and publishes its own. `@task.branch` / `@task.short_circuit`, or a callable reading task context/XCom, route to a placeholder + gap. |
-| File sensors (`S3KeySensor`, `GCSObjectExistenceSensor`, `FileSensor`, `HdfsSensor`, `WebHdfsSensor`) | Root sensor with no schedule → `file_arrival` trigger; otherwise a `dbutils.fs` polling notebook task. |
-| Table/SQL sensors (`DatabricksPartitionSensor`, `DatabricksSqlSensor`, `DatabricksSQLStatementsSensor`, `SqlSensor`) | Root sensor naming a literal table with no schedule → `table_update` trigger; otherwise a `spark.sql` polling notebook task. |
+| File sensors (`S3KeySensor`, `GCSObjectExistenceSensor`, `FileSensor`, `HdfsSensor`, `WebHdfsSensor`) | With no schedule, a root sensor whose descendants cover every non-sensor task → `file_arrival` trigger; otherwise a `dbutils.fs` polling notebook task. |
+| Table/SQL sensors (`DatabricksPartitionSensor`, `DatabricksSqlSensor`, `DatabricksSQLStatementsSensor`, `SqlSensor`) | With no schedule, a root literal-table sensor whose descendants cover every non-sensor task → `table_update` trigger; otherwise a `spark.sql` polling notebook task. |
 | `ExternalTaskSensor` | Placeholder explaining logical-run-aware migration options; polling the latest Databricks job run is not equivalent to Airflow's matching logical run. |
 | `HttpSensor` / `PythonSensor` / `DateTimeSensor` | Polling notebook tasks for absolute HTTP URLs, callable polls, and wait-until. Relative HTTP endpoints and Python callables reading task context route to placeholders. |
 | Time sensors (`TimeSensor`, `TimeDeltaSensor`) | Placeholder; their per-run wait semantics are not silently folded into or removed from the job schedule. |
 | `DummyOperator` / `EmptyOperator` | Dropped, downstream dependencies rewired. |
-| `.expand()` on an operator or `@task` | `for_each_task` when the mapped iterable is a literal list; a non-literal iterable (e.g. an upstream task's output) routes to a placeholder + gap. |
+| `.expand()` on `@task` | `for_each_task` when exactly one mapped argument is a literal list and no `.partial()` arguments are present; other forms route to a placeholder + gap. |
+| Classic operator `.partial().expand()` / `.expand()` | `for_each_task` containing a linked failing placeholder until every mapped and fixed argument can be proven bound into the inner Databricks task. |
 | Dependencies | `>>` / `<<` chains (incl. list/tuple fan-out and inline TaskFlow calls) and `set_upstream` / `set_downstream`. |
 | **TaskGroups** (context-manager `with TaskGroup(...)`) | Static nesting → task-key namespacing (`group__subgroup__task`); group-level edges (`group_a >> group_b`, `task >> group`) expand to leaf→root edges between member tasks. |
 | **`@task_group`** (decorator form) | Placeholder + gap (with dependency edges preserved); a decorator group is a sub-pipeline flowx doesn't lower — the agentic round expands it into its member tasks / a for_each when mapped. |
 | Schedule | Cron `schedule_interval` → Quartz (Unix DOW 0–6 → Quartz 1–7); exact sub-hour `timedelta` → Quartz, longer intervals → periodic, `@continuous` → continuous mode. |
-| `trigger_rule` | DAB `run_if` constant per edge (`ALL_DONE`, `ALL_FAILED`, `AT_LEAST_ONE_SUCCESS`, `NONE_FAILED`, …). |
+| `trigger_rule` | Exact supported rules map to `run_if`; `none_failed_min_one_success` maps to `NONE_FAILED` with the all-skipped delta recorded. Rules without an equivalent become linked placeholders. |
 | Job parameters | `params={...}` / `Param(default=...)` → job parameters with defaults; `{{ params.x }}` / `{{ var.value.x }}` / `{{ dag_run.conf['x'] }}` → `{{job.parameters.x}}`. |
 | `Variable.get` in a callable | Rewritten to `dbutils.widgets.get`; a callable using an Airflow `Connection` object routes to a placeholder because one secret string cannot preserve the object API. |
 | Multiple DAGs | Every DAG, including multiple declarations in one Python file, becomes a sibling job in one shared Airflow bundle so `TriggerDagRunOperator` resource references resolve. |
@@ -46,13 +47,17 @@ emitting code that fails at runtime.
 
 ## Not yet supported
 
-These are absent but fail safely — routed to a placeholder + `gaps.json`, or simply not exploded — or
-are deliberate scope decisions.
+These are absent but fail safely — routed to a linked placeholder notebook that raises
+`NotImplementedError`, explicitly excluded, or rejected by reconciliation — or are deliberate scope
+decisions.
 
 - **Full TaskGroup expansion** — a `@task_group` invocation (mapped `pair.expand(...)` or plain
   `pair(...)`) and `TaskGroup.partial().expand()` aren't lowered into their member tasks. They route
-  to a placeholder + gap with dependency edges preserved (never a silent drop); the agentic round
-  expands the group. `.expand()` on an operator/`@task` *call* is supported (see the table).
+  to a placeholder + gap with dependency edges preserved; the agentic round expands the group.
+- **Dynamic operator construction** — operators created inside comprehensions are not statically
+  expanded. Helper factories are supported only when their body is an optional docstring followed
+  by one statically bindable `return RecognizedOperator(...)`; other forms fail reconciliation and
+  block package output.
 - **Sensors beyond the mapped families** (`S3PrefixSensor`, custom sensors, etc.) → placeholder + gap.
   A file sensor with a non-literal path, or a table/SQL sensor with no literal `sql` / `table_name`,
   also falls back to a placeholder.
diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py
index b0ac1e4..83e57d9 100644
--- a/src/flowx/adapter/__main__.py
+++ b/src/flowx/adapter/__main__.py
@@ -514,6 +514,7 @@ def _run_phase(phase: str, forward: list[str]) -> int:
         aliases = {_SOURCE_PATH_FLAG: "--source-dir", source.source_path_flag: "--source-dir"}
 
     module = importlib.import_module(module_path)
+
     # Alias both the bare form (`--source-path X`) and the equals form (`--source-path=X`) so a
     # documented alias works either way; the phase module only knows `--source-dir`.
     def _alias(token: str) -> str:
diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py
index 8dfe9ee..f751e36 100644
--- a/src/flowx/mcp/server.py
+++ b/src/flowx/mcp/server.py
@@ -475,8 +475,9 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A
         - "inputs": phase(req: "discover"|"convert"|"package"), source(req for discover/convert) —
           list a phase's input prompts.
         - "discover": source(req), one ADF source key | airflow_source_path (req), output_dir,
-          pipeline — parse the source's definitions, classify activities.
-        - "convert": source(req), (one ADF source key | airflow_source_path), output_dir, pipeline.
+          pipeline, exclude_dag | exclude_dags (Airflow, repeatable list) — parse and audit definitions.
+        - "convert": source(req), (one ADF source key | airflow_source_path), output_dir, pipeline,
+          exclude_dag | exclude_dags (Airflow, repeatable list).
         - "merge_agentic": source(req), report_path(req), agentic_results_dir(req), output_path —
           merge agent results.
         - "inspect": report_path(req) — return the full translation-option schema (every option with
@@ -490,6 +491,7 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A
           download_workspace_files(bool), keep_intermediates(bool).
         - "migrate": source(req), one ADF source key | airflow_source_path (req), output_dir,
           output_volume_path, output_workspace_path, catalog, schema, pipeline,
+          exclude_dag | exclude_dags (Airflow, repeatable list),
           answers(list of "ID=VALUE"), interactive(bool, default true), lookup_csv — runs
           discover→convert→package, returning the full option schema once (status "needs_input") when
           configuration is available; re-call once with the complete answers to apply (see below).
diff --git a/src/flowx/reporting/coverage.py b/src/flowx/reporting/coverage.py
index 356c562..ca41d23 100644
--- a/src/flowx/reporting/coverage.py
+++ b/src/flowx/reporting/coverage.py
@@ -4,8 +4,8 @@
 
 * ``profile_report.csv`` -- per-pipeline complexity (activity/dataset/linked-service
   counts, collapsible patterns, activity-category counts, complexity score + size).
-* ``inventory.json`` -- per-activity translation strategy, from which the
-  deterministic / agentic / unsupported counts and coverage % are derived.
+* ``inventory.json`` -- per-activity translation strategy plus source-audit counts
+  and reconciliation status when the source supports independent auditing.
 
 The result is one metric row per pipeline (no run metadata -- ``run_id`` /
 ``run_date`` / ``run_by`` are stamped on at write time by :mod:`reporting.results`).
@@ -22,6 +22,7 @@
 COVERAGE_METRIC_COLUMNS: tuple[str, ...] = (
     "pipeline",
     "activities",
+    "audited_activities",
     "datasets",
     "linked_services",
     "collapsible_patterns",
@@ -31,7 +32,14 @@
     "deterministic_activities",
     "agentic_activities",
     "unsupported_activities",
+    "failed_activities",
+    "excluded_activities",
+    "reconciliation_status",
+    "migration_status",
     "coverage_pct",
+    "deterministic_coverage_pct",
+    "finding_count",
+    "finding_fingerprints",
     "complexity_score",
     "complexity_size",
 )
@@ -54,6 +62,13 @@ def _coverage_pct(deterministic: int, agentic: int, total: int) -> float:
     return round((deterministic + agentic) / total * 100, 1)
 
 
+def _deterministic_coverage_pct(deterministic: int, total: int) -> float:
+    """Deterministic coverage over audited activity candidates, rounded to 1dp."""
+    if total <= 0:
+        return 0.0
+    return round(deterministic / total * 100, 1)
+
+
 def build_coverage_rows(metadata_dir: Path) -> list[dict[str, Any]]:
     """Builds per-pipeline coverage rows from a migration ``metadata/`` directory.
 
@@ -83,10 +98,19 @@ def build_coverage_rows(metadata_dir: Path) -> list[dict[str, Any]]:
     for pipeline in inventory.get("pipelines", []):
         name = pipeline.get("name", "")
         strategies = [activity.get("strategy") for activity in pipeline.get("activities", [])]
-        deterministic = strategies.count("deterministic")
-        agentic = strategies.count("agentic")
+        has_audit = "audited_activity_count" in pipeline
+        deterministic = int(pipeline.get("deterministic_count", 0)) if has_audit else strategies.count("deterministic")
+        agentic = int(pipeline.get("agentic_count", 0)) if has_audit else strategies.count("agentic")
         unsupported = strategies.count("unsupported")
-        total = len(strategies)
+        failed = int(pipeline.get("failed_count", 0)) if has_audit else 0
+        excluded = int(pipeline.get("excluded_count", 0)) if has_audit else 0
+        total = int(pipeline.get("audited_activity_count", 0)) if has_audit else len(strategies)
+        findings = pipeline.get("findings", [])
+        fingerprints = [
+            finding["fingerprint"]
+            for finding in findings
+            if isinstance(finding, dict) and isinstance(finding.get("fingerprint"), str)
+        ]
         csv_row = csv_by_pipeline.get(name, {})
 
         def _csv_int(col: str, _csv_row: dict[str, str] = csv_row) -> int:
@@ -99,6 +123,7 @@ def _csv_int(col: str, _csv_row: dict[str, str] = csv_row) -> int:
             {
                 "pipeline": name,
                 "activities": total,
+                "audited_activities": total,
                 "datasets": _csv_int("datasets"),
                 "linked_services": _csv_int("linked_services"),
                 "collapsible_patterns": _csv_int("collapsible_patterns"),
@@ -108,7 +133,14 @@ def _csv_int(col: str, _csv_row: dict[str, str] = csv_row) -> int:
                 "deterministic_activities": deterministic,
                 "agentic_activities": agentic,
                 "unsupported_activities": unsupported,
+                "failed_activities": failed,
+                "excluded_activities": excluded,
+                "reconciliation_status": pipeline.get("reconciliation_status", "not_applicable"),
+                "migration_status": pipeline.get("migration_status", "included"),
                 "coverage_pct": _coverage_pct(deterministic, agentic, total),
+                "deterministic_coverage_pct": _deterministic_coverage_pct(deterministic, total),
+                "finding_count": len(findings),
+                "finding_fingerprints": json.dumps(fingerprints, separators=(",", ":")),
                 "complexity_score": _csv_int("complexity_score"),
                 "complexity_size": csv_row.get("complexity_size", "") or "",
             }
diff --git a/src/flowx/reporting/dashboard_template.json b/src/flowx/reporting/dashboard_template.json
index 25ed878..d6ca7ee 100644
--- a/src/flowx/reporting/dashboard_template.json
+++ b/src/flowx/reporting/dashboard_template.json
@@ -5,11 +5,14 @@
       "displayName": "Latest run summary",
       "queryLines": [
         "SELECT COUNT(*) AS pipelines, ",
-        "SUM(activities) AS activities, ",
+        "SUM(audited_activities) AS audited_activities, ",
         "SUM(deterministic_activities) AS deterministic_activities, ",
         "SUM(agentic_activities) AS agentic_activities, ",
         "SUM(unsupported_activities) AS unsupported_activities, ",
-        "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(activities),0),1) AS coverage_pct ",
+        "SUM(failed_activities) AS failed_activities, ",
+        "SUM(excluded_activities) AS excluded_activities, ",
+        "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(audited_activities),0),1) AS coverage_pct, ",
+        "ROUND(100.0*SUM(deterministic_activities)/NULLIF(SUM(audited_activities),0),1) AS deterministic_coverage_pct ",
         "FROM {{RESULTS_TABLE}} ",
         "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1)"
       ]
@@ -28,11 +31,12 @@
       "name": "latest_pipelines",
       "displayName": "Pipeline coverage (latest run)",
       "queryLines": [
-        "SELECT pipeline, activities, deterministic_activities, agentic_activities, ",
-        "unsupported_activities, coverage_pct, collapsible_patterns, complexity_size ",
+        "SELECT pipeline, audited_activities, deterministic_activities, agentic_activities, ",
+        "unsupported_activities, failed_activities, excluded_activities, reconciliation_status, ",
+        "migration_status, coverage_pct, deterministic_coverage_pct, finding_count, collapsible_patterns, complexity_size ",
         "FROM {{RESULTS_TABLE}} ",
         "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1) ",
-        "ORDER BY coverage_pct ASC, activities DESC"
+        "ORDER BY coverage_pct ASC, audited_activities DESC"
       ]
     },
     {
@@ -40,8 +44,10 @@
       "displayName": "Coverage over runs",
       "queryLines": [
         "SELECT DATE_TRUNC('SECOND', run_date) AS run_ts, ",
-        "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(activities),0),1) AS coverage_pct, ",
-        "SUM(activities) AS activities ",
+        "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(audited_activities),0),1) AS coverage_pct, ",
+        "ROUND(100.0*SUM(deterministic_activities)/NULLIF(SUM(audited_activities),0),1) AS deterministic_coverage_pct, ",
+        "SUM(audited_activities) AS audited_activities, SUM(failed_activities) AS failed_activities, ",
+        "SUM(excluded_activities) AS excluded_activities ",
         "FROM {{RESULTS_TABLE}} ",
         "GROUP BY DATE_TRUNC('SECOND', run_date) ",
         "ORDER BY run_ts"
@@ -59,7 +65,7 @@
             "name": "title",
             "multilineTextboxSpec": {
               "lines": [
-                "## ADF \u2192 Databricks Migration Coverage"
+                "## flowx Migration Coverage"
               ]
             }
           },
@@ -75,7 +81,7 @@
             "name": "subtitle",
             "multilineTextboxSpec": {
               "lines": [
-                "Per-pipeline translation coverage from the latest flowx run. Coverage % = (deterministic + agentic) / total activities."
+                "Per-pipeline translation coverage from the latest flowx run. Coverage % = (deterministic + agentic) / audited activities."
               ]
             }
           },
@@ -176,8 +182,8 @@
                   "datasetName": "latest_summary",
                   "fields": [
                     {
-                      "name": "activities",
-                      "expression": "`activities`"
+                      "name": "audited_activities",
+                      "expression": "`audited_activities`"
                     }
                   ],
                   "disaggregated": true
@@ -189,12 +195,12 @@
               "widgetType": "counter",
               "encodings": {
                 "value": {
-                  "fieldName": "activities",
-                  "displayName": "Activities"
+                  "fieldName": "audited_activities",
+                  "displayName": "Audited activities"
                 }
               },
               "frame": {
-                "title": "Activities",
+                "title": "Audited activities",
                 "showTitle": true
               }
             }
@@ -448,8 +454,8 @@
                       "expression": "`pipeline`"
                     },
                     {
-                      "name": "activities",
-                      "expression": "`activities`"
+                      "name": "audited_activities",
+                      "expression": "`audited_activities`"
                     },
                     {
                       "name": "deterministic_activities",
@@ -460,8 +466,20 @@
                       "expression": "`agentic_activities`"
                     },
                     {
-                      "name": "unsupported_activities",
-                      "expression": "`unsupported_activities`"
+                      "name": "failed_activities",
+                      "expression": "`failed_activities`"
+                    },
+                    {
+                      "name": "excluded_activities",
+                      "expression": "`excluded_activities`"
+                    },
+                    {
+                      "name": "reconciliation_status",
+                      "expression": "`reconciliation_status`"
+                    },
+                    {
+                      "name": "deterministic_coverage_pct",
+                      "expression": "`deterministic_coverage_pct`"
                     },
                     {
                       "name": "coverage_pct",
@@ -490,8 +508,8 @@
                     "displayName": "Pipeline"
                   },
                   {
-                    "fieldName": "activities",
-                    "displayName": "Activities"
+                    "fieldName": "audited_activities",
+                    "displayName": "Audited"
                   },
                   {
                     "fieldName": "deterministic_activities",
@@ -502,8 +520,20 @@
                     "displayName": "Agentic"
                   },
                   {
-                    "fieldName": "unsupported_activities",
-                    "displayName": "Unsupported"
+                    "fieldName": "failed_activities",
+                    "displayName": "Failed"
+                  },
+                  {
+                    "fieldName": "excluded_activities",
+                    "displayName": "Excluded"
+                  },
+                  {
+                    "fieldName": "reconciliation_status",
+                    "displayName": "Reconciliation"
+                  },
+                  {
+                    "fieldName": "deterministic_coverage_pct",
+                    "displayName": "Deterministic %"
                   },
                   {
                     "fieldName": "coverage_pct",
@@ -535,4 +565,4 @@
       ]
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/src/flowx/reporting/results.py b/src/flowx/reporting/results.py
index b5d3af7..f9d7d4a 100644
--- a/src/flowx/reporting/results.py
+++ b/src/flowx/reporting/results.py
@@ -23,6 +23,7 @@
 _METRIC_SQL_TYPES: dict[str, str] = {
     "pipeline": "STRING",
     "activities": "INT",
+    "audited_activities": "INT",
     "datasets": "INT",
     "linked_services": "INT",
     "collapsible_patterns": "INT",
@@ -32,7 +33,14 @@
     "deterministic_activities": "INT",
     "agentic_activities": "INT",
     "unsupported_activities": "INT",
+    "failed_activities": "INT",
+    "excluded_activities": "INT",
+    "reconciliation_status": "STRING",
+    "migration_status": "STRING",
     "coverage_pct": "DOUBLE",
+    "deterministic_coverage_pct": "DOUBLE",
+    "finding_count": "INT",
+    "finding_fingerprints": "STRING",
     "complexity_score": "INT",
     "complexity_size": "STRING",
 }
@@ -44,7 +52,10 @@
     *((col, _METRIC_SQL_TYPES[col]) for col in COVERAGE_METRIC_COLUMNS),
 )
 
-_STRING_METRICS: frozenset[str] = frozenset({"pipeline", "complexity_size"})
+_STRING_METRICS: frozenset[str] = frozenset(
+    {"pipeline", "reconciliation_status", "migration_status", "finding_fingerprints", "complexity_size"}
+)
+_FLOAT_METRICS: frozenset[str] = frozenset({"coverage_pct", "deterministic_coverage_pct"})
 
 
 def _sql_str(value: Any) -> str:
@@ -56,7 +67,7 @@ def _metric_value_sql(column: str, value: Any) -> str:
     """Renders one metric column value as a SQL literal."""
     if column in _STRING_METRICS:
         return _sql_str(value)
-    if column == "coverage_pct":
+    if column in _FLOAT_METRICS:
         return repr(float(value or 0))
     return str(int(value or 0))
 
@@ -67,6 +78,16 @@ def build_create_table_sql(table_fqn: str) -> str:
     return f"CREATE TABLE IF NOT EXISTS {table_fqn} (\n  {cols}\n)"
 
 
+def build_add_columns_sql(table_fqn: str, existing_columns: set[str]) -> str:
+    """Returns an ALTER TABLE statement for result columns absent from an existing table."""
+    normalized_existing = {name.lower() for name in existing_columns}
+    missing = [(name, sql_type) for name, sql_type in RESULTS_COLUMNS if name.lower() not in normalized_existing]
+    if not missing:
+        return ""
+    columns = ",\n  ".join(f"{name} {sql_type}" for name, sql_type in missing)
+    return f"ALTER TABLE {table_fqn} ADD COLUMNS (\n  {columns}\n)"
+
+
 def build_insert_sql(table_fqn: str, rows: list[dict[str, Any]], run_id: str) -> str:
     """Returns a single multi-row ``INSERT`` stamping run metadata onto every row.
 
@@ -121,7 +142,7 @@ def _rank(warehouse: Any) -> tuple[int, int]:
     return best.id
 
 
-def _execute(client: Any, statement: str, warehouse_id: str) -> None:
+def _execute(client: Any, statement: str, warehouse_id: str) -> Any:
     """Runs a SQL statement via the Statement Execution API; raises on failure."""
     resp = client.statement_execution.execute_statement(
         statement=statement, warehouse_id=warehouse_id, wait_timeout="50s"
@@ -131,6 +152,14 @@ def _execute(client: Any, statement: str, warehouse_id: str) -> None:
     if state_str.upper() not in ("", "SUCCEEDED"):
         err = getattr(getattr(resp, "status", None), "error", None)
         raise RuntimeError(f"Statement failed ({state_str}): {getattr(err, 'message', err)}")
+    return resp
+
+
+def _existing_columns(client: Any, table_fqn: str, warehouse_id: str) -> set[str]:
+    """Reads the current table column names through Databricks SQL."""
+    response = _execute(client, f"SHOW COLUMNS IN {table_fqn}", warehouse_id)
+    data = getattr(getattr(response, "result", None), "data_array", None) or []
+    return {str(row[0]) for row in data if row}
 
 
 def write_results(
@@ -166,6 +195,10 @@ def write_results(
     resolved_wh = resolve_warehouse_id(client, warehouse_id)
     run_id = str(uuid.uuid4())
     _execute(client, build_create_table_sql(table_fqn), resolved_wh)
+    existing_columns = _existing_columns(client, table_fqn, resolved_wh)
+    add_columns_sql = build_add_columns_sql(table_fqn, existing_columns)
+    if add_columns_sql:
+        _execute(client, add_columns_sql, resolved_wh)
     _execute(client, build_insert_sql(table_fqn, rows, run_id), resolved_wh)
     logger.info("Recorded %d pipeline rows to %s (run_id=%s).", len(rows), table_fqn, run_id)
     return run_id, len(rows)
diff --git a/src/flowx/sources/airflow/audit.py b/src/flowx/sources/airflow/audit.py
index e41fcb6..45d3c39 100644
--- a/src/flowx/sources/airflow/audit.py
+++ b/src/flowx/sources/airflow/audit.py
@@ -18,6 +18,8 @@ class AuditCandidate:
     line: int
     column: int
     occurrence: int
+    end_line: int = 0
+    end_column: int = 0
     details: dict[str, Any] = field(default_factory=dict)
 
 
@@ -43,7 +45,9 @@ def finding(
     """Builds a stable, serializable reconciliation finding."""
     line = candidate.line if candidate else 0
     column = candidate.column if candidate else 0
-    identity = f"{source_file}:{line}:{column}:{code}"
+    end_line = candidate.end_line if candidate else 0
+    end_column = candidate.end_column if candidate else 0
+    identity = f"{source_file}:{line}:{column}:{end_line}:{end_column}:{code}"
     return {
         "fingerprint": hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16],
         "code": code,
@@ -52,6 +56,8 @@ def finding(
         "source_file": source_file,
         "line": line,
         "column": column,
+        "end_line": end_line,
+        "end_column": end_column,
         "details": {**(candidate.details if candidate else {}), **(details or {})},
     }
 
@@ -89,9 +95,7 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None) -> No
             if isinstance(node, ast.FunctionDef) and _decorator_leaf(node) in _TASK_DECORATORS
         }
         self.dag_defs = {
-            node.name
-            for node in module.body
-            if isinstance(node, ast.FunctionDef) and _decorator_leaf(node) == "dag"
+            node.name for node in module.body if isinstance(node, ast.FunctionDef) and _decorator_leaf(node) == "dag"
         }
         self.factories = {
             node.name
@@ -109,6 +113,8 @@ def _candidate(self, kind: str, code: str, node: ast.AST, **details: Any) -> Aud
             line=key[1],
             column=key[2],
             occurrence=occurrence,
+            end_line=getattr(node, "end_lineno", key[1]),
+            end_column=getattr(node, "end_col_offset", key[2]),
             details=details,
         )
 
@@ -243,6 +249,15 @@ def _audit_task_call(self, call: ast.Call) -> bool:
                     mapped=mapped,
                 )
             )
+            if call.args or any(keyword.arg is None for keyword in call.keywords):
+                self.audit.unresolved.append(
+                    self._candidate(
+                        "unresolved",
+                        "dynamic_operator_arguments",
+                        call,
+                        expression=ast.unparse(call),
+                    )
+                )
             return True
         base = _base_call_name(call)
         if base in self.factories:
@@ -320,9 +335,7 @@ def _add_edges(self, node: ast.AST, upstreams: list[str], downstreams: list[str]
     def _audit_settings(self, call: ast.Call) -> None:
         for keyword in call.keywords:
             if keyword.arg:
-                self.audit.settings.append(
-                    self._candidate("setting", "dag_setting", keyword.value, name=keyword.arg)
-                )
+                self.audit.settings.append(self._candidate("setting", "dag_setting", keyword.value, name=keyword.arg))
                 if keyword.arg == "default_args" and isinstance(keyword.value, ast.Dict):
                     for key, value in zip(keyword.value.keys, keyword.value.values):
                         if isinstance(key, ast.Constant) and isinstance(key.value, str):
@@ -389,9 +402,7 @@ def _operator_call(call: ast.Call, aliases: dict[str, str]) -> tuple[str, dict[s
         operator = _leaf(inner.func.value, aliases)
     if not _is_operator(operator):
         return "", {}, False
-    keywords = {
-        keyword.arg: keyword.value for keyword in [*inner.keywords, *call.keywords] if keyword.arg
-    }
+    keywords = {keyword.arg: keyword.value for keyword in [*inner.keywords, *call.keywords] if keyword.arg}
     return operator, keywords, True
 
 
diff --git a/src/flowx/sources/airflow/discover.py b/src/flowx/sources/airflow/discover.py
index d0ddff6..e64f52f 100644
--- a/src/flowx/sources/airflow/discover.py
+++ b/src/flowx/sources/airflow/discover.py
@@ -1,9 +1,9 @@
 """Airflow discover phase: parse DAGs into a classified inventory.
 
 Mirrors the ADF discover contract: writes ``metadata/inventory.json`` and
-``metadata/profile_report.csv`` under the shared output dir, classifying each
-task as deterministic (a mapped operator -> NotebookActivity) or agentic (an
-unmapped operator -> PlaceholderActivity, needing LLM-assisted translation).
+``metadata/profile_report.csv`` under the shared output dir. Independently audited
+task candidates drive deterministic, agentic, failed, and excluded counts; emitted
+IR tasks remain available for the per-task inventory.
 Exposes ``main(argv)`` so the adapter runs it in-process, like the ADF loader.
 """
 
@@ -45,27 +45,75 @@ def _classify(pipeline: Pipeline) -> list[dict[str, str]]:
 def build_inventory_dict(pipelines: list[Pipeline], source_dir: str) -> dict[str, Any]:
     """Builds the inventory.json payload matching the ADF discover shape."""
     pipeline_entries: list[dict[str, Any]] = []
-    deterministic = agentic = 0
+    audited = deterministic = agentic = failed = excluded = 0
     for pipeline in pipelines:
         items = _classify(pipeline)
-        deterministic += sum(1 for i in items if i["strategy"] == "deterministic")
-        agentic += sum(1 for i in items if i["strategy"] == "agentic")
-        pipeline_entries.append({"name": pipeline.name, "activities": items})
-    activity_count = deterministic + agentic
-    # Coverage counts both deterministic and agentic as "has a translation path", matching the
-    # shared reporting.coverage formula (agentic gaps are translated in the convert phase).
-    coverage = round(100.0 * (deterministic + agentic) / activity_count, 1) if activity_count else 0.0
+        pipeline_audited = int(pipeline.audit.get("audited_activity_count", len(items)))
+        pipeline_deterministic = int(
+            pipeline.audit.get("deterministic_count", sum(1 for item in items if item["strategy"] == "deterministic"))
+        )
+        pipeline_agentic = int(
+            pipeline.audit.get("agentic_count", sum(1 for item in items if item["strategy"] == "agentic"))
+        )
+        pipeline_failed = int(pipeline.audit.get("failed_count", 0))
+        pipeline_excluded = int(pipeline.audit.get("excluded_count", 0))
+        coverage = (
+            round(100.0 * (pipeline_deterministic + pipeline_agentic) / pipeline_audited, 1)
+            if pipeline_audited
+            else 0.0
+        )
+        deterministic_coverage = (
+            round(100.0 * pipeline_deterministic / pipeline_audited, 1) if pipeline_audited else 0.0
+        )
+        audited += pipeline_audited
+        deterministic += pipeline_deterministic
+        agentic += pipeline_agentic
+        failed += pipeline_failed
+        excluded += pipeline_excluded
+        pipeline_entries.append(
+            {
+                "name": pipeline.name,
+                "activities": items,
+                "audited_activity_count": pipeline_audited,
+                "deterministic_count": pipeline_deterministic,
+                "agentic_count": pipeline_agentic,
+                "failed_count": pipeline_failed,
+                "excluded_count": pipeline_excluded,
+                "reconciliation_status": pipeline.reconciliation_status or "verified",
+                "migration_status": pipeline.migration_status,
+                "coverage_pct": coverage,
+                "deterministic_coverage_pct": deterministic_coverage,
+                "findings": pipeline.not_translatable,
+                "transformations": pipeline.audit.get("transformations", []),
+            }
+        )
+    coverage = round(100.0 * (deterministic + agentic) / audited, 1) if audited else 0.0
+    deterministic_coverage = round(100.0 * deterministic / audited, 1) if audited else 0.0
+    reconciliation_status = (
+        "failed"
+        if any(pipeline.reconciliation_status == "failed" for pipeline in pipelines)
+        else "verified_with_gaps"
+        if any(pipeline.reconciliation_status == "verified_with_gaps" for pipeline in pipelines)
+        else "excluded"
+        if pipelines and all(pipeline.migration_status == "excluded" for pipeline in pipelines)
+        else "verified"
+    )
     return {
         "source": "airflow",
         "source_dir": source_dir,
         "pipelines": pipeline_entries,
         "summary": {
             "pipeline_count": len(pipelines),
-            "activity_count": activity_count,
+            "activity_count": audited,
+            "audited_activity_count": audited,
             "deterministic_count": deterministic,
             "agentic_count": agentic,
             "unsupported_count": 0,
+            "failed_count": failed,
+            "excluded_count": excluded,
             "coverage_pct": coverage,
+            "deterministic_coverage_pct": deterministic_coverage,
+            "reconciliation_status": reconciliation_status,
         },
     }
 
@@ -93,7 +141,7 @@ def build_inventory_dict(pipelines: list[Pipeline], source_dir: str) -> dict[str
 
 def _profile_row(pipeline: Pipeline) -> dict[str, Any]:
     """Computes one profile row for *pipeline* over the full column set."""
-    type_names = [type(task).__name__ for task in pipeline.tasks]
+    type_names = [type(task).__name__ for task in pipeline.tasks if not task.task_key.startswith("__flowx_")]
     total = len(type_names)
     native = sum(1 for name in type_names if name in _NATIVE_TYPES)
     control = sum(1 for name in type_names if name in _CONTROL_FLOW_TYPES)
@@ -161,7 +209,11 @@ def main(argv: list[str] | None = None) -> int:
     print(f"Total tasks:        {summary['activity_count']}")
     print(f"  Deterministic:    {summary['deterministic_count']}")
     print(f"  Agentic:          {summary['agentic_count']}")
-    print(f"Coverage:           {summary['coverage_pct']}%")
+    print(f"  Failed:           {summary['failed_count']}")
+    print(f"  Excluded:         {summary['excluded_count']}")
+    print(f"Translation path:   {summary['coverage_pct']}%")
+    print(f"Deterministic:      {summary['deterministic_coverage_pct']}%")
+    print(f"Reconciliation:     {summary['reconciliation_status']}")
     return 1 if any(pipeline.reconciliation_status == "failed" for pipeline in pipelines) else 0
 
 
diff --git a/src/flowx/sources/airflow/operators.py b/src/flowx/sources/airflow/operators.py
index 033cc56..46bb86d 100644
--- a/src/flowx/sources/airflow/operators.py
+++ b/src/flowx/sources/airflow/operators.py
@@ -814,7 +814,6 @@ def build_placeholder_with_comment(ctx: OperatorContext, comment: str) -> Activi
     return _placeholder(ctx, comment)
 
 
-_LOADER_CONSUMED_KWARGS = frozenset({"task_id", "dag", "trigger_rule", "retries", "retry_delay", "execution_timeout"})
 _OPERATOR_CONSUMED_KWARGS: dict[str, frozenset[str]] = {
     "PythonOperator": frozenset({"python_callable", "op_args", "op_kwargs"}),
     "BranchPythonOperator": frozenset({"python_callable", "op_args", "op_kwargs"}),
@@ -850,20 +849,46 @@ def build_placeholder_with_comment(ctx: OperatorContext, comment: str) -> Activi
     {"bucket_key", "bucket_name", "object", "bucket", "filepath", "filepath_", "poke_interval", "timeout"}
 )
 _TABLE_SENSOR_KWARGS = frozenset({"sql", "table_name", "poke_interval", "timeout"})
+_LOADER_KWARG_RATIONALES: dict[str, str] = {
+    "task_id": "capture_identity",
+    "dag": "dag_membership",
+    "trigger_rule": "dependency_outcome",
+    "retries": "retry_policy",
+    "retry_delay": "retry_policy",
+    "execution_timeout": "timeout_policy",
+}
 
 
-def unconsumed_kwargs(operator: str, kwargs: dict[str, ast.expr]) -> set[str]:
-    """Returns supplied arguments with no declared loader or adapter semantics."""
-    consumed: frozenset[str] | None
+def argument_classification(operator: str, kwargs: dict[str, ast.expr]) -> list[dict[str, str]]:
+    """Classifies every supplied operator argument and records why it is represented."""
     if operator in FILE_SENSORS:
-        consumed = _FILE_SENSOR_KWARGS
+        adapter_consumed: frozenset[str] | None = _FILE_SENSOR_KWARGS
     elif operator in TABLE_SENSORS:
-        consumed = _TABLE_SENSOR_KWARGS
+        adapter_consumed = _TABLE_SENSOR_KWARGS
     else:
-        consumed = _OPERATOR_CONSUMED_KWARGS.get(operator)
-    if consumed is None:
-        return set()
-    return set(kwargs) - _LOADER_CONSUMED_KWARGS - consumed
+        adapter_consumed = _OPERATOR_CONSUMED_KWARGS.get(operator)
+
+    classified: list[dict[str, str]] = []
+    for name in sorted(kwargs):
+        if name in _LOADER_KWARG_RATIONALES:
+            status = "consumed"
+            rationale = _LOADER_KWARG_RATIONALES[name]
+        elif adapter_consumed is None:
+            status = "preserved"
+            rationale = "placeholder_raw_definition"
+        elif name in adapter_consumed:
+            status = "consumed"
+            rationale = "operator_adapter"
+        else:
+            status = "unconsumed"
+            rationale = "no_declared_semantics"
+        classified.append({"name": name, "status": status, "rationale": rationale})
+    return classified
+
+
+def unconsumed_kwargs(operator: str, kwargs: dict[str, ast.expr]) -> set[str]:
+    """Returns supplied arguments with no declared loader or adapter semantics."""
+    return {item["name"] for item in argument_classification(operator, kwargs) if item["status"] == "unconsumed"}
 
 
 # --------------------------------------------------------------------------------------
diff --git a/tests/unit/test_airflow_adapter_reporting.py b/tests/unit/test_airflow_adapter_reporting.py
index 44b5e27..7502673 100644
--- a/tests/unit/test_airflow_adapter_reporting.py
+++ b/tests/unit/test_airflow_adapter_reporting.py
@@ -8,7 +8,9 @@
 import pytest
 
 from flowx.adapter.session import MigrationInputSession
+from flowx.models.ir import NotebookActivity, Pipeline
 from flowx.reporting.coverage import COVERAGE_METRIC_COLUMNS, build_coverage_rows
+from flowx.sources.airflow.discover import _profile_row, build_inventory_dict
 from flowx.sources.airflow.discover import main as discover_main
 
 
@@ -65,3 +67,71 @@ def test_airflow_profile_csv_has_all_coverage_columns():
         assert row["databricks_native_activities"] == 1  # the PythonOperator
         assert row["other_activities"] == 1  # the placeholder
         assert row["complexity_score"] == 4  # 1*1 + 1*3
+
+
+def test_airflow_inventory_persists_audit_status_counts_and_findings() -> None:
+    finding = {
+        "code": "unsupported_operator",
+        "severity": "gap",
+        "fingerprint": "stable123",
+        "message": "manual translation required",
+    }
+    pipeline = Pipeline(
+        name="audited",
+        reconciliation_status="verified_with_gaps",
+        migration_status="included",
+        not_translatable=[finding],
+        audit={
+            "audited_activity_count": 8,
+            "deterministic_count": 7,
+            "agentic_count": 1,
+            "failed_count": 0,
+            "excluded_count": 0,
+            "transformations": [{"code": "task_key_collision_resolved"}],
+        },
+    )
+
+    inventory = build_inventory_dict([pipeline], "/src")
+    entry = inventory["pipelines"][0]
+
+    assert entry["audited_activity_count"] == 8
+    assert entry["deterministic_count"] == 7
+    assert entry["agentic_count"] == 1
+    assert entry["failed_count"] == 0
+    assert entry["excluded_count"] == 0
+    assert entry["coverage_pct"] == 100.0
+    assert entry["deterministic_coverage_pct"] == 87.5
+    assert entry["reconciliation_status"] == "verified_with_gaps"
+    assert entry["findings"] == [finding]
+    assert entry["transformations"] == [{"code": "task_key_collision_resolved"}]
+    assert inventory["summary"]["activity_count"] == 8
+    assert inventory["summary"]["deterministic_coverage_pct"] == 87.5
+
+
+def test_airflow_profile_categories_never_mix_audited_and_synthetic_tasks() -> None:
+    pipeline = Pipeline(
+        name="profile",
+        tasks=[],
+        audit={
+            "audited_activity_count": 1,
+            "deterministic_count": 0,
+            "agentic_count": 0,
+            "failed_count": 1,
+            "excluded_count": 0,
+        },
+    )
+    pipeline.tasks.extend(
+        [
+            NotebookActivity(name="first", task_key="first", notebook_path="/Shared/first"),
+            NotebookActivity(name="second", task_key="second", notebook_path="/Shared/second"),
+        ]
+    )
+
+    row = _profile_row(pipeline)
+
+    assert row["other_activities"] >= 0
+    assert row["complexity_score"] >= 0
+    assert (
+        row["databricks_native_activities"] + row["control_flow_activities"] + row["other_activities"]
+        == row["activities"]
+    )
diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py
index cd80984..401e007 100644
--- a/tests/unit/test_airflow_operators.py
+++ b/tests/unit/test_airflow_operators.py
@@ -775,7 +775,7 @@ def test_table_sensor_escapes_quotes_in_table_name():
         "def w():\n    pass\n"
         "with DAG(dag_id='d') as dag:\n"
         "    prep = PythonOperator(task_id='prep', python_callable=w)\n"
-        '    wait = DatabricksPartitionSensor(task_id=\'wait\', table_name=\'main.silver.we"ird\')\n'
+        "    wait = DatabricksPartitionSensor(task_id='wait', table_name='main.silver.we\"ird')\n"
         "    prep >> wait\n"
     )
     wait = _by_key(p)["wait"]
diff --git a/tests/unit/test_airflow_production_readiness.py b/tests/unit/test_airflow_production_readiness.py
index c6f0d1f..fa6be39 100644
--- a/tests/unit/test_airflow_production_readiness.py
+++ b/tests/unit/test_airflow_production_readiness.py
@@ -2,12 +2,52 @@
 
 from pathlib import Path
 
+import pytest
+
 from flowx.models.ir import ForEachActivity, NotebookActivity, PlaceholderActivity, SparkPythonActivity
 from flowx.preparer.workflow_preparer import prepare_workflow
 from flowx.sources.airflow.loader import load_airflow_dag, load_airflow_dags
 
 _REPROS = Path(__file__).parents[1] / "resources" / "airflow" / "review_repros"
 
+_REPRO_CORPUS = {
+    "a1_assigned_dag.py": [("legacy_etl", "verified", 2)],
+    "a2_task_key_collision.py": [("collide", "verified", 3)],
+    "a8_classic_mapping.py": [("fan", "verified_with_gaps", 1)],
+    "t1_loop.py": [("loop_dag", "verified", 3)],
+    "t2_sparksubmit.py": [("ss_dag", "verified", 3)],
+    "t3_collide.py": [("collide_dag", "verified", 3)],
+    "t4_bashjinja.py": [("jinja_dag", "verified_with_gaps", 1)],
+    "t5_alias.py": [("alias_dag", "verified", 2)],
+    "t6_chain.py": [("chain_dag", "verified", 4)],
+    "t7_subclass.py": [("sub_dag", "verified_with_gaps", 2)],
+    "t8_helperfn.py": [("helper_dag", "verified", 2)],
+    "t9_triggerrule.py": [("tr_dag", "verified", 4)],
+    "t10_loopliteral.py": [("loop2", "verified", 2)],
+    "t11_dagvar.py": [("assigned_dag", "verified", 2)],
+    "t12_globals.py": [("etl_alpha", "verified", 1), ("etl_beta", "verified", 1)],
+    "t13_sqlescape.py": [("sqlesc", "verified_with_gaps", 1)],
+    "t14_retries.py": [("ret", "verified", 2)],
+    "t15_magic.py": [("magic", "verified", 1)],
+    "t16_sensor.py": [("sensor_mid", "verified", 3)],
+    "t17_taskflow.py": [("tf", "verified", 3)],
+    "t18_xcompush.py": [("deps", "verified", 1)],
+    "t19_fncollide.py": [("fnc", "verified", 1)],
+    "t20_sqlesc.py": [("sqlq", "verified", 2)],
+    "t21_partialexpand.py": [("pe", "verified_with_gaps", 1)],
+    "t22_expandbash.py": [("eb", "verified_with_gaps", 2)],
+    "t23_tr2.py": [("tr2", "verified_with_gaps", 5)],
+    "t24_sensorscope.py": [("ss2", "verified", 3)],
+    "t25_tr3.py": [("tr3", "verified_with_gaps", 4)],
+    "t26_loopedge.py": [("le", "verified", 3)],
+    "t27_ss.py": [("ss3", "verified_with_gaps", 1)],
+    "t28_nodash.py": [("nd", "verified_with_gaps", 1)],
+    "t29_dagsem.py": [("dsem", "verified_with_gaps", 2)],
+    "t30_dagvar2.py": [("legacy_etl", "verified", 2)],
+    "t31_inject.py": [("inj", "verified", 1)],
+    "t32_multiassigned.py": [("team_a_etl", "verified", 2), ("team_b_etl", "verified", 2)],
+}
+
 
 def _dependencies(pipeline) -> dict[str, list[str]]:
     return {
@@ -15,6 +55,13 @@ def _dependencies(pipeline) -> dict[str, list[str]]:
     }
 
 
+@pytest.mark.parametrize(("fixture_name", "expected"), sorted(_REPRO_CORPUS.items()))
+def test_promoted_review_repro_corpus_is_exercised(fixture_name: str, expected: list[tuple[str, str, int]]) -> None:
+    pipelines = load_airflow_dags(_REPROS / fixture_name)
+
+    assert [(pipeline.name, pipeline.reconciliation_status, len(pipeline.tasks)) for pipeline in pipelines] == expected
+
+
 def test_assigned_dag_preserves_configuration_and_tasks() -> None:
     pipeline = load_airflow_dag(_REPROS / "a1_assigned_dag.py")
 
@@ -36,6 +83,11 @@ def test_task_key_collisions_allocate_distinct_keys_without_losing_edges() -> No
 
     assert [task.task_key for task in pipeline.tasks] == ["load_data", "load_data__2", "final"]
     assert _dependencies(pipeline)["final"] == ["load_data", "load_data__2"]
+    edge_proofs = [item for item in pipeline.audit["transformations"] if item["code"] == "edge_captured"]
+    assert {(item["upstream_capture_id"], item["downstream_capture_id"]) for item in edge_proofs} == {
+        ("x", "z"),
+        ("y", "z"),
+    }
 
 
 def test_bounded_loops_preserve_generated_tasks_and_edges() -> None:
@@ -63,6 +115,10 @@ def test_aliases_chain_cross_downstream_and_single_return_factories_are_captured
     }
     assert [task.task_key for task in helper_pipeline.tasks] == ["first", "second"]
     assert _dependencies(helper_pipeline)["second"] == ["first"]
+    helper_proofs = [
+        item for item in helper_pipeline.audit["transformations"] if item["code"] == "helper_factory_expanded"
+    ]
+    assert [item["helper"] for item in helper_proofs] == ["make", "make"]
 
 
 def test_module_callable_wins_over_unrelated_nested_definitions() -> None:
@@ -74,6 +130,79 @@ def test_module_callable_wins_over_unrelated_nested_definitions() -> None:
     assert "WRONG_BODY" not in (task.generated_source or "")
 
 
+def test_classic_callable_uses_nearest_lexical_definition(tmp_path: Path) -> None:
+    dag = tmp_path / "lexical.py"
+    dag.write_text(
+        "from airflow.decorators import dag\n"
+        "from airflow.operators.python import PythonOperator\n"
+        "def process():\n"
+        "    return 'MODULE_BODY'\n"
+        "@dag(dag_id='lexical')\n"
+        "def workflow():\n"
+        "    def process():\n"
+        "        return 'NESTED_BODY'\n"
+        "    run = PythonOperator(task_id='run', python_callable=process)\n"
+        "workflow()\n",
+        encoding="utf-8",
+    )
+
+    task = load_airflow_dag(dag).tasks[0]
+
+    assert isinstance(task, NotebookActivity)
+    assert "NESTED_BODY" in (task.generated_source or "")
+    assert "MODULE_BODY" not in (task.generated_source or "")
+
+
+def test_classic_callable_respects_same_scope_definition_order(tmp_path: Path) -> None:
+    dag = tmp_path / "definition_order.py"
+    dag.write_text(
+        "from airflow.decorators import dag\n"
+        "from airflow.operators.python import PythonOperator\n"
+        "@dag(dag_id='definition_order')\n"
+        "def workflow():\n"
+        "    def process():\n"
+        "        return 'FIRST_BODY'\n"
+        "    first = PythonOperator(task_id='first', python_callable=process)\n"
+        "    def process():\n"
+        "        return 'SECOND_BODY'\n"
+        "    second = PythonOperator(task_id='second', python_callable=process)\n"
+        "workflow()\n",
+        encoding="utf-8",
+    )
+
+    first, second = load_airflow_dag(dag).tasks
+
+    assert "FIRST_BODY" in (first.generated_source or "")
+    assert "SECOND_BODY" not in (first.generated_source or "")
+    assert "SECOND_BODY" in (second.generated_source or "")
+
+
+def test_conditionally_ambiguous_classic_callable_becomes_placeholder(tmp_path: Path) -> None:
+    dag = tmp_path / "ambiguous_callable.py"
+    dag.write_text(
+        "from airflow.decorators import dag\n"
+        "from airflow.operators.python import PythonOperator\n"
+        "FLAG = object()\n"
+        "@dag(dag_id='ambiguous_callable')\n"
+        "def workflow():\n"
+        "    if FLAG:\n"
+        "        def process():\n"
+        "            return 'LEFT'\n"
+        "    else:\n"
+        "        def process():\n"
+        "            return 'RIGHT'\n"
+        "    run = PythonOperator(task_id='run', python_callable=process)\n"
+        "workflow()\n",
+        encoding="utf-8",
+    )
+
+    pipeline = load_airflow_dag(dag)
+    task = next(task for task in pipeline.tasks if task.task_key == "run")
+
+    assert isinstance(task, PlaceholderActivity)
+    assert pipeline.reconciliation_status == "verified_with_gaps"
+
+
 def test_literal_dag_factory_loop_and_multiple_assigned_dags_remain_distinct() -> None:
     generated = load_airflow_dags(_REPROS / "t12_globals.py")
     assigned = load_airflow_dags(_REPROS / "t32_multiassigned.py")
@@ -222,3 +351,8 @@ def test_unconsumed_operator_arguments_become_placeholder() -> None:
     assert isinstance(task, PlaceholderActivity)
     finding = next(item for item in pipeline.not_translatable if item["code"] == "unconsumed_operator_arguments")
     assert finding["details"]["arguments"] == ["pool", "priority_weight", "queue"]
+    proof = next(item for item in pipeline.audit["transformations"] if item["code"] == "operator_arguments_classified")
+    classified = {item["name"]: item for item in proof["arguments"]}
+    assert classified["task_id"]["rationale"] == "capture_identity"
+    assert classified["bash_command"]["rationale"] == "operator_adapter"
+    assert classified["pool"]["status"] == "unconsumed"
diff --git a/tests/unit/test_reporting_coverage.py b/tests/unit/test_reporting_coverage.py
index 5c1a8f9..a9b6237 100644
--- a/tests/unit/test_reporting_coverage.py
+++ b/tests/unit/test_reporting_coverage.py
@@ -88,3 +88,84 @@ def test_build_coverage_rows_full_coverage_and_missing_csv(tmp_path: Path):
     beta = rows["p_beta"]
     assert beta["coverage_pct"] == 100.0  # 1/1 deterministic
     assert beta["datasets"] == 0 and beta["complexity_size"] == ""  # defaulted, no CSV
+
+
+def test_audited_counts_drive_translation_and_deterministic_coverage(tmp_path: Path) -> None:
+    metadata = tmp_path / "metadata"
+    metadata.mkdir()
+    inventory = {
+        "pipelines": [
+            {
+                "name": "verified_with_gap",
+                "activities": [],
+                "audited_activity_count": 8,
+                "deterministic_count": 7,
+                "agentic_count": 1,
+                "failed_count": 0,
+                "excluded_count": 0,
+                "reconciliation_status": "verified_with_gaps",
+                "migration_status": "included",
+                "findings": [{"fingerprint": "abc123", "severity": "gap"}],
+            },
+            {
+                "name": "failed",
+                "activities": [],
+                "audited_activity_count": 9,
+                "deterministic_count": 7,
+                "agentic_count": 1,
+                "failed_count": 1,
+                "excluded_count": 0,
+                "reconciliation_status": "failed",
+                "migration_status": "included",
+                "findings": [{"fingerprint": "def456", "severity": "failed"}],
+            },
+        ]
+    }
+    (metadata / "inventory.json").write_text(json.dumps(inventory), encoding="utf-8")
+
+    rows = {row["pipeline"]: row for row in build_coverage_rows(metadata)}
+
+    verified = rows["verified_with_gap"]
+    assert verified["activities"] == 8
+    assert verified["audited_activities"] == 8
+    assert verified["coverage_pct"] == 100.0
+    assert verified["deterministic_coverage_pct"] == 87.5
+    assert verified["finding_count"] == 1
+    assert json.loads(verified["finding_fingerprints"]) == ["abc123"]
+
+    failed = rows["failed"]
+    assert failed["activities"] == 9
+    assert failed["failed_activities"] == 1
+    assert failed["coverage_pct"] == 88.9
+    assert failed["deterministic_coverage_pct"] == 77.8
+    assert failed["reconciliation_status"] == "failed"
+
+
+def test_excluded_activities_remain_in_coverage_denominator(tmp_path: Path) -> None:
+    metadata = tmp_path / "metadata"
+    metadata.mkdir()
+    inventory = {
+        "pipelines": [
+            {
+                "name": "excluded",
+                "activities": [],
+                "audited_activity_count": 3,
+                "deterministic_count": 0,
+                "agentic_count": 0,
+                "failed_count": 0,
+                "excluded_count": 3,
+                "reconciliation_status": "verified",
+                "migration_status": "excluded",
+                "findings": [],
+            }
+        ]
+    }
+    (metadata / "inventory.json").write_text(json.dumps(inventory), encoding="utf-8")
+
+    row = build_coverage_rows(metadata)[0]
+
+    assert row["activities"] == 3
+    assert row["excluded_activities"] == 3
+    assert row["coverage_pct"] == 0.0
+    assert row["deterministic_coverage_pct"] == 0.0
+    assert row["migration_status"] == "excluded"
diff --git a/tests/unit/test_reporting_dashboard.py b/tests/unit/test_reporting_dashboard.py
index 319e75d..4b7ce17 100644
--- a/tests/unit/test_reporting_dashboard.py
+++ b/tests/unit/test_reporting_dashboard.py
@@ -16,6 +16,11 @@ def test_build_serialized_dashboard_injects_table_and_is_valid_json():
     # every dataset query references the fully-qualified table
     joined = " ".join(line for ds in spec["datasets"] for line in ds["queryLines"])
     assert "cat.sch.results" in joined
+    assert "audited_activities" in joined
+    assert "failed_activities" in joined
+    assert "excluded_activities" in joined
+    assert "reconciliation_status" in joined
+    assert "deterministic_coverage_pct" in joined
     assert spec["pages"][0]["pageType"] == "PAGE_TYPE_CANVAS"
     # widget field names match their dataset fields (counter references a real column)
     widget_names = {w["widget"]["name"] for w in spec["pages"][0]["layout"]}
diff --git a/tests/unit/test_reporting_results.py b/tests/unit/test_reporting_results.py
index 6351ecf..7e39655 100644
--- a/tests/unit/test_reporting_results.py
+++ b/tests/unit/test_reporting_results.py
@@ -19,14 +19,34 @@ def test_create_table_sql_has_run_metadata_and_all_columns():
     assert "run_date TIMESTAMP" in sql
     assert "run_by STRING" in sql
     assert "coverage_pct DOUBLE" in sql
+    assert "audited_activities INT" in sql
+    assert "failed_activities INT" in sql
+    assert "excluded_activities INT" in sql
+    assert "reconciliation_status STRING" in sql
+    assert "deterministic_coverage_pct DOUBLE" in sql
+    assert "finding_fingerprints STRING" in sql
     assert "complexity_size STRING" in sql
 
 
+def test_schema_evolution_sql_adds_only_missing_metric_columns() -> None:
+    sql = R.build_add_columns_sql(
+        "cat.sch.tbl",
+        existing_columns={"RUN_ID", "PIPELINE", "activities", "coverage_pct"},
+    )
+
+    assert sql.startswith("ALTER TABLE cat.sch.tbl ADD COLUMNS")
+    assert "audited_activities INT" in sql
+    assert "deterministic_coverage_pct DOUBLE" in sql
+    assert "pipeline STRING" not in sql
+    assert "\n  coverage_pct DOUBLE" not in sql
+
+
 def test_insert_sql_stamps_run_metadata_and_escapes():
     rows = [
         {
             "pipeline": "p1",
             "activities": 3,
+            "audited_activities": 3,
             "datasets": 1,
             "linked_services": 0,
             "collapsible_patterns": 0,
@@ -36,13 +56,21 @@ def test_insert_sql_stamps_run_metadata_and_escapes():
             "deterministic_activities": 2,
             "agentic_activities": 1,
             "unsupported_activities": 0,
+            "failed_activities": 0,
+            "excluded_activities": 0,
+            "reconciliation_status": "verified_with_gaps",
+            "migration_status": "included",
             "coverage_pct": 100.0,
+            "deterministic_coverage_pct": 66.7,
+            "finding_count": 1,
+            "finding_fingerprints": '["abc"]',
             "complexity_score": 7,
             "complexity_size": "M",
         },
         {
             "pipeline": "O'Brien's pipe",
             "activities": 1,
+            "audited_activities": 1,
             "datasets": 0,
             "linked_services": 0,
             "collapsible_patterns": 0,
@@ -52,7 +80,14 @@ def test_insert_sql_stamps_run_metadata_and_escapes():
             "deterministic_activities": 0,
             "agentic_activities": 0,
             "unsupported_activities": 1,
+            "failed_activities": 0,
+            "excluded_activities": 0,
+            "reconciliation_status": "not_applicable",
+            "migration_status": "included",
             "coverage_pct": 0.0,
+            "deterministic_coverage_pct": 0.0,
+            "finding_count": 0,
+            "finding_fingerprints": "[]",
             "complexity_score": 3,
             "complexity_size": "S",
         },
@@ -68,6 +103,8 @@ def test_insert_sql_stamps_run_metadata_and_escapes():
     assert "'O''Brien''s pipe'" in sql
     # numeric + float rendered unquoted
     assert "100.0" in sql
+    assert "'verified_with_gaps'" in sql
+    assert "'[\"abc\"]'" in sql
 
 
 class _FakeWarehouse:
@@ -88,8 +125,9 @@ def list(self):
 
 
 class _FakeStmtAPI:
-    def __init__(self):
+    def __init__(self, columns=None):
         self.statements = []
+        self.columns = columns or [name for name, _sql_type in R.RESULTS_COLUMNS]
 
     def execute_statement(self, statement, warehouse_id, wait_timeout=None):
         self.statements.append((warehouse_id, statement))
@@ -98,13 +136,20 @@ class _Resp:
             class status:
                 state = "SUCCEEDED"
 
+        if statement.startswith("SHOW COLUMNS"):
+
+            class _Result:
+                data_array = [[column] for column in self.columns]
+
+            _Resp.result = _Result()
+
         return _Resp()
 
 
 class _FakeClient:
-    def __init__(self, warehouses):
+    def __init__(self, warehouses, columns=None):
         self.warehouses = _FakeWarehousesAPI(warehouses)
-        self.statement_execution = _FakeStmtAPI()
+        self.statement_execution = _FakeStmtAPI(columns)
 
 
 def test_resolve_warehouse_prefers_running_serverless():
@@ -168,13 +213,46 @@ def _metadata(tmp_path: Path) -> Path:
     return md
 
 
-def test_write_results_executes_create_then_insert(tmp_path: Path):
+def test_write_results_executes_create_schema_check_then_insert(tmp_path: Path):
     client = _FakeClient([_FakeWarehouse("wh1", "RUNNING", serverless=True)])
     run_id, rows = R.write_results(_metadata(tmp_path), "cat.sch.tbl", client=client)
     assert rows == 1
     uuid.UUID(run_id)  # valid uuid
     stmts = client.statement_execution.statements
-    assert len(stmts) == 2
+    assert len(stmts) == 3
     assert stmts[0][0] == "wh1" and stmts[0][1].startswith("CREATE TABLE IF NOT EXISTS")
-    assert stmts[1][1].startswith("INSERT INTO cat.sch.tbl")
-    assert run_id in stmts[1][1]
+    assert stmts[1][1] == "SHOW COLUMNS IN cat.sch.tbl"
+    assert stmts[2][1].startswith("INSERT INTO cat.sch.tbl")
+    assert run_id in stmts[2][1]
+
+
+def test_write_results_evolves_an_existing_legacy_schema_before_insert(tmp_path: Path) -> None:
+    legacy_columns = {
+        "run_id",
+        "run_date",
+        "run_by",
+        "pipeline",
+        "activities",
+        "datasets",
+        "linked_services",
+        "collapsible_patterns",
+        "databricks_native_activities",
+        "control_flow_activities",
+        "other_activities",
+        "deterministic_activities",
+        "agentic_activities",
+        "unsupported_activities",
+        "coverage_pct",
+        "complexity_score",
+        "complexity_size",
+    }
+    client = _FakeClient([_FakeWarehouse("wh1", "RUNNING", serverless=True)], columns=legacy_columns)
+
+    R.write_results(_metadata(tmp_path), "cat.sch.tbl", client=client)
+
+    statements = [statement for _warehouse, statement in client.statement_execution.statements]
+    assert len(statements) == 4
+    assert statements[2].startswith("ALTER TABLE cat.sch.tbl ADD COLUMNS")
+    assert "audited_activities INT" in statements[2]
+    assert "deterministic_coverage_pct DOUBLE" in statements[2]
+    assert statements[3].startswith("INSERT INTO cat.sch.tbl")
diff --git a/tests/unit/test_source_router.py b/tests/unit/test_source_router.py
index 22f8b75..3a50832 100644
--- a/tests/unit/test_source_router.py
+++ b/tests/unit/test_source_router.py
@@ -92,9 +92,7 @@ def test_source_path_alias_equals_form_routes():
     # or the phase module rejects it with a usage error.
     with tempfile.TemporaryDirectory() as tmp:
         out = Path(tmp)
-        rc = _run_phase(
-            "discover", ["--source", "airflow", f"--source-path={_DAG_FIXTURE}", f"--output-dir={out}"]
-        )
+        rc = _run_phase("discover", ["--source", "airflow", f"--source-path={_DAG_FIXTURE}", f"--output-dir={out}"])
         assert rc == 0
         assert (out / "metadata" / "inventory.json").exists()
 

From 5a9efccc0e10be01001172f3c5eac9833deb1fef Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Sat, 8 Aug 2026 00:51:28 -0700
Subject: [PATCH 53/77] Fix Airflow DAG discovery and factory capture

---
 README.md                                     |   7 +-
 .../flowx-convert/sources/airflow-coverage.md |  17 +-
 skills/flowx-convert/sources/airflow.md       |  42 +-
 src/flowx/sources/airflow/audit.py            |  32 +-
 src/flowx/sources/airflow/loader.py           | 478 +++++++++++++++---
 tests/unit/test_airflow_reconciliation.py     | 139 +++++
 6 files changed, 611 insertions(+), 104 deletions(-)

diff --git a/README.md b/README.md
index f4f22ab..4ee120a 100644
--- a/README.md
+++ b/README.md
@@ -165,8 +165,9 @@ execution) and maps ~35 operator/sensor families to the shared IR. Highlights:
 - **Scheduling & semantics** — cron → Quartz, `timedelta` → periodic, `trigger_rule` → `run_if`,
   `params={...}` → job parameters, `>>` / `<<` / `set_upstream` / TaskGroup edges.
 
-Operators without a deterministic mapping become a placeholder recorded in `gaps.json` for the
-agentic round. Full matrix: [`skills/flowx-convert/sources/airflow-coverage.md`](skills/flowx-convert/sources/airflow-coverage.md).
+Operators without a deterministic mapping become a failing placeholder and are recorded in
+`gaps.json` for review. Airflow agentic resolution is not a supported workflow yet. Full matrix:
+[`skills/flowx-convert/sources/airflow-coverage.md`](skills/flowx-convert/sources/airflow-coverage.md).
 
 Airflow discovery independently audits DAG declarations, task candidates, dependency declarations,
 DAG settings, mapped calls, and operator arguments before comparing them with captured IR. An
@@ -183,7 +184,7 @@ to the supported static subset; flowx never imports or executes DAG modules.
 Parses the source into typed nodes and classifies each activity/operator as deterministic, agentic, or unsupported — ADF JSON from Unity Catalog volumes (or a `/Workspace` Git folder, normalizing ARM template format), or Airflow DAG `.py` modules read statically with `ast`. Airflow inventory includes audited/deterministic/agentic/failed/excluded counts, reconciliation status, stable finding fingerprints, translation-path coverage, and deterministic coverage. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`.
 
 ### Phase 2: Convert
-Applies deterministic translators (ADF activity registry / Airflow operator mapping), resolves dependencies, and flags agentic gaps for LLM-assisted translation. Produces the shared Pipeline IR consumed unchanged by the package phase.
+Applies deterministic translators (ADF activity registry / Airflow operator mapping), resolves dependencies, and records unresolved gaps. ADF supports the guided agentic translation workflow; Airflow currently retains failing placeholders for explicit review. Produces the shared Pipeline IR consumed unchanged by the package phase.
 
 ### Phase 3: Package
 Converts Pipeline IR into a deployable DABs project: `databricks.yml`, per-job YAML resource files, generated Python notebooks, and setup scripts for UC volumes, secrets, and connections.
diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md
index 876218a..32acdcc 100644
--- a/skills/flowx-convert/sources/airflow-coverage.md
+++ b/skills/flowx-convert/sources/airflow-coverage.md
@@ -13,7 +13,7 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't
 | --- | --- |
 | `PythonOperator` (classic) | Notebook task; callable `def` preserved, transitive helpers/constants/non-Airflow imports carried, `op_args`/`op_kwargs` passed as JSON widgets, return value via `dbutils.jobs.taskValues.set`. |
 | `PythonVirtualenvOperator` / `ExternalPythonOperator` | Notebook task with a `%pip install` cell for `requirements`. |
-| `BranchPythonOperator` / `ShortCircuitOperator` | Placeholder routed to the agentic-gap round (runtime branch selection can't be lowered statically). |
+| `BranchPythonOperator` / `ShortCircuitOperator` | Failing placeholder + review gap (runtime branch selection can't be lowered statically). |
 | `BashOperator` / `SSHOperator` | `%sh` notebook; a single unchained `spark-submit` invocation is lifted only when every option arity is known. |
 | `SparkSubmitOperator` | Spark JAR or Python task. |
 | Databricks provider operators (`DatabricksSubmitRun*`, `DatabricksRunNow*`, `DatabricksNotebookOperator`) | Notebook / run-job tasks. |
@@ -21,7 +21,7 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't
 | `TriggerDagRunOperator` | `run_job_task` referencing the target DAG by sanitized job name. |
 | `EmailOperator` | Placeholder recommending job-level email notifications. |
 | dbt CLI operators (`DbtRun/Test/Seed/Snapshot/Build/Deps`) and Cosmos `DbtDag` / `DbtTaskGroup` | Single `DbtFactoryActivity`, **static explosion** (default) or **PyDABs** (`--dbt-mode pydabs`); see [dbt factory](#dbt-factory-mode). |
-| **TaskFlow API** (`@dag`, `@task`, `@task.virtualenv`) | Each `@task` invocation → a task; implicit XCom data flow (`transform(extract())`) → a notebook that reads upstream return values via `dbutils.jobs.taskValues.get`, calls the function, and publishes its own. `@task.branch` / `@task.short_circuit`, or a callable reading task context/XCom, route to a placeholder + gap. |
+| **TaskFlow API** (`@dag`, `@task`, `@task.virtualenv`) | Canonical, aliased, and qualified Airflow decorators are resolved statically. Each `@task` invocation → a task; implicit XCom data flow (`transform(extract())`) → a notebook that reads upstream return values via `dbutils.jobs.taskValues.get`, calls the function, and publishes its own. `@task.branch` / `@task.short_circuit`, or a callable reading task context/XCom, route to a placeholder + gap. |
 | File sensors (`S3KeySensor`, `GCSObjectExistenceSensor`, `FileSensor`, `HdfsSensor`, `WebHdfsSensor`) | With no schedule, a root sensor whose descendants cover every non-sensor task → `file_arrival` trigger; otherwise a `dbutils.fs` polling notebook task. |
 | Table/SQL sensors (`DatabricksPartitionSensor`, `DatabricksSqlSensor`, `DatabricksSQLStatementsSensor`, `SqlSensor`) | With no schedule, a root literal-table sensor whose descendants cover every non-sensor task → `table_update` trigger; otherwise a `spark.sql` polling notebook task. |
 | `ExternalTaskSensor` | Placeholder explaining logical-run-aware migration options; polling the latest Databricks job run is not equivalent to Airflow's matching logical run. |
@@ -32,16 +32,16 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't
 | Classic operator `.partial().expand()` / `.expand()` | `for_each_task` containing a linked failing placeholder until every mapped and fixed argument can be proven bound into the inner Databricks task. |
 | Dependencies | `>>` / `<<` chains (incl. list/tuple fan-out and inline TaskFlow calls) and `set_upstream` / `set_downstream`. |
 | **TaskGroups** (context-manager `with TaskGroup(...)`) | Static nesting → task-key namespacing (`group__subgroup__task`); group-level edges (`group_a >> group_b`, `task >> group`) expand to leaf→root edges between member tasks. |
-| **`@task_group`** (decorator form) | Placeholder + gap (with dependency edges preserved); a decorator group is a sub-pipeline flowx doesn't lower — the agentic round expands it into its member tasks / a for_each when mapped. |
+| **`@task_group`** (decorator form) | Placeholder + gap with dependency edges preserved; a decorator group is a sub-pipeline flowx doesn't lower deterministically. |
 | Schedule | Cron `schedule_interval` → Quartz (Unix DOW 0–6 → Quartz 1–7); exact sub-hour `timedelta` → Quartz, longer intervals → periodic, `@continuous` → continuous mode. |
 | `trigger_rule` | Exact supported rules map to `run_if`; `none_failed_min_one_success` maps to `NONE_FAILED` with the all-skipped delta recorded. Rules without an equivalent become linked placeholders. |
 | Job parameters | `params={...}` / `Param(default=...)` → job parameters with defaults; `{{ params.x }}` / `{{ var.value.x }}` / `{{ dag_run.conf['x'] }}` → `{{job.parameters.x}}`. |
 | `Variable.get` in a callable | Rewritten to `dbutils.widgets.get`; a callable using an Airflow `Connection` object routes to a placeholder because one secret string cannot preserve the object API. |
-| Multiple DAGs | Every DAG, including multiple declarations in one Python file, becomes a sibling job in one shared Airflow bundle so `TriggerDagRunOperator` resource references resolve. |
+| Multiple DAGs | Every DAG, including multiple declarations and repeated static `@dag` factory invocations in one Python file, becomes a sibling job in one shared Airflow bundle so `TriggerDagRunOperator` resource references resolve. Narrow classic factories shaped as one DAG declaration followed by `return dag` are expanded with statically bindable arguments. |
 
 Any operator not listed becomes a `PlaceholderActivity` **and** a `gaps.json` entry carrying the
-operator's raw source, for LLM-assisted translation in the agentic-gap round. That is the safe
-fallback: a flagged manual task, not a silent omission. Callables that read Airflow task context
+operator's raw source for review. Airflow agentic replacement is not a supported workflow yet. The
+safe fallback is a flagged, failing task rather than a silent omission. Callables that read Airflow task context
 (`**context` / `ti`) or XCom, and runtime-branching decorators, take the same route rather than
 emitting code that fails at runtime.
 
@@ -53,11 +53,14 @@ decisions.
 
 - **Full TaskGroup expansion** — a `@task_group` invocation (mapped `pair.expand(...)` or plain
   `pair(...)`) and `TaskGroup.partial().expand()` aren't lowered into their member tasks. They route
-  to a placeholder + gap with dependency edges preserved; the agentic round expands the group.
+  to a placeholder + gap with dependency edges preserved.
 - **Dynamic operator construction** — operators created inside comprehensions are not statically
   expanded. Helper factories are supported only when their body is an optional docstring followed
   by one statically bindable `return RecognizedOperator(...)`; other forms fail reconciliation and
   block package output.
+- **Dynamic DAG factories** — classic DAG factories outside the documented single-declaration shape,
+  non-literal factory arguments, and non-literal `dag_id` overrides fail reconciliation and block
+  package output rather than emitting a filename-derived empty Job.
 - **Sensors beyond the mapped families** (`S3PrefixSensor`, custom sensors, etc.) → placeholder + gap.
   A file sensor with a non-literal path, or a table/SQL sensor with no literal `sql` / `table_name`,
   also falls back to a placeholder.
diff --git a/skills/flowx-convert/sources/airflow.md b/skills/flowx-convert/sources/airflow.md
index 324f60c..5be40d8 100644
--- a/skills/flowx-convert/sources/airflow.md
+++ b/skills/flowx-convert/sources/airflow.md
@@ -3,18 +3,17 @@
 Source guide for `--source airflow`. Translate parsed Airflow DAGs into Databricks IR. See the
 parent `SKILL.md` for how to run the phase and the report contract.
 
-Airflow translation is **deterministic-first with an agentic-gap round**, like the ADF source. The
-static parse maps ~35 operator/sensor families directly to IR (Tier 1-3). Operators with no
-deterministic mapping become `PlaceholderActivity` tasks *and* are recorded in `gaps.json`, each
-carrying the operator's raw source so an agent can reason out the translation and replace the
-placeholder — the same `gaps.json` + `merge_agentic` flow the ADF source uses.
+Airflow translation is currently **deterministic-only**. The static parse maps ~35 operator/sensor
+families directly to IR (Tier 1-3). Operators with no deterministic mapping become
+`PlaceholderActivity` tasks and are recorded in `gaps.json` with their raw source for review. The
+placeholder remains a deliberate runtime failure until it is resolved manually; a supported Airflow
+agentic replacement workflow is not available yet.
 
 **Before converting, check [`sources/airflow-coverage.md`](airflow-coverage.md)** — the verified
 support matrix (classic operators, TaskFlow, sensors, TaskGroups, dbt factory) and the constructs
-still **not** handled (including dynamic TaskGroup mapping). Constructs flowx can't
-lower deterministically — callables reading task context (`**context` / `ti`) or XCom, and
-runtime-branching decorators — are routed to a placeholder + `gaps.json` for the agentic round rather
-than emitted as broken code.
+still **not** handled (including dynamic TaskGroup mapping). Constructs flowx can't lower
+deterministically — callables reading task context (`**context` / `ti`) or XCom, and runtime-branching
+decorators — are routed to a placeholder + `gaps.json` rather than emitted as broken code.
 
 dbt workloads default to static explosion; pass `--dbt-mode pydabs` to emit a deploy-time PyDABs hook
 instead (see the dbt factory section of the coverage doc).
@@ -39,27 +38,12 @@ PythonOperator callable or BashOperator command, carrying `generated_source`) or
 `PlaceholderActivity` (an unmapped operator). Dependencies come from `>>` / `<<`; the DAG's cron
 `schedule_interval` is carried as the pipeline `schedule`.
 
-## Step 3 — Handle agentic gaps
+## Step 3 — Review deterministic gaps
 
-If convert wrote `/.work/gaps.json`, each entry is an unmapped operator awaiting
-LLM-assisted translation. For each gap, read its `raw_definition` (the operator's source, embedded
-in the placeholder notebook too) and translate it into a real Databricks task — most portably a
-notebook you write to the workspace. Reason from the operator's arguments: e.g. a
-`KubernetesPodOperator` running a Python image becomes a notebook (or `%pip install` + the image's
-entrypoint logic); an `HttpSensor` becomes a polling notebook using `requests`; a `LivyOperator`
-submits Spark directly.
-
-Write one result JSON per gap into `/agentic_results/` and merge them with the shared
-`merge_agentic` command (see the parent `SKILL.md`):
-
-```bash
-"$PY" -m flowx.adapter convert --source airflow --merge-agentic \
-  --report /.work/translation_report.json \
-  --agentic-results /agentic_results
-```
-
-The ADF just-in-time option chain (notify motifs, metadata-driven consolidation) does not apply to
-Airflow; only the agentic-gap round does.
+If convert wrote `/.work/gaps.json`, each entry describes an unmapped construct whose
+generated Job task points to a notebook that raises `NotImplementedError`. Review every gap before
+deployment. The shared `merge_agentic` command is not a supported Airflow workflow yet; keep the
+placeholder, exclude the DAG, or implement and validate the replacement explicitly.
 
 ## Step 4 — Proceed to package
 
diff --git a/src/flowx/sources/airflow/audit.py b/src/flowx/sources/airflow/audit.py
index 45d3c39..f7f5abd 100644
--- a/src/flowx/sources/airflow/audit.py
+++ b/src/flowx/sources/airflow/audit.py
@@ -92,10 +92,12 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None) -> No
         self.taskflow_defs = {
             node.name
             for node in ast.walk(module)
-            if isinstance(node, ast.FunctionDef) and _decorator_leaf(node) in _TASK_DECORATORS
+            if isinstance(node, ast.FunctionDef) and _has_decorator(node, _TASK_DECORATORS, self.aliases)
         }
         self.dag_defs = {
-            node.name for node in module.body if isinstance(node, ast.FunctionDef) and _decorator_leaf(node) == "dag"
+            node.name
+            for node in module.body
+            if isinstance(node, ast.FunctionDef) and _has_decorator(node, {"dag"}, self.aliases)
         }
         self.factories = {
             node.name
@@ -120,7 +122,9 @@ def _candidate(self, kind: str, code: str, node: ast.AST, **details: Any) -> Aud
 
     def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
         if node.name in self.dag_defs:
-            decorator = next((item for item in node.decorator_list if _leaf(item, self.aliases) == "dag"), None)
+            decorator = next(
+                (item for item in node.decorator_list if _decorator_name(item, self.aliases) == "dag"), None
+            )
             if isinstance(decorator, ast.Call):
                 self._audit_settings(decorator)
             for statement in node.body:
@@ -349,7 +353,15 @@ def _audit_settings(self, call: ast.Call) -> None:
                             )
 
 
-_TASK_DECORATORS = {"task", "branch", "virtualenv", "short_circuit", "sensor", "external_python"}
+_TASK_DECORATORS = {
+    "task",
+    "task.branch",
+    "task.virtualenv",
+    "task.short_circuit",
+    "task.sensor",
+    "task.external_python",
+}
+_ALL_AIRFLOW_DECORATORS = {"dag", "task_group", *_TASK_DECORATORS}
 
 
 def _aliases(module: ast.Module) -> dict[str, str]:
@@ -380,8 +392,16 @@ def _leaf(node: ast.expr, aliases: dict[str, str]) -> str:
     return _dotted(node, aliases).rsplit(".", 1)[-1]
 
 
-def _decorator_leaf(node: ast.FunctionDef) -> str:
-    return _leaf(node.decorator_list[0], {}) if node.decorator_list else ""
+def _decorator_name(node: ast.expr, aliases: dict[str, str]) -> str:
+    canonical = _dotted(node, aliases)
+    for name in sorted(_ALL_AIRFLOW_DECORATORS, key=len, reverse=True):
+        if canonical == name or (canonical.startswith("airflow.") and canonical.endswith(f".{name}")):
+            return name
+    return canonical
+
+
+def _has_decorator(function: ast.FunctionDef, names: set[str], aliases: dict[str, str]) -> bool:
+    return any(_decorator_name(decorator, aliases) in names for decorator in function.decorator_list)
 
 
 def _is_operator(name: str) -> bool:
diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py
index 29cc79e..b9da56c 100644
--- a/src/flowx/sources/airflow/loader.py
+++ b/src/flowx/sources/airflow/loader.py
@@ -91,6 +91,12 @@ class DagDeclaration:
     variable: str | None
     node: ast.stmt
     span: SourceSpan
+    kind: str = "direct"
+    factory: ast.FunctionDef | None = None
+    bindings: dict[str, Any] = field(default_factory=dict)
+    target_dag_variable: str | None = None
+    decorator_overrides: dict[str, ast.expr] = field(default_factory=dict)
+    unsupported_reason: str | None = None
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)
@@ -431,6 +437,8 @@ def __init__(self, constants: dict[str, Any]) -> None:
         self.constants = constants
 
     def visit_Name(self, node: ast.Name) -> ast.expr:
+        if not isinstance(node.ctx, ast.Load):
+            return node
         value = self.constants.get(node.id, _UNRESOLVED)
         return ast.copy_location(_value_node(value), node) if value is not _UNRESOLVED else node
 
@@ -617,11 +625,16 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None = None
         self.taskgroup_defs: set[str] = set()
         for fn in _iter_functions(module):
             decorator = next(
-                (_decorator_name(d) for d in fn.decorator_list if _decorator_name(d) in _TASK_DECORATORS), None
+                (
+                    _decorator_name(d, self._aliases)
+                    for d in fn.decorator_list
+                    if _decorator_name(d, self._aliases) in _TASK_DECORATORS
+                ),
+                None,
             )
             if decorator is not None:
                 self.taskflow_defs[fn.name] = (fn, decorator)
-            elif _has_decorator(fn, _TASK_GROUP_DECORATORS):
+            elif _has_decorator(fn, _TASK_GROUP_DECORATORS, self._aliases):
                 self.taskgroup_defs.add(fn.name)
         # TaskFlow task instances: var name -> _TaskFlowTask (id, def-name, decorator, arg bindings).
         self.taskflow_tasks: dict[str, _TaskFlowTask] = {}
@@ -653,18 +666,23 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
         # which is internal logic rather than DAG structure, so don't descend. @dag marks the
         # DAG-defining function: read its config off the decorator, then descend so the body's task
         # instances / edges are collected.
-        if _has_decorator(node, _TASK_DECORATORS) or _has_decorator(node, _TASK_GROUP_DECORATORS):
+        if _has_decorator(node, _TASK_DECORATORS, self._aliases) or _has_decorator(
+            node, _TASK_GROUP_DECORATORS, self._aliases
+        ):
             if self._dag_scope_depth:
                 self._claimed_statement_ids.add(id(node))
             return
-        is_dag_definition = _has_decorator(node, _DAG_DECORATORS)
+        is_dag_definition = _has_decorator(node, _DAG_DECORATORS, self._aliases)
         if not is_dag_definition:
             if self._dag_scope_depth:
                 self._claimed_statement_ids.add(id(node))
             return
         if is_dag_definition:
             self.is_taskflow_dag = True
-            dag_kwargs = _decorator_kwargs(node.decorator_list, _DAG_DECORATORS)
+            dag_kwargs = {
+                name: _bind_constants(value, self._constants)
+                for name, value in _decorator_kwargs(node.decorator_list, _DAG_DECORATORS, self._aliases).items()
+            }
             self._apply_dag_kwargs(dag_kwargs)
             if self.dag_id is None:
                 self.dag_id = ops.literal_str(dag_kwargs.get("dag_id")) or node.name
@@ -1571,23 +1589,25 @@ def _is_task_construct(name: str) -> bool:
     return name.endswith("Operator") or name.endswith("Sensor") or name in ops.COSMOS_CONSTRUCTS
 
 
-def _decorator_name(node: ast.expr) -> str:
-    """Dotted name of a decorator, ignoring call args: ``@task`` / ``@task.branch()`` -> 'task.branch'."""
+def _decorator_name(node: ast.expr, aliases: dict[str, str] | None = None) -> str:
+    """Returns the normalized Airflow decorator name without importing its module."""
     if isinstance(node, ast.Call):
         node = node.func
-    parts: list[str] = []
-    while isinstance(node, ast.Attribute):
-        parts.append(node.attr)
-        node = node.value
-    if isinstance(node, ast.Name):
-        parts.append(node.id)
-    return ".".join(reversed(parts))
-
-
-def _decorator_kwargs(decorators: list[ast.expr], names: frozenset[str]) -> dict[str, ast.expr]:
+    canonical = _canonical_name(node, aliases or {})
+    for name in sorted(_ALL_AIRFLOW_DECORATORS, key=len, reverse=True):
+        if canonical == name or (canonical.startswith("airflow.") and canonical.endswith(f".{name}")):
+            return name
+    return canonical
+
+
+def _decorator_kwargs(
+    decorators: list[ast.expr],
+    names: frozenset[str],
+    aliases: dict[str, str] | None = None,
+) -> dict[str, ast.expr]:
     """Merged keyword args of the first decorator whose dotted name is in *names* (if it's a call)."""
     for dec in decorators:
-        if _decorator_name(dec) in names and isinstance(dec, ast.Call):
+        if _decorator_name(dec, aliases) in names and isinstance(dec, ast.Call):
             return {kw.arg: kw.value for kw in dec.keywords if kw.arg}
     return {}
 
@@ -1600,10 +1620,184 @@ def _decorator_kwargs(decorators: list[ast.expr], names: frozenset[str]) -> dict
     {"task", "task.branch", "task.virtualenv", "task.short_circuit", "task.sensor", "task.external_python"}
 )
 _TASK_GROUP_DECORATORS: frozenset[str] = frozenset({"task_group"})
+_ALL_AIRFLOW_DECORATORS = _DAG_DECORATORS | _TASK_DECORATORS | _TASK_GROUP_DECORATORS
 
 
-def _has_decorator(func: ast.FunctionDef, names: frozenset[str]) -> bool:
-    return any(_decorator_name(dec) in names for dec in func.decorator_list)
+def _has_decorator(
+    func: ast.FunctionDef,
+    names: frozenset[str],
+    aliases: dict[str, str] | None = None,
+) -> bool:
+    return any(_decorator_name(dec, aliases) in names for dec in func.decorator_list)
+
+
+def _statement_call(statement: ast.stmt) -> ast.Call | None:
+    """Returns the top-level call produced by an expression or simple assignment."""
+    if isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Call):
+        return statement.value
+    if isinstance(statement, ast.Assign) and isinstance(statement.value, ast.Call):
+        return statement.value
+    if isinstance(statement, ast.AnnAssign) and isinstance(statement.value, ast.Call):
+        return statement.value
+    return None
+
+
+def _statement_binding(statement: ast.stmt) -> str | None:
+    """Returns the single name bound by a top-level statement, when present."""
+    if isinstance(statement, ast.Assign) and len(statement.targets) == 1 and isinstance(statement.targets[0], ast.Name):
+        return statement.targets[0].id
+    if isinstance(statement, ast.AnnAssign) and isinstance(statement.target, ast.Name):
+        return statement.target.id
+    return None
+
+
+def _static_function_bindings(
+    function: ast.FunctionDef,
+    call: ast.Call,
+    constants: dict[str, Any],
+) -> dict[str, Any] | None:
+    """Binds a factory invocation when every argument is statically knowable."""
+    if function.args.vararg or function.args.kwarg or any(keyword.arg is None for keyword in call.keywords):
+        return None
+    positional = [*function.args.posonlyargs, *function.args.args]
+    keyword_only = list(function.args.kwonlyargs)
+    all_parameters = {parameter.arg for parameter in [*positional, *keyword_only]}
+    if len(call.args) > len(positional):
+        return None
+    expressions: dict[str, ast.expr] = {parameter.arg: argument for parameter, argument in zip(positional, call.args)}
+    for keyword in call.keywords:
+        if keyword.arg not in all_parameters or keyword.arg in expressions:
+            return None
+        expressions[keyword.arg] = keyword.value
+    positional_defaults = [None] * (len(positional) - len(function.args.defaults)) + list(function.args.defaults)
+    defaults = {
+        parameter.arg: default for parameter, default in zip(positional, positional_defaults) if default is not None
+    }
+    defaults.update(
+        {
+            parameter.arg: default
+            for parameter, default in zip(keyword_only, function.args.kw_defaults)
+            if default is not None
+        }
+    )
+    bindings: dict[str, Any] = {}
+    for parameter in [*positional, *keyword_only]:
+        expression = expressions.get(parameter.arg) or defaults.get(parameter.arg)
+        if expression is None:
+            return None
+        bound = _bind_constants(expression, constants)
+        value = _safe_static_value(bound, constants) if isinstance(bound, ast.expr) else _UNRESOLVED
+        if value is _UNRESOLVED:
+            return None
+        bindings[parameter.arg] = value
+    return bindings
+
+
+def _classic_dag_factory_body(
+    function: ast.FunctionDef,
+    aliases: dict[str, str],
+) -> tuple[list[ast.stmt], str | None] | None:
+    """Returns the narrow classic DAG-factory body and any assigned DAG variable."""
+    if function.decorator_list or function.args.vararg or function.args.kwarg:
+        return None
+    body = list(function.body)
+    if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant):
+        if isinstance(body[0].value.value, str):
+            body = body[1:]
+    if len(body) < 2 or not isinstance(body[-1], ast.Return) or not isinstance(body[-1].value, ast.Name):
+        return None
+    returned_name = body[-1].value.id
+    statements = body[:-1]
+    if len(statements) == 1 and isinstance(statements[0], ast.With):
+        dag_items = [
+            item
+            for item in statements[0].items
+            if isinstance(item.context_expr, ast.Call) and _construct_name(item.context_expr.func, aliases) == "DAG"
+        ]
+        if len(dag_items) != 1 or not isinstance(dag_items[0].optional_vars, ast.Name):
+            return None
+        if dag_items[0].optional_vars.id != returned_name:
+            return None
+        return statements, None
+    assigned_names: list[str] = []
+    for statement in statements:
+        if not (
+            isinstance(statement, ast.Assign)
+            and len(statement.targets) == 1
+            and isinstance(statement.targets[0], ast.Name)
+            and isinstance(statement.value, ast.Call)
+            and _construct_name(statement.value.func, aliases) == "DAG"
+        ):
+            continue
+        assigned_names.append(statement.targets[0].id)
+    if assigned_names != [returned_name]:
+        return None
+    return statements, returned_name
+
+
+def _decorated_factory_invocation(
+    call: ast.Call,
+    factories: dict[str, ast.FunctionDef],
+) -> tuple[ast.FunctionDef, ast.Call, dict[str, ast.expr]] | None:
+    """Resolves ``factory()`` and ``factory.override(...)(...)`` DAG invocations."""
+    if isinstance(call.func, ast.Name) and call.func.id in factories:
+        return factories[call.func.id], call, {}
+    if not isinstance(call.func, ast.Call):
+        return None
+    configuration = call.func
+    configuration_function = configuration.func
+    if not isinstance(configuration_function, ast.Attribute):
+        return None
+    if configuration_function.attr != "override" or not isinstance(configuration_function.value, ast.Name):
+        return None
+    name = configuration_function.value.id
+    if name not in factories or any(keyword.arg is None for keyword in configuration.keywords):
+        return None
+    overrides = {keyword.arg: keyword.value for keyword in configuration.keywords if keyword.arg}
+    return factories[name], call, overrides
+
+
+def _function_contains_dag_constructor(function: ast.FunctionDef, aliases: dict[str, str]) -> bool:
+    """Returns whether a function body contains a static Airflow ``DAG(...)`` call."""
+    return any(
+        isinstance(node, ast.Call) and _construct_name(node.func, aliases) == "DAG"
+        for statement in function.body
+        for node in ast.walk(statement)
+    )
+
+
+def _bound_decorated_factory(
+    declaration: DagDeclaration,
+    aliases: dict[str, str],
+) -> ast.FunctionDef:
+    """Clones one decorated factory invocation into an isolated static DAG definition."""
+    if declaration.factory is None:
+        raise ValueError("Decorated factory declaration has no function definition")
+    function = copy.deepcopy(declaration.factory)
+    function.body = [_bind_constants(statement, declaration.bindings) for statement in function.body]
+    for index, decorator in enumerate(function.decorator_list):
+        if _decorator_name(decorator, aliases) != "dag":
+            continue
+        if isinstance(decorator, ast.Call):
+            keywords = {keyword.arg: keyword for keyword in decorator.keywords if keyword.arg}
+            for name, value in declaration.decorator_overrides.items():
+                keywords[name] = ast.keyword(arg=name, value=copy.deepcopy(value))
+            decorator.keywords = list(keywords.values())
+        elif declaration.decorator_overrides:
+            function.decorator_list[index] = ast.copy_location(
+                ast.Call(
+                    func=decorator,
+                    args=[],
+                    keywords=[
+                        ast.keyword(arg=name, value=copy.deepcopy(value))
+                        for name, value in declaration.decorator_overrides.items()
+                    ],
+                ),
+                decorator,
+            )
+        break
+    ast.fix_missing_locations(function)
+    return function
 
 
 def load_airflow_dag(dag_path: Path, *, dbt_mode: str = "static") -> Pipeline:
@@ -1624,43 +1818,79 @@ def load_airflow_dags(
     source = Path(dag_path).read_text(encoding="utf-8")
     module = _expand_top_level_loops(ast.parse(source))
     declarations = _top_level_dag_declarations(module)
-    if not declarations:
-        return [
+    pipelines: list[Pipeline] = []
+    for declaration in declarations:
+        if declaration.unsupported_reason is not None:
+            pipelines.append(
+                _failed_dag_declaration_pipeline(
+                    dag_path,
+                    declaration,
+                    source_file=source_file or dag_path.name,
+                )
+            )
+            continue
+        pipelines.append(
             _load_airflow_module(
                 dag_path,
                 source,
-                module,
+                _module_for_dag(module, declaration, declarations),
                 dbt_mode=dbt_mode,
+                target_dag_variable=declaration.target_dag_variable,
                 source_file=source_file or dag_path.name,
             )
-        ]
-    return [
-        _load_airflow_module(
-            dag_path,
-            source,
-            _module_for_dag(module, declaration),
-            dbt_mode=dbt_mode,
-            target_dag_variable=declaration.variable,
-            source_file=source_file or dag_path.name,
         )
-        for declaration in declarations
-    ]
+    return pipelines
 
 
 def _top_level_dag_declarations(module: ast.Module) -> list[DagDeclaration]:
-    """Returns context-manager, decorated, and assigned top-level DAG declarations."""
+    """Returns direct DAG declarations and statically invoked DAG factories."""
     aliases = _import_aliases(module)
+    functions = {node.name: node for node in module.body if isinstance(node, ast.FunctionDef)}
+    decorated_factories = {
+        name: function for name, function in functions.items() if _has_decorator(function, _DAG_DECORATORS, aliases)
+    }
+    classic_factories = {
+        name: function
+        for name, function in functions.items()
+        if name not in decorated_factories and _function_contains_dag_constructor(function, aliases)
+    }
     declarations: list[DagDeclaration] = []
+    constants: dict[str, Any] = {}
+
+    def add(
+        node: ast.stmt,
+        *,
+        variable: str | None = None,
+        kind: str = "direct",
+        factory: ast.FunctionDef | None = None,
+        bindings: dict[str, Any] | None = None,
+        target_dag_variable: str | None = None,
+        decorator_overrides: dict[str, ast.expr] | None = None,
+        unsupported_reason: str | None = None,
+    ) -> None:
+        span = _span(node)
+        declarations.append(
+            DagDeclaration(
+                capture_id=f"dag:{span.line}:{span.column}:{len(declarations) + 1}",
+                variable=variable,
+                node=node,
+                span=span,
+                kind=kind,
+                factory=factory,
+                bindings=bindings or {},
+                target_dag_variable=target_dag_variable,
+                decorator_overrides=decorator_overrides or {},
+                unsupported_reason=unsupported_reason,
+            )
+        )
+
     for node in module.body:
-        variable: str | None = None
-        is_dag = False
-        if isinstance(node, ast.FunctionDef) and _has_decorator(node, _DAG_DECORATORS):
-            is_dag = True
-        elif isinstance(node, ast.With) and any(
+        if isinstance(node, ast.With) and any(
             isinstance(item.context_expr, ast.Call) and _construct_name(item.context_expr.func, aliases) == "DAG"
             for item in node.items
         ):
-            is_dag = True
+            add(node)
+            continue
         elif (
             isinstance(node, ast.Assign)
             and len(node.targets) == 1
@@ -1668,26 +1898,156 @@ def _top_level_dag_declarations(module: ast.Module) -> list[DagDeclaration]:
             and isinstance(node.value, ast.Call)
             and _construct_name(node.value.func, aliases) == "DAG"
         ):
-            is_dag = True
             variable = node.targets[0].id
-        if is_dag:
-            span = _span(node)
-            declarations.append(
-                DagDeclaration(
-                    capture_id=f"dag:{span.line}:{span.column}:{len(declarations) + 1}",
-                    variable=variable,
-                    node=node,
-                    span=span,
+            add(node, variable=variable, target_dag_variable=variable)
+            continue
+
+        call = _statement_call(node)
+        binding = _statement_binding(node)
+        if call is not None:
+            decorated = _decorated_factory_invocation(call, decorated_factories)
+            if decorated is not None:
+                factory, invocation, overrides = decorated
+                bindings = _static_function_bindings(factory, invocation, constants)
+                bound_overrides = {name: _bind_constants(value, constants) for name, value in overrides.items()}
+                dag_id_override = bound_overrides.get("dag_id")
+                reason = None
+                if bindings is None:
+                    reason = "Decorated DAG factory arguments are not statically bindable."
+                elif dag_id_override is not None and ops.literal_str(dag_id_override) is None:
+                    reason = "Decorated DAG factory dag_id override is not a literal string."
+                add(
+                    node,
+                    variable=binding,
+                    kind="decorated_factory",
+                    factory=factory,
+                    bindings=bindings,
+                    decorator_overrides=bound_overrides,
+                    unsupported_reason=reason,
                 )
+                continue
+
+            if isinstance(call.func, ast.Name) and call.func.id in classic_factories:
+                factory = classic_factories[call.func.id]
+                factory_body = _classic_dag_factory_body(factory, aliases)
+                bindings = _static_function_bindings(factory, call, constants)
+                reason = None
+                target_dag_variable = None
+                if factory_body is None:
+                    reason = "Classic DAG factory body is outside the supported static shape."
+                elif bindings is None:
+                    reason = "Classic DAG factory arguments are not statically bindable."
+                else:
+                    _statements, target_dag_variable = factory_body
+                add(
+                    node,
+                    variable=binding,
+                    kind="classic_factory",
+                    factory=factory,
+                    bindings=bindings,
+                    target_dag_variable=target_dag_variable,
+                    unsupported_reason=reason,
+                )
+                continue
+
+        if not isinstance(node, ast.FunctionDef) and any(
+            isinstance(candidate, ast.Call) and _construct_name(candidate.func, aliases) == "DAG"
+            for candidate in ast.walk(node)
+        ):
+            add(
+                node,
+                variable=binding,
+                kind="unsupported",
+                unsupported_reason="DAG construction is outside the supported static declaration shapes.",
             )
+            continue
+
+        if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+            value = _safe_static_value(node.value, constants)
+            if value is not _UNRESOLVED:
+                constants[node.targets[0].id] = value
     return declarations
 
 
-def _module_for_dag(module: ast.Module, declaration: DagDeclaration) -> ast.Module:
+def _module_for_dag(
+    module: ast.Module,
+    declaration: DagDeclaration,
+    declarations: list[DagDeclaration],
+) -> ast.Module:
     """Returns a module containing shared definitions and one DAG declaration."""
-    dag_nodes = {item.node for item in _top_level_dag_declarations(module)}
-    body = [node for node in module.body if node is declaration.node or node not in dag_nodes]
-    return ast.Module(body=body, type_ignores=list(module.type_ignores))
+    aliases = _import_aliases(module)
+    declaration_nodes = {item.node for item in declarations}
+    decorated_factories = {
+        item.factory for item in declarations if item.kind == "decorated_factory" and item.factory is not None
+    }
+    body: list[ast.stmt] = []
+    for node in module.body:
+        if node in decorated_factories:
+            if declaration.kind == "decorated_factory" and node is declaration.factory:
+                body.append(_bound_decorated_factory(declaration, aliases))
+            continue
+        if node in declaration_nodes:
+            if node is not declaration.node:
+                continue
+            if declaration.kind == "direct":
+                body.append(node)
+            elif declaration.kind == "classic_factory" and declaration.factory is not None:
+                factory_body = _classic_dag_factory_body(declaration.factory, aliases)
+                if factory_body is None:
+                    raise ValueError("Supported classic DAG factory has no static body")
+                statements, _target = factory_body
+                body.extend(_bind_constants(statement, declaration.bindings) for statement in statements)
+            continue
+        body.append(node)
+    isolated = ast.Module(body=body, type_ignores=list(module.type_ignores))
+    ast.fix_missing_locations(isolated)
+    return isolated
+
+
+def _failed_dag_declaration_pipeline(
+    dag_path: Path,
+    declaration: DagDeclaration,
+    *,
+    source_file: str,
+) -> Pipeline:
+    """Returns a failed, reportable pipeline for an unrepresentable DAG declaration."""
+    candidate = source_audit.AuditCandidate(
+        kind="dag",
+        code="unsupported_dag_factory",
+        line=declaration.span.line,
+        column=declaration.span.column,
+        occurrence=1,
+        end_line=declaration.span.end_line,
+        end_column=declaration.span.end_column,
+        details={"expression": ast.unparse(declaration.node)},
+    )
+    finding = source_audit.finding(
+        source_file=source_file,
+        code="unsupported_dag_factory",
+        severity="failed",
+        message=declaration.unsupported_reason or "Airflow DAG declaration could not be captured statically.",
+        candidate=candidate,
+    )
+    name = declaration.variable or Path(dag_path).stem
+    return Pipeline(
+        name=name,
+        tasks=[],
+        tags={"source": "airflow", "dag_id": name},
+        not_translatable=[finding],
+        reconciliation_status="failed",
+        audit={
+            "source_file": source_file,
+            "audited_activity_count": 1,
+            "captured_task_count": 0,
+            "audited_edge_count": 0,
+            "captured_edge_count": 0,
+            "deterministic_count": 0,
+            "agentic_count": 0,
+            "failed_count": 1,
+            "excluded_count": 0,
+            "transformations": [],
+        },
+    )
 
 
 def _load_airflow_module(
@@ -3081,8 +3441,8 @@ def discover_dags(source_path: Path) -> list[Path]:
     """Returns the DAG ``.py`` files under *source_path*.
 
     Accepts either a single ``.py`` file or a directory (scanned recursively).
-    Files whose source contains no ``DAG(`` construct are skipped so helper
-    modules in a DAGs folder are not mistaken for DAG definitions.
+    Discovery uses the same static declaration model as loading, including
+    import aliases and qualified TaskFlow decorators.
     """
     source_path = Path(source_path)
     candidates = [source_path] if source_path.is_file() else sorted(source_path.rglob("*.py"))
@@ -3091,10 +3451,10 @@ def discover_dags(source_path: Path) -> list[Path]:
         if candidate.suffix != ".py":
             continue
         try:
-            text = candidate.read_text(encoding="utf-8")
-        except OSError:
+            module = _expand_top_level_loops(ast.parse(candidate.read_text(encoding="utf-8")))
+        except (OSError, SyntaxError):
             continue
-        if "DAG(" in text or "@dag" in text:
+        if _top_level_dag_declarations(module):
             dags.append(candidate)
     return dags
 
diff --git a/tests/unit/test_airflow_reconciliation.py b/tests/unit/test_airflow_reconciliation.py
index 283ae2e..80255c8 100644
--- a/tests/unit/test_airflow_reconciliation.py
+++ b/tests/unit/test_airflow_reconciliation.py
@@ -407,3 +407,142 @@ def test_excluded_dag_stays_audited_and_included_reference_becomes_placeholder(t
     assert isinstance(by_name["caller"].tasks[0], PlaceholderActivity)
     assert by_name["caller"].tasks[0].raw_definition == {"excluded_dag": "target"}
     assert by_name["caller"].reconciliation_status == "verified_with_gaps"
+
+
+@pytest.mark.parametrize(
+    ("filename", "dag_import", "decorator"),
+    [
+        ("aliased.py", "from airflow.decorators import dag as workflow", "workflow"),
+        ("qualified.py", "import airflow.sdk", "airflow.sdk.dag"),
+    ],
+)
+def test_discovery_resolves_aliased_and_qualified_dag_decorators(
+    tmp_path: Path,
+    filename: str,
+    dag_import: str,
+    decorator: str,
+) -> None:
+    dag_path = tmp_path / filename
+    dag_id = dag_path.stem
+    dag_path.write_text(
+        f"{dag_import}\n"
+        "from airflow.operators.bash import BashOperator\n"
+        f"@{decorator}(dag_id='{dag_id}')\n"
+        "def build():\n"
+        "    BashOperator(task_id='work', bash_command='echo work')\n"
+        "build()\n",
+        encoding="utf-8",
+    )
+
+    assert airflow_loader.discover_dags(dag_path) == [dag_path]
+    pipeline = airflow_loader.load_pipelines(dag_path)[0]
+    assert pipeline.name == dag_id
+    assert pipeline.reconciliation_status == "verified"
+    assert [task.task_key for task in pipeline.tasks] == ["work"]
+
+
+def test_mixed_directory_does_not_hide_an_aliased_dag(tmp_path: Path) -> None:
+    canonical = tmp_path / "canonical.py"
+    canonical.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='canonical') as dag:\n"
+        "    BashOperator(task_id='work', bash_command='echo canonical')\n",
+        encoding="utf-8",
+    )
+    aliased = tmp_path / "aliased.py"
+    aliased.write_text(
+        "from airflow.decorators import dag as workflow\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "@workflow(dag_id='aliased')\n"
+        "def build():\n"
+        "    BashOperator(task_id='work', bash_command='echo aliased')\n"
+        "build()\n",
+        encoding="utf-8",
+    )
+
+    assert airflow_loader.discover_dags(tmp_path) == [aliased, canonical]
+    assert {pipeline.name for pipeline in airflow_loader.load_pipelines(tmp_path)} == {"aliased", "canonical"}
+
+
+def test_taskflow_alias_is_captured_inside_a_recognized_dag(tmp_path: Path) -> None:
+    dag_path = tmp_path / "task_alias.py"
+    dag_path.write_text(
+        "from airflow.decorators import dag, task as step\n"
+        "@step\n"
+        "def work():\n"
+        "    return 1\n"
+        "@dag(dag_id='task_alias')\n"
+        "def build():\n"
+        "    work()\n"
+        "build()\n",
+        encoding="utf-8",
+    )
+
+    pipeline = airflow_loader.load_pipelines(dag_path)[0]
+
+    assert pipeline.reconciliation_status == "verified"
+    assert [task.task_key for task in pipeline.tasks] == ["work"]
+
+
+def test_static_classic_dag_factory_preserves_dag_identity_and_tasks(tmp_path: Path) -> None:
+    dag_path = tmp_path / "classic_factory.py"
+    dag_path.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "def make_dag(dag_id, message='default'):\n"
+        "    with DAG(dag_id=dag_id) as dag:\n"
+        "        BashOperator(task_id='work', bash_command=f'echo {message}')\n"
+        "    return dag\n"
+        "factory_dag = make_dag('factory_dag', message='hello')\n",
+        encoding="utf-8",
+    )
+
+    pipeline = airflow_loader.load_pipelines(dag_path)[0]
+
+    assert pipeline.name == "factory_dag"
+    assert pipeline.reconciliation_status == "verified"
+    assert [task.task_key for task in pipeline.tasks] == ["work"]
+    assert "echo hello" in (pipeline.tasks[0].generated_source or "")
+
+
+def test_decorated_dag_factory_override_emits_each_invocation(tmp_path: Path) -> None:
+    dag_path = tmp_path / "decorated_factory.py"
+    dag_path.write_text(
+        "from airflow.decorators import dag\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "@dag\n"
+        "def build(message='default'):\n"
+        "    BashOperator(task_id='work', bash_command=f'echo {message}')\n"
+        "first = build.override(dag_id='first')('one')\n"
+        "second = build.override(dag_id='second')('two')\n",
+        encoding="utf-8",
+    )
+
+    pipelines = airflow_loader.load_pipelines(dag_path)
+
+    assert [pipeline.name for pipeline in pipelines] == ["first", "second"]
+    assert all(pipeline.reconciliation_status == "verified" for pipeline in pipelines)
+    assert "echo one" in (pipelines[0].tasks[0].generated_source or "")
+    assert "echo two" in (pipelines[1].tasks[0].generated_source or "")
+
+
+def test_dynamic_dag_factory_fails_closed_instead_of_emitting_verified_empty_ir(tmp_path: Path) -> None:
+    dag_path = tmp_path / "dynamic_factory.py"
+    dag_path.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "def make_dag(dag_id):\n"
+        "    with DAG(dag_id=dag_id) as dag:\n"
+        "        BashOperator(task_id='work', bash_command='echo work')\n"
+        "    return dag\n"
+        "factory_dag = make_dag(runtime_dag_id())\n",
+        encoding="utf-8",
+    )
+
+    pipeline = airflow_loader.load_pipelines(dag_path)[0]
+
+    assert pipeline.name == "factory_dag"
+    assert pipeline.reconciliation_status == "failed"
+    assert pipeline.audit["failed_count"] == 1
+    assert any(finding["code"] == "unsupported_dag_factory" for finding in pipeline.not_translatable)

From 163b53a7b96bf328835939aba93215d75ab2e722 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Sat, 8 Aug 2026 14:32:55 -0700
Subject: [PATCH 54/77] Close unsafe Airflow agentic merge path

---
 app/README.md                                 |   2 +-
 skills/flowx-convert/SKILL.md                 |  13 +-
 .../flowx-convert/sources/airflow-coverage.md |   2 +-
 skills/flowx-convert/sources/airflow.md       |   5 +-
 skills/flowx-migrate/SKILL.md                 |   7 +-
 src/flowx/bundler/dab_writer.py               |  99 ++++++++--
 src/flowx/ir_serde.py                         |   5 +-
 src/flowx/mcp/server.py                       |  13 +-
 src/flowx/sources/airflow/convert.py          |  22 +--
 tests/unit/test_airflow_operators.py          |  29 +--
 tests/unit/test_airflow_reconciliation.py     |  11 ++
 tests/unit/test_mcp_source_routing.py         |  16 ++
 tests/unit/test_package_invariants.py         | 186 +++++++++++++++---
 tests/unit/test_source_router.py              |  17 ++
 14 files changed, 343 insertions(+), 84 deletions(-)

diff --git a/app/README.md b/app/README.md
index 3faa3d7..beafbec 100644
--- a/app/README.md
+++ b/app/README.md
@@ -20,7 +20,7 @@ operation; `parameters` is its keyword-argument dict.
 | `inputs` | `adapter inputs` | List a phase's input prompts/defaults |
 | `discover` | `adapter discover` | Parse ADF JSON, classify activities |
 | `convert` | `adapter convert` | ADF activities → Databricks IR |
-| `merge_agentic` | `adapter convert --merge-agentic` | Merge agent-produced results into the report |
+| `merge_agentic` | `adapter convert --merge-agentic` | Merge ADF agent-produced results into the report |
 | `inspect` | `adapter inspect` | Surface pending translation options |
 | `apply_answers` | `adapter modify` | Apply answers → stamped IR |
 | `materialize_lookup` | `adapter materialize-lookup` | CSV → lookup-values JSON |
diff --git a/skills/flowx-convert/SKILL.md b/skills/flowx-convert/SKILL.md
index 4a1221e..ab12fb8 100644
--- a/skills/flowx-convert/SKILL.md
+++ b/skills/flowx-convert/SKILL.md
@@ -20,7 +20,8 @@ phase 2 of the flowx migration workflow; it produces a transient translation rep
 
 Translation is **source-specific** (ADF activity translators vs. Airflow operator mapping), so this
 skill routes to the right source guide. The shared mechanics — how to run the phase, the report
-contract, and the `inspect`/`modify`/`merge_agentic` machinery — live here.
+contract, and the `inspect`/`modify` machinery — live here. The legacy `merge_agentic` command is
+ADF-only; Airflow rejects it until the fingerprint-bound resolution workflow is available.
 
 ## Step 1 — Identify the source (required)
 
@@ -64,15 +65,15 @@ across ADF and Airflow.
 
 ## Shared adapter commands
 
-These operate on the report, not on a source's raw definitions, so they are the same for every
-source (the ADF guide uses them heavily; Airflow currently needs only the base convert):
+`inspect` and `modify` operate on the report rather than raw source definitions. The ADF guide uses
+them heavily; Airflow currently needs only the base conversion:
 
 - `inspect ` — emit the full just-in-time option schema (each option annotated with a
   `show_when` condition). Walk it locally; ask an option only when its `show_when` is satisfied.
 - `modify  --output-dir  --answer OPTION_ID=VALUE ...` — validate and apply collected
   answers, writing `.work/translation_report.stamped.json` + `metadata/configuration.json`.
-- `merge_agentic --report  --agentic-results ` — fold agent-produced per-activity
-  translations into the report (placeholders replaced in place).
+- `merge_agentic --report  --agentic-results ` — **ADF only**. Fold agent-produced
+  per-activity translations into an ADF report. Airflow's legacy name-based merge is disabled.
 
 ## Output artifacts (shared, transient under `/.work/`)
 
@@ -80,7 +81,7 @@ source (the ADF guide uses them heavily; Airflow currently needs only the base c
 |---|---|
 | `.work/translation_report.json` | Full translation report with IR for all tasks |
 | `.work/.json` | Per-pipeline Databricks IR |
-| `.work/gaps.json` | Agentic gaps awaiting LLM-assisted conversion (ADF) |
+| `.work/gaps.json` | Unmapped source constructs; agentic inputs for ADF and review-only gaps for Airflow |
 | `.work/translation_report.stamped.json` | Configuration-stamped report (written by `modify`) |
 
 ## Reference
diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md
index 32acdcc..8eeb35e 100644
--- a/skills/flowx-convert/sources/airflow-coverage.md
+++ b/skills/flowx-convert/sources/airflow-coverage.md
@@ -40,7 +40,7 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't
 | Multiple DAGs | Every DAG, including multiple declarations and repeated static `@dag` factory invocations in one Python file, becomes a sibling job in one shared Airflow bundle so `TriggerDagRunOperator` resource references resolve. Narrow classic factories shaped as one DAG declaration followed by `return dag` are expanded with statically bindable arguments. |
 
 Any operator not listed becomes a `PlaceholderActivity` **and** a `gaps.json` entry carrying the
-operator's raw source for review. Airflow agentic replacement is not a supported workflow yet. The
+operator's raw source for review. The legacy `merge_agentic` command is disabled for Airflow. The
 safe fallback is a flagged, failing task rather than a silent omission. Callables that read Airflow task context
 (`**context` / `ti`) or XCom, and runtime-branching decorators, take the same route rather than
 emitting code that fails at runtime.
diff --git a/skills/flowx-convert/sources/airflow.md b/skills/flowx-convert/sources/airflow.md
index 5be40d8..4c5bbc6 100644
--- a/skills/flowx-convert/sources/airflow.md
+++ b/skills/flowx-convert/sources/airflow.md
@@ -42,8 +42,9 @@ PythonOperator callable or BashOperator command, carrying `generated_source`) or
 
 If convert wrote `/.work/gaps.json`, each entry describes an unmapped construct whose
 generated Job task points to a notebook that raises `NotImplementedError`. Review every gap before
-deployment. The shared `merge_agentic` command is not a supported Airflow workflow yet; keep the
-placeholder, exclude the DAG, or implement and validate the replacement explicitly.
+deployment. The shared `merge_agentic` command is disabled for Airflow and rejects
+`--source airflow`; keep the placeholder or exclude the DAG. This prevents an unsafe name-based
+replacement from bypassing source reconciliation.
 
 ## Step 4 — Proceed to package
 
diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md
index 81cca14..361cfab 100644
--- a/skills/flowx-migrate/SKILL.md
+++ b/skills/flowx-migrate/SKILL.md
@@ -96,14 +96,15 @@ To accept all defaults and skip the prompts, pass `"interactive": false`. (Re-ca
 
 For step-by-step control, run the commands in order (the app reuses `output_dir` across calls, so
 only `discover` needs the source input). `source` ("adf" | "airflow") is required for
-discover/convert/merge_agentic and for `inputs discover`/`inputs convert`; for Airflow, swap
-`adf_definitions` for `airflow_source_path`. `package` and `inputs package` are source-independent:
+discover/convert and for `inputs discover`/`inputs convert`; for Airflow, swap `adf_definitions` for
+`airflow_source_path`. `merge_agentic` is ADF-only. `package` and `inputs package` are
+source-independent:
 
 ```
 flowx(command="inputs", parameters={"phase": "discover", "source": "adf"})  # source req for discover/convert
 flowx(command="discover", parameters={"source": "adf", "adf_definitions": {...}, "output_dir": ..., "pipeline": ...})
 flowx(command="convert", parameters={"source": "adf", "output_dir": ..., "pipeline": ...})
-flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...})  # if agentic results
+flowx(command="merge_agentic", parameters={"source": "adf", "report_path": ..., "agentic_results_dir": ..., "output_path": ...})  # ADF only, if agentic results
 flowx(command="inspect", parameters={"report_path": ...})
 flowx(command="apply_answers", parameters={"report_path": ..., "answers": [...], "output_dir": ...})
 flowx(command="package", parameters={"output_dir": ..., "catalog": ..., "schema": ...})
diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py
index abcdb5d..81c0c5a 100644
--- a/src/flowx/bundler/dab_writer.py
+++ b/src/flowx/bundler/dab_writer.py
@@ -439,10 +439,10 @@ def main(argv: list[str] | None = None) -> int:
         print(f"Error: Report file not found: {args.report}", file=sys.stderr)
         return 1
 
-    reconciliation_failures = _report_reconciliation_failures(args.report)
-    if reconciliation_failures:
-        print("Error: source reconciliation failed; no bundle files were written.", file=sys.stderr)
-        for failure in reconciliation_failures:
+    report_failures = _report_reconciliation_failures(args.report)
+    if report_failures:
+        print("Error: translation report preflight failed; no bundle files were written.", file=sys.stderr)
+        for failure in report_failures:
             print(f"  - {failure}", file=sys.stderr)
         return 1
 
@@ -1715,21 +1715,96 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]:
 
 
 def _report_reconciliation_failures(report_path: Path) -> list[str]:
-    """Returns included pipelines whose source reconciliation is not package-safe."""
-    with report_path.open(encoding="utf-8") as handle:
-        report = json.load(handle)
-    if isinstance(report, dict) and isinstance(report.get("pipelines"), list):
+    """Returns report-shape, source-contract, and reconciliation failures.
+
+    Packaging is a security boundary for source reconciliation. A malformed report or an
+    unrecognized status must fail closed rather than falling through as an empty/safe report.
+    """
+    try:
+        with report_path.open(encoding="utf-8") as handle:
+            report = json.load(handle)
+    except json.JSONDecodeError as error:
+        return [f"translation report contains invalid JSON: {error}"]
+    except OSError as error:
+        return [f"translation report could not be read: {error}"]
+
+    if not isinstance(report, dict):
+        return ["translation report must be a top-level object"]
+
+    shape_keys = [key for key in ("tasks", "pipelines", "translations") if key in report]
+    if not shape_keys:
+        return ["translation report does not match a recognized report shape"]
+    if len(shape_keys) > 1:
+        return [f"translation report contains ambiguous report shapes: {', '.join(shape_keys)}"]
+
+    shape = shape_keys[0]
+    if shape == "pipelines":
+        if not isinstance(report["pipelines"], list):
+            return ["translation report pipelines must be a list"]
+        if not report["pipelines"]:
+            return ["translation report pipelines must contain at least one pipeline"]
         pipelines = report["pipelines"]
-    elif isinstance(report, dict) and "tasks" in report:
+    elif shape == "tasks":
         pipelines = [report]
     else:
+        if report.get("source") not in {None, "adf"}:
+            return ["legacy ADF translations report cannot declare a non-ADF source"]
+        translations = report["translations"]
+        if not isinstance(translations, list) or not translations:
+            return ["legacy ADF translations must be a non-empty list"]
+        for index, translation in enumerate(translations):
+            if not isinstance(translation, dict):
+                return [f"legacy ADF translation at index {index} must be an object"]
+            if not isinstance(translation.get("pipeline"), str) or not isinstance(translation.get("ir"), dict):
+                return [f"legacy ADF translation at index {index} is missing pipeline or IR data"]
         return []
+
     failures: list[str] = []
-    for pipeline in pipelines:
-        if not isinstance(pipeline, dict) or pipeline.get("migration_status") == "excluded":
+    airflow_statuses = {"verified", "verified_with_gaps", "failed"}
+    required_airflow_audit_fields = {"source_file", "audited_activity_count", "transformations"}
+    for index, pipeline in enumerate(pipelines):
+        label = f"pipeline[{index}]"
+        if not isinstance(pipeline, dict):
+            failures.append(f"{label}: pipeline must be an object")
+            continue
+        name = pipeline.get("name")
+        if not isinstance(name, str) or not name:
+            failures.append(f"{label}: pipeline name must be a non-empty string")
+            continue
+        label = name
+        if not isinstance(pipeline.get("tasks"), list):
+            failures.append(f"{label}: pipeline tasks must be a list")
+            continue
+        tags = pipeline.get("tags")
+        source = tags.get("source") if isinstance(tags, dict) else None
+        if source not in {"adf", "airflow"}:
+            failures.append(f"{label}: pipeline tags.source must be 'adf' or 'airflow'")
+            continue
+
+        status = pipeline.get("reconciliation_status")
+        if source == "adf":
+            if status not in {None, "not_applicable"}:
+                failures.append(f"{label}: unknown reconciliation_status {status!r} for ADF")
+            continue
+
+        audit = pipeline.get("audit")
+        if not isinstance(audit, dict) or not required_airflow_audit_fields.issubset(audit):
+            failures.append(f"{label}: Airflow source-audit metadata is missing or incomplete")
+            continue
+        migration_status = pipeline.get("migration_status", "included")
+        if migration_status not in {"included", "excluded"}:
+            failures.append(f"{label}: unknown migration_status {migration_status!r} for Airflow")
             continue
-        if pipeline.get("reconciliation_status") != "failed":
+        if status == "excluded":
+            if migration_status != "excluded":
+                failures.append(f"{label}: reconciliation_status 'excluded' requires migration_status 'excluded'")
             continue
+        if status not in airflow_statuses:
+            failures.append(f"{label}: unknown reconciliation_status {status!r} for Airflow")
+            continue
+        if migration_status == "excluded" or status != "failed":
+            continue
+
         findings = [
             finding
             for finding in pipeline.get("not_translatable") or []
diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py
index a0ca902..b253ee8 100644
--- a/src/flowx/ir_serde.py
+++ b/src/flowx/ir_serde.py
@@ -7,8 +7,9 @@
 belongs to the IR, not to ADF: the Airflow convert phase and the bundler import
 it just as the ADF engine does.
 
-Also hosts ``merge_agentic_results`` -- the agentic-gap merge operates purely on
-serialised report dicts, so it is source-neutral too.
+Also hosts the legacy ``merge_agentic_results`` implementation used by the ADF
+source. Airflow does not expose this name-based merge because it cannot preserve
+the source-audit and graph-identity guarantees.
 """
 
 from __future__ import annotations
diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py
index f751e36..9ef808a 100644
--- a/src/flowx/mcp/server.py
+++ b/src/flowx/mcp/server.py
@@ -214,10 +214,16 @@ def _cmd_convert(p: dict[str, Any]) -> dict[str, Any]:
 
 
 def _cmd_merge_agentic(p: dict[str, Any]) -> dict[str, Any]:
+    source_name = _source_name(p)
+    if source_name == "airflow":
+        return {
+            "ok": False,
+            "error": "Airflow agentic merge is disabled until the fingerprint-bound resolution workflow is available.",
+        }
     args = [
         "convert",
         "--source",
-        _source_name(p),
+        source_name,
         "--merge-agentic",
         "--report",
         p["report_path"],
@@ -478,8 +484,9 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A
           pipeline, exclude_dag | exclude_dags (Airflow, repeatable list) — parse and audit definitions.
         - "convert": source(req), (one ADF source key | airflow_source_path), output_dir, pipeline,
           exclude_dag | exclude_dags (Airflow, repeatable list).
-        - "merge_agentic": source(req), report_path(req), agentic_results_dir(req), output_path —
-          merge agent results.
+        - "merge_agentic": source(req: "adf"), report_path(req), agentic_results_dir(req), output_path —
+          merge ADF agent results. Airflow's legacy name-based merge is disabled pending the
+          fingerprint-bound resolution workflow.
         - "inspect": report_path(req) — return the full translation-option schema (every option with
           a `show_when` condition) for the agent to walk locally. See "Collecting options" below.
         - "apply_answers": report_path(req), answers(req, list of "ID=VALUE"), output_dir, lookup_csv.
diff --git a/src/flowx/sources/airflow/convert.py b/src/flowx/sources/airflow/convert.py
index e0bbe28..9ae29cb 100644
--- a/src/flowx/sources/airflow/convert.py
+++ b/src/flowx/sources/airflow/convert.py
@@ -14,7 +14,6 @@
 import logging
 from pathlib import Path
 
-from flowx import ir_serde
 from flowx.adapter.predicates import walk_activities
 from flowx.ir_serde import pipeline_to_dict
 from flowx.models.ir import PlaceholderActivity
@@ -45,7 +44,7 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument(
         "--merge-agentic",
         action="store_true",
-        help="Merge agent-produced results from --agentic-results into --report instead of translating.",
+        help="Deprecated and disabled for Airflow; retained only to return a migration error.",
     )
     parser.add_argument("--report", type=Path, default=None, help="Translation report to merge agentic results into.")
     parser.add_argument(
@@ -60,17 +59,11 @@ def main(argv: list[str] | None = None) -> int:
     logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
 
     if args.merge_agentic:
-        if not args.report or not args.agentic_results:
-            parser.error("--merge-agentic requires --report and --agentic-results")
-        merged_count, unmatched_count = ir_serde.merge_agentic_results(args.report, args.agentic_results, args.output)
-        print("\nAgentic Merge Summary")
-        print("=====================")
-        print(f"Merged:    {merged_count}")
-        print(f"Unmatched: {unmatched_count}")
-        return 0 if unmatched_count == 0 else 1
+        logger.error("Airflow agentic merge is disabled until the fingerprint-bound resolution workflow is available.")
+        return 2
 
     if not args.source_dir:
-        parser.error("--source-dir is required (unless using --merge-agentic)")
+        parser.error("--source-dir is required")
 
     pipelines = load_pipelines(
         args.source_dir,
@@ -91,9 +84,7 @@ def main(argv: list[str] | None = None) -> int:
     report_file = work_dir / "translation_report.json"
     report_file.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
 
-    # Emit gaps.json for every unmapped operator so the agentic-gap round (driven by the
-    # convert SKILL guide + merge_agentic) can reason from the operator source and replace
-    # the placeholder with a real notebook -- the same flow the ADF source uses.
+    # Preserve unmapped operator context for review and the future fingerprint-bound resolver.
     gaps = _collect_gaps(pipelines)
     if gaps:
         (work_dir / "gaps.json").write_text(json.dumps(gaps, indent=2, default=str), encoding="utf-8")
@@ -112,8 +103,7 @@ def _collect_gaps(pipelines: list) -> list[dict]:
     """Returns one AgenticGap-shaped dict per PlaceholderActivity across all pipelines.
 
     Each carries the placeholder's ``activity_name``, ``activity_type`` (the Airflow
-    operator), and ``raw_definition`` (the operator's source) so the agentic round can
-    translate it -- the Airflow analog of ADF's gaps.json.
+    operator), and ``raw_definition`` (the operator's source).
     """
     gaps: list[dict] = []
     for pipeline in pipelines:
diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py
index 401e007..c690dd9 100644
--- a/tests/unit/test_airflow_operators.py
+++ b/tests/unit/test_airflow_operators.py
@@ -946,8 +946,8 @@ def test_unknown_operator_becomes_placeholder():
 
 
 def test_placeholder_carries_operator_source_for_agentic_round():
-    # The placeholder must carry the operator's raw source so the agentic-gap round
-    # (gaps.json + merge_agentic) can reason from it, like the ADF source's ARM JSON.
+    # The placeholder must carry the operator's raw source so a reviewed resolution workflow can
+    # reason from it without reparsing or executing the DAG.
     p = _load(
         "from airflow import DAG\n"
         "with DAG(dag_id='d') as dag:\n"
@@ -981,7 +981,7 @@ def test_convert_emits_gaps_json_for_unmapped_operators():
         assert gaps[0]["raw_definition"]["source"]
 
 
-def test_convert_merges_agentic_results_without_source_dir(tmp_path):
+def test_convert_rejects_legacy_agentic_merge_without_modifying_report(tmp_path):
     import json
 
     from flowx.sources.airflow.convert import main
@@ -994,8 +994,10 @@ def test_convert_merges_agentic_results_without_source_dir(tmp_path):
                 "tasks": [
                     {
                         "type": "PlaceholderActivity",
-                        "name": "pod",
-                        "task_key": "pod",
+                        "name": "b",
+                        "task_key": "b",
+                        "depends_on": [{"task_key": "a"}],
+                        "max_retries": 1,
                         "original_type": "KubernetesPodOperator",
                     }
                 ],
@@ -1007,20 +1009,23 @@ def test_convert_merges_agentic_results_without_source_dir(tmp_path):
     (results / "pod.json").write_text(
         json.dumps(
             {
-                "activity_name": "pod",
+                "activity_name": "b",
                 "task": {
                     "type": "NotebookActivity",
-                    "name": "pod",
-                    "task_key": "pod",
-                    "notebook_path": "notebooks/pod.py",
+                    "name": "b",
+                    "task_key": "HIJACKED_KEY",
+                    "depends_on": [],
+                    "max_retries": 99,
+                    "notebook_path": "/Workspace/evil",
                 },
             }
         )
     )
 
-    assert main(["--merge-agentic", "--report", str(report), "--agentic-results", str(results)]) == 0
-    merged = json.loads(report.read_text())
-    assert merged["tasks"][0]["type"] == "NotebookActivity"
+    original = report.read_text()
+
+    assert main(["--merge-agentic", "--report", str(report), "--agentic-results", str(results)]) == 2
+    assert report.read_text() == original
 
 
 # --------------------------------------------------------------------------------------
diff --git a/tests/unit/test_airflow_reconciliation.py b/tests/unit/test_airflow_reconciliation.py
index 80255c8..e46623d 100644
--- a/tests/unit/test_airflow_reconciliation.py
+++ b/tests/unit/test_airflow_reconciliation.py
@@ -116,6 +116,7 @@ def test_failed_report_blocks_package_before_bundle_writes(tmp_path: Path) -> No
     output = tmp_path / "bundle"
     pipeline = Pipeline(
         name="failed",
+        tags={"source": "airflow"},
         reconciliation_status="failed",
         not_translatable=[
             {
@@ -124,6 +125,11 @@ def test_failed_report_blocks_package_before_bundle_writes(tmp_path: Path) -> No
                 "message": "one source task was not captured",
             }
         ],
+        audit={
+            "source_file": "failed.py",
+            "audited_activity_count": 1,
+            "transformations": [],
+        },
     )
     report.write_text(json.dumps(ir_serde.pipeline_to_dict(pipeline)), encoding="utf-8")
 
@@ -356,6 +362,11 @@ def test_bundle_invariant_failure_is_preflighted_before_destination_writes(tmp_p
         name="parent",
         tags={"source": "airflow"},
         reconciliation_status="verified",
+        audit={
+            "source_file": "parent.py",
+            "audited_activity_count": 1,
+            "transformations": [],
+        },
         tasks=[
             PlaceholderActivity(
                 name="dangling",
diff --git a/tests/unit/test_mcp_source_routing.py b/tests/unit/test_mcp_source_routing.py
index 491f641..54877d1 100644
--- a/tests/unit/test_mcp_source_routing.py
+++ b/tests/unit/test_mcp_source_routing.py
@@ -109,6 +109,22 @@ def test_merge_agentic_threads_source(captured):
     assert argv[argv.index("--source") + 1] == "adf"
 
 
+def test_merge_agentic_rejects_airflow_without_invoking_adapter(captured):
+    result = server._cmd_merge_agentic(
+        {
+            "source": "airflow",
+            "report_path": "/tmp/report.json",
+            "agentic_results_dir": "/tmp/results",
+        }
+    )
+
+    assert result == {
+        "ok": False,
+        "error": "Airflow agentic merge is disabled until the fingerprint-bound resolution workflow is available.",
+    }
+    assert captured == []
+
+
 def test_inputs_threads_source(captured):
     server._cmd_inputs({"phase": "discover", "source": "airflow"})
     argv = _argv(captured, "inputs")
diff --git a/tests/unit/test_package_invariants.py b/tests/unit/test_package_invariants.py
index c1a0416..5420273 100644
--- a/tests/unit/test_package_invariants.py
+++ b/tests/unit/test_package_invariants.py
@@ -8,6 +8,7 @@
 
 import yaml
 
+from flowx.bundler.dab_writer import _report_reconciliation_failures
 from flowx.bundler.dab_writer import main as package_main
 
 
@@ -30,14 +31,37 @@ def _notebook_task(name: str, task_key: str) -> dict:
     }
 
 
+def _adf_pipeline(name: str, tasks: list[dict]) -> dict:
+    return {
+        "name": name,
+        "tags": {"source": "adf"},
+        "reconciliation_status": None,
+        "tasks": tasks,
+    }
+
+
+def _airflow_pipeline(name: str, tasks: list[dict], *, status: str = "verified") -> dict:
+    return {
+        "name": name,
+        "tags": {"source": "airflow"},
+        "reconciliation_status": status,
+        "audit": {
+            "source_file": f"{name}.py",
+            "audited_activity_count": len(tasks),
+            "transformations": [],
+        },
+        "tasks": tasks,
+    }
+
+
 def test_package_passes_invariants_for_clean_bundle():
-    report = {"name": "clean", "tasks": [_notebook_task("a", "a"), _notebook_task("b", "b")]}
+    report = _adf_pipeline("clean", [_notebook_task("a", "a"), _notebook_task("b", "b")])
     assert _run_package(report) == 0
 
 
 def test_package_fails_on_duplicate_task_key():
     # Two tasks sharing a task_key -> duplicate_task_key violation -> non-zero exit.
-    report = {"name": "bad", "tasks": [_notebook_task("a", "dup"), _notebook_task("b", "dup")]}
+    report = _adf_pipeline("bad", [_notebook_task("a", "dup"), _notebook_task("b", "dup")])
     assert _run_package(report) == 1
 
 
@@ -48,8 +72,8 @@ def test_package_loads_multi_pipeline_report():
 
     report = {
         "pipelines": [
-            {"name": "first", "tasks": [_notebook_task("x", "x")]},
-            {"name": "second", "tasks": [_notebook_task("y", "y")]},
+            _adf_pipeline("first", [_notebook_task("x", "x")]),
+            _adf_pipeline("second", [_notebook_task("y", "y")]),
         ]
     }
     with tempfile.TemporaryDirectory() as tmp:
@@ -66,10 +90,9 @@ def test_package_loads_multi_pipeline_report():
 def test_package_writes_airflow_dags_as_jobs_in_one_shared_bundle():
     report = {
         "pipelines": [
-            {
-                "name": "parent",
-                "tags": {"source": "airflow"},
-                "tasks": [
+            _airflow_pipeline(
+                "parent",
+                [
                     _notebook_task("extract", "extract"),
                     {
                         "name": "trigger_child",
@@ -78,12 +101,8 @@ def test_package_writes_airflow_dags_as_jobs_in_one_shared_bundle():
                         "job_name": "child",
                     },
                 ],
-            },
-            {
-                "name": "child",
-                "tags": {"source": "airflow"},
-                "tasks": [_notebook_task("extract", "extract")],
-            },
+            ),
+            _airflow_pipeline("child", [_notebook_task("extract", "extract")]),
         ]
     }
     with tempfile.TemporaryDirectory() as tmp:
@@ -109,10 +128,9 @@ def test_shared_bundle_cross_dag_ref_resolves_for_hyphenated_dag_id():
     # job by its normalized resource key, not a differently-sanitized name, or the ref dangles.
     report = {
         "pipelines": [
-            {
-                "name": "downstream",
-                "tags": {"source": "airflow"},
-                "tasks": [
+            _airflow_pipeline(
+                "downstream",
+                [
                     {
                         "name": "trig",
                         "task_key": "trig",
@@ -120,12 +138,8 @@ def test_shared_bundle_cross_dag_ref_resolves_for_hyphenated_dag_id():
                         "job_name": "upstream_dag",  # normalize_task_key("Upstream-DAG")
                     },
                 ],
-            },
-            {
-                "name": "Upstream-DAG",
-                "tags": {"source": "airflow"},
-                "tasks": [_notebook_task("a", "a")],
-            },
+            ),
+            _airflow_pipeline("Upstream-DAG", [_notebook_task("a", "a")]),
         ]
     }
     with tempfile.TemporaryDirectory() as tmp:
@@ -156,8 +170,8 @@ def test_shared_airflow_bundle_namespaces_pydabs_hooks_and_jobs():
     }
     report = {
         "pipelines": [
-            {"name": "first", "tags": {"source": "airflow"}, "tasks": [dbt_task]},
-            {"name": "second", "tags": {"source": "airflow"}, "tasks": [dbt_task]},
+            _airflow_pipeline("first", [dbt_task]),
+            _airflow_pipeline("second", [dbt_task]),
         ]
     }
     with tempfile.TemporaryDirectory() as tmp:
@@ -184,3 +198,123 @@ def test_shared_airflow_bundle_namespaces_pydabs_hooks_and_jobs():
         assert (out / "resources" / "second_dbt_dbt_job.py").exists()
         assert "${resources.jobs.first_dbt_dbt.id}" in (out / "resources" / "first.yml").read_text()
         assert "${resources.jobs.second_dbt_dbt.id}" in (out / "resources" / "second.yml").read_text()
+
+
+def _preflight_failures(tmp_path: Path, report: object) -> list[str]:
+    path = tmp_path / "report.json"
+    path.write_text(json.dumps(report), encoding="utf-8")
+    return _report_reconciliation_failures(path)
+
+
+def test_report_preflight_rejects_unknown_reconciliation_status(tmp_path: Path):
+    report = _airflow_pipeline("typo", [_notebook_task("a", "a")], status="verifed")
+
+    failures = _preflight_failures(tmp_path, report)
+
+    assert any("unknown reconciliation_status" in failure for failure in failures)
+
+
+def test_report_preflight_rejects_reviewed_resolution_status_until_validator_exists(tmp_path: Path):
+    report = _airflow_pipeline(
+        "premature_resolution",
+        [_notebook_task("a", "a")],
+        status="verified_with_reviewed_resolutions",
+    )
+
+    failures = _preflight_failures(tmp_path, report)
+
+    assert any("unknown reconciliation_status" in failure for failure in failures)
+
+
+def test_report_preflight_rejects_excluded_status_for_included_dag(tmp_path: Path):
+    report = _airflow_pipeline("false_exclusion", [_notebook_task("a", "a")], status="excluded")
+
+    failures = _preflight_failures(tmp_path, report)
+
+    assert any("requires migration_status 'excluded'" in failure for failure in failures)
+
+
+def test_report_preflight_accepts_explicitly_excluded_dag(tmp_path: Path):
+    report = _airflow_pipeline("excluded", [_notebook_task("a", "a")], status="excluded")
+    report["migration_status"] = "excluded"
+
+    assert _preflight_failures(tmp_path, report) == []
+
+
+def test_report_preflight_rejects_airflow_without_audit_metadata(tmp_path: Path):
+    report = _airflow_pipeline("missing_audit", [_notebook_task("a", "a")])
+    report.pop("audit")
+
+    failures = _preflight_failures(tmp_path, report)
+
+    assert any("source-audit metadata" in failure for failure in failures)
+
+
+def test_report_preflight_rejects_top_level_list(tmp_path: Path):
+    failures = _preflight_failures(tmp_path, [_adf_pipeline("p", [_notebook_task("a", "a")])])
+
+    assert any("top-level object" in failure for failure in failures)
+
+
+def test_package_rejects_malformed_report_before_writing_bundle(tmp_path: Path, capsys):
+    report_path = tmp_path / "malformed.json"
+    report_path.write_text("[]", encoding="utf-8")
+    output_dir = tmp_path / "bundle"
+
+    exit_code = package_main(["--report", str(report_path), "--output-dir", str(output_dir)])
+
+    assert exit_code == 1
+    assert "translation report preflight failed" in capsys.readouterr().err
+    assert not output_dir.exists()
+
+
+def test_report_preflight_rejects_unrecognized_dictionary(tmp_path: Path):
+    failures = _preflight_failures(tmp_path, {"name": "missing_tasks"})
+
+    assert any("recognized report shape" in failure for failure in failures)
+
+
+def test_report_preflight_rejects_malformed_pipelines_wrapper(tmp_path: Path):
+    failures = _preflight_failures(tmp_path, {"pipelines": "not-a-list"})
+
+    assert any("pipelines must be a list" in failure for failure in failures)
+
+
+def test_report_preflight_accepts_legacy_adf_translations(tmp_path: Path):
+    report = {
+        "translations": [
+            {
+                "pipeline": "legacy_adf",
+                "status": "translated",
+                "ir": _notebook_task("a", "a"),
+            }
+        ]
+    }
+
+    assert _preflight_failures(tmp_path, report) == []
+
+
+def test_report_preflight_rejects_airflow_claiming_legacy_adf_shape(tmp_path: Path):
+    report = {
+        "source": "airflow",
+        "translations": [
+            {
+                "pipeline": "not_airflow_contract",
+                "status": "translated",
+                "ir": _notebook_task("a", "a"),
+            }
+        ],
+    }
+
+    failures = _preflight_failures(tmp_path, report)
+
+    assert any("legacy ADF" in failure for failure in failures)
+
+
+def test_report_preflight_rejects_invalid_json(tmp_path: Path):
+    path = tmp_path / "report.json"
+    path.write_text("{not-json", encoding="utf-8")
+
+    failures = _report_reconciliation_failures(path)
+
+    assert any("invalid JSON" in failure for failure in failures)
diff --git a/tests/unit/test_source_router.py b/tests/unit/test_source_router.py
index 3a50832..9e778a9 100644
--- a/tests/unit/test_source_router.py
+++ b/tests/unit/test_source_router.py
@@ -87,6 +87,23 @@ def test_airflow_discover_then_convert_route():
         assert (out / ".work" / "translation_report.json").exists()
 
 
+def test_airflow_legacy_agentic_merge_is_rejected_by_adapter():
+    rc = _run_phase(
+        "convert",
+        [
+            "--source",
+            "airflow",
+            "--merge-agentic",
+            "--report",
+            "/tmp/report.json",
+            "--agentic-results",
+            "/tmp/results",
+        ],
+    )
+
+    assert rc == 2
+
+
 def test_source_path_alias_equals_form_routes():
     # The `--source-path=` equals form must normalise to --source-dir just like the space form,
     # or the phase module rejects it with a usage error.

From 637521eb306c1488c849d72da499ba056396c210 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Sat, 8 Aug 2026 15:02:41 -0700
Subject: [PATCH 55/77] Add fingerprint-bound Airflow gap resolution

---
 app/README.md                                 |   1 +
 skills/flowx-convert/SKILL.md                 |   7 +-
 .../flowx-convert/sources/airflow-coverage.md |   3 +-
 skills/flowx-convert/sources/airflow.md       |  13 +-
 skills/flowx-migrate/SKILL.md                 |   5 +
 skills/flowx-resolve-airflow-gaps/SKILL.md    | 102 ++
 .../references/contract-v1.md                 |  62 ++
 src/flowx/adapter/__main__.py                 |  79 +-
 src/flowx/agentic.py                          | 943 ++++++++++++++++++
 src/flowx/bundler/dab_writer.py               |  23 +-
 src/flowx/mcp/server.py                       |  58 +-
 src/flowx/sources/airflow/convert.py          |   2 +-
 tests/unit/test_airflow_agentic_resolution.py | 565 +++++++++++
 tests/unit/test_mcp_source_routing.py         |  45 +-
 tests/unit/test_package_invariants.py         |   4 +-
 15 files changed, 1891 insertions(+), 21 deletions(-)
 create mode 100644 skills/flowx-resolve-airflow-gaps/SKILL.md
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/contract-v1.md
 create mode 100644 src/flowx/agentic.py
 create mode 100644 tests/unit/test_airflow_agentic_resolution.py

diff --git a/app/README.md b/app/README.md
index beafbec..a0b3563 100644
--- a/app/README.md
+++ b/app/README.md
@@ -21,6 +21,7 @@ operation; `parameters` is its keyword-argument dict.
 | `discover` | `adapter discover` | Parse ADF JSON, classify activities |
 | `convert` | `adapter convert` | ADF activities → Databricks IR |
 | `merge_agentic` | `adapter convert --merge-agentic` | Merge ADF agent-produced results into the report |
+| `resolve_agentic` | `adapter resolve-agentic` | Prepare, stage, and apply reviewed Airflow leaf-gap resolutions |
 | `inspect` | `adapter inspect` | Surface pending translation options |
 | `apply_answers` | `adapter modify` | Apply answers → stamped IR |
 | `materialize_lookup` | `adapter materialize-lookup` | CSV → lookup-values JSON |
diff --git a/skills/flowx-convert/SKILL.md b/skills/flowx-convert/SKILL.md
index ab12fb8..a9d5546 100644
--- a/skills/flowx-convert/SKILL.md
+++ b/skills/flowx-convert/SKILL.md
@@ -21,7 +21,7 @@ phase 2 of the flowx migration workflow; it produces a transient translation rep
 Translation is **source-specific** (ADF activity translators vs. Airflow operator mapping), so this
 skill routes to the right source guide. The shared mechanics — how to run the phase, the report
 contract, and the `inspect`/`modify` machinery — live here. The legacy `merge_agentic` command is
-ADF-only; Airflow rejects it until the fingerprint-bound resolution workflow is available.
+ADF-only; Airflow uses `resolve-agentic prepare|stage|apply` instead.
 
 ## Step 1 — Identify the source (required)
 
@@ -36,7 +36,8 @@ There is no default source. Every phase invocation passes `--source ` expl
 ## Step 2 — Follow the source guide
 
 Read the matching `sources/.md` and follow it. ADF has a rich deterministic-first +
-agentic-gap flow with just-in-time configuration; Airflow is currently deterministic-only.
+agentic-gap flow with just-in-time configuration. Airflow converts deterministically first and may
+then use the separately reviewed, fingerprint-bound `flowx-resolve-airflow-gaps` workflow.
 
 ## How to run this phase — MCP tool or venv CLI
 
@@ -88,4 +89,4 @@ them heavily; Airflow currently needs only the base conversion:
 
 - `sources/adf.md` — ADF translation: deterministic engine, agentic gaps, just-in-time config,
   notify motifs, metadata-driven consolidation. See also `references/activity-mapping.md`.
-- `sources/airflow.md` — Airflow translation (operator → IR, deterministic-only today).
+- `sources/airflow.md` — Airflow deterministic-first translation plus reviewed leaf-gap resolution.
diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md
index 8eeb35e..8af3c08 100644
--- a/skills/flowx-convert/sources/airflow-coverage.md
+++ b/skills/flowx-convert/sources/airflow-coverage.md
@@ -40,7 +40,8 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't
 | Multiple DAGs | Every DAG, including multiple declarations and repeated static `@dag` factory invocations in one Python file, becomes a sibling job in one shared Airflow bundle so `TriggerDagRunOperator` resource references resolve. Narrow classic factories shaped as one DAG declaration followed by `return dag` are expanded with statically bindable arguments. |
 
 Any operator not listed becomes a `PlaceholderActivity` **and** a `gaps.json` entry carrying the
-operator's raw source for review. The legacy `merge_agentic` command is disabled for Airflow. The
+operator's raw source for review. The legacy `merge_agentic` command is disabled for Airflow;
+eligible one-task leaf gaps may use the fingerprint-bound `flowx-resolve-airflow-gaps` workflow. The
 safe fallback is a flagged, failing task rather than a silent omission. Callables that read Airflow task context
 (`**context` / `ti`) or XCom, and runtime-branching decorators, take the same route rather than
 emitting code that fails at runtime.
diff --git a/skills/flowx-convert/sources/airflow.md b/skills/flowx-convert/sources/airflow.md
index 4c5bbc6..c47a93d 100644
--- a/skills/flowx-convert/sources/airflow.md
+++ b/skills/flowx-convert/sources/airflow.md
@@ -3,11 +3,11 @@
 Source guide for `--source airflow`. Translate parsed Airflow DAGs into Databricks IR. See the
 parent `SKILL.md` for how to run the phase and the report contract.
 
-Airflow translation is currently **deterministic-only**. The static parse maps ~35 operator/sensor
-families directly to IR (Tier 1-3). Operators with no deterministic mapping become
+Airflow translation is **deterministic-first**. The static parse maps ~35 operator/sensor families
+directly to IR (Tier 1-3). Operators with no deterministic mapping become
 `PlaceholderActivity` tasks and are recorded in `gaps.json` with their raw source for review. The
-placeholder remains a deliberate runtime failure until it is resolved manually; a supported Airflow
-agentic replacement workflow is not available yet.
+placeholder remains a deliberate runtime failure until it is resolved manually or through the
+fingerprint-bound `flowx-resolve-airflow-gaps` workflow.
 
 **Before converting, check [`sources/airflow-coverage.md`](airflow-coverage.md)** — the verified
 support matrix (classic operators, TaskFlow, sensors, TaskGroups, dbt factory) and the constructs
@@ -43,8 +43,9 @@ PythonOperator callable or BashOperator command, carrying `generated_source`) or
 If convert wrote `/.work/gaps.json`, each entry describes an unmapped construct whose
 generated Job task points to a notebook that raises `NotImplementedError`. Review every gap before
 deployment. The shared `merge_agentic` command is disabled for Airflow and rejects
-`--source airflow`; keep the placeholder or exclude the DAG. This prevents an unsafe name-based
-replacement from bypassing source reconciliation.
+`--source airflow`; keep the placeholder, exclude the DAG, or invoke the
+`flowx-resolve-airflow-gaps` skill. That workflow binds one leaf resolution to the finding
+fingerprint and revalidates graph and policy invariants before package.
 
 ## Step 4 — Proceed to package
 
diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md
index 361cfab..a8b5593 100644
--- a/skills/flowx-migrate/SKILL.md
+++ b/skills/flowx-migrate/SKILL.md
@@ -111,6 +111,11 @@ flowx(command="package", parameters={"output_dir": ..., "catalog": ..., "schema"
 flowx(command="record_results", parameters={...}) / flowx(command="install_dashboard", parameters={...})
 ```
 
+For an Airflow report with eligible leaf placeholders, use the `flowx-resolve-airflow-gaps` skill
+between convert and package. It calls `resolve_agentic` with `action="prepare"`, stages one or more
+provider candidates, and applies only the gap fingerprints the user explicitly accepts. Package
+must then receive `/.work/translation_report.agentic.json` as `report_path`.
+
 The server's `output_dir` is ephemeral and not reachable from your workspace, so **have `migrate`/
 `package` write the DAB to the target via the SDK** — pass `"output_volume_path": "/Volumes/…"` or
 `"output_workspace_path": "/Workspace/…"` and the bundle is uploaded there (returned as
diff --git a/skills/flowx-resolve-airflow-gaps/SKILL.md b/skills/flowx-resolve-airflow-gaps/SKILL.md
new file mode 100644
index 0000000..cca896d
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/SKILL.md
@@ -0,0 +1,102 @@
+---
+name: flowx-resolve-airflow-gaps
+description: >
+  Resolve source-reconciled Airflow leaf gaps through the fingerprint-bound flowx contract. Use
+  after Airflow conversion emits PlaceholderActivity tasks and before packaging the reviewed report.
+---
+
+# Resolve Airflow Leaf Gaps
+
+Use this workflow only for Airflow reports whose deterministic conversion succeeded with gaps.
+Flowx owns source parsing, task identity, dependencies, task policy, IR, and packaging. This skill
+reasons about one prepared gap at a time using the migration knowledge from
+[`park-peter/airflow-to-dabs` v0.1.0](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.1.0).
+It must not parse the DAG independently or generate a second bundle.
+
+Read [`references/contract-v1.md`](references/contract-v1.md) before authoring a resolution.
+
+## 1. Prepare immutable gap envelopes
+
+```bash
+"$PY" -m flowx.adapter resolve-agentic prepare \
+  --source airflow \
+  --source-path  \
+  --report /.work/translation_report.json \
+  --output-dir 
+```
+
+Preparation reparses the source, proves that it reproduces the deterministic report, and writes an
+immutable baseline, source snapshot, manifest, and `GapEnvelope v1` objects under
+`/.work/agentic/`. If the source or report no longer agrees, rerun convert first.
+
+With MCP, call `flowx(command="resolve_agentic", parameters={"action": "prepare", "source":
+"airflow", "airflow_source_path": ..., "report_path": ..., "output_dir": ...})`.
+
+## 2. Produce one candidate per gap
+
+Read the prepared envelope rather than reopening or reparsing the DAG. Return one of:
+
+- `resolved`: exactly one self-contained Python notebook or SQL payload.
+- `needs_input`: a concrete question or prerequisite blocks a safe migration.
+- `deferred`: the gap is outside the leaf-only contract and remains a linked failing placeholder.
+
+`KubernetesPodOperator` commonly returns `needs_input` when the image, secrets, storage, networking,
+or compute assumptions cannot be preserved from the envelope alone. Do not present it as the default
+successful example.
+
+Every source argument must appear exactly once in `argument_disposition` as `consumed`,
+`preserved_by_flowx`, or `ignored`. Every disposition needs a rationale; an ignored argument must
+state the specific semantic loss. Never include task names, task keys, dependencies, retries,
+timeouts, clusters, schedules, or other graph/policy fields in the replacement.
+
+Generated code must be self-contained, contain no Airflow import statements, and contain no
+unresolved Airflow Jinja. Comments may mention Airflow for provenance.
+
+## 3. Stage candidates
+
+```bash
+"$PY" -m flowx.adapter resolve-agentic stage \
+  --source airflow \
+  --output-dir  \
+  --candidate  [--candidate  ...]
+```
+
+Stage validates fingerprints, source/report hashes, the pinned provider version, argument
+disposition, generated-file hashes, Python imports, templates, and the constrained replacement
+schema. Tampering after staging is a hard failure.
+
+MCP accepts candidate objects inline with `action="stage"` and `candidates=[...]`.
+
+## 4. Review and explicitly apply
+
+Show the user each candidate's code, prerequisites, warnings, semantic deltas, ignored arguments,
+provider version, and model provenance. Apply only the fingerprints the user accepts:
+
+```bash
+"$PY" -m flowx.adapter resolve-agentic apply \
+  --source airflow \
+  --output-dir  \
+  --accept-gap  [--accept-gap  ...]
+```
+
+`--accept-all` is only for replaying candidates already staged in a prior step; never combine it
+with live candidate generation. Apply always rebuilds from the immutable deterministic baseline,
+then proves task count, location, keys, dependencies, policy, and enclosing control flow are
+unchanged. It writes `.work/translation_report.agentic.json` and keeps accepted evidence under
+`metadata/agentic/` so package pruning does not destroy provenance.
+
+Use a reduced `--accept-gap` allowlist to reject selected candidates while retaining others. Use
+`--reset` to discard all accepted resolutions and start over from the deterministic baseline. A
+source edit after prepare is a hard failure: rerun convert and prepare instead of applying stale
+results.
+
+Package the reviewed report explicitly:
+
+```bash
+"$PY" -m flowx.adapter package \
+  --report /.work/translation_report.agentic.json \
+  --output-dir 
+```
+
+Package replays the kept baseline and accepted candidates before writing bundle files. Missing,
+modified, or inconsistent evidence fails preflight.
diff --git a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
new file mode 100644
index 0000000..a773d71
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
@@ -0,0 +1,62 @@
+# Airflow Agentic Gap Contract v1
+
+The provider receives a `GapEnvelope` produced by flowx. It does not receive authority to alter the
+captured graph.
+
+## Resolution shape
+
+```json
+{
+  "contract_version": "1",
+  "gap_id": "finding fingerprint from the envelope",
+  "status": "resolved",
+  "baseline_report_sha256": "copied from the envelope",
+  "source_sha256": "copied from the envelope",
+  "provider": {
+    "name": "airflow-to-dabs",
+    "version": "0.1.0",
+    "repository": "https://github.com/park-peter/airflow-to-dabs"
+  },
+  "model": {"name": "model identifier"},
+  "replacement": {"kind": "notebook", "file": "task.py", "base_parameters": {}},
+  "generated_files": [
+    {
+      "path": "task.py",
+      "language": "python",
+      "content": "# Databricks notebook source\nprint('resolved')\n",
+      "sha256": "SHA-256 of content bytes"
+    }
+  ],
+  "argument_disposition": [
+    {
+      "name": "task_id",
+      "disposition": "preserved_by_flowx",
+      "rationale": "Flowx preserves the collision-safe task identity."
+    }
+  ],
+  "prerequisites": [],
+  "warnings": [],
+  "semantic_deltas": []
+}
+```
+
+SQL uses `{"kind": "sql", "file": "task.sql", "parameters": {}}` and a single generated file
+whose language is `sql`.
+
+`needs_input` and `deferred` omit `replacement` and `generated_files` and add a non-empty `reason`.
+They are terminal reviewed outcomes: the linked `NotImplementedError` placeholder remains and no
+automatic retry occurs.
+
+## Hard boundaries
+
+- Only `notebook` and `sql` leaf replacements are allowed in v1.
+- The replacement cannot express `name`, `task_key`, `depends_on`, retries, timeouts, compute,
+  libraries, schedules, or control-flow fields.
+- Generated file paths are relative and cannot contain `..`.
+- Every generated file is inline and hash-bound; external workspace paths are not accepted.
+- Python payloads may mention Airflow in comments but may not contain `import airflow` or
+  `from airflow ...` statements.
+- Airflow Jinja is rejected. Databricks dynamic references such as `{{job.parameters.x}}`,
+  `{{tasks.upstream.values.x}}`, and `{{input}}` remain valid.
+- Every source argument in the envelope has exactly one disposition and a non-empty rationale.
+- The provider identity must match the pinned `airflow-to-dabs` v0.1.0 knowledge release.
diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py
index 83e57d9..25e826a 100644
--- a/src/flowx/adapter/__main__.py
+++ b/src/flowx/adapter/__main__.py
@@ -1,9 +1,9 @@
 """Unified CLI entry point that the flowx skills and MCP tools drive via subprocesses.
 
 Exposes stateless subcommands -- the ``discover``/``convert``/``package`` phase runners plus
-``inspect``, ``modify``, ``inputs``, ``materialize-lookup``, ``workspace-paths``, ``record-results``,
-and ``install-dashboard`` -- so each agent turn runs as an independent process holding no session
-state across user prompts.
+``inspect``, ``modify``, ``resolve-agentic``, ``inputs``, ``materialize-lookup``, ``workspace-paths``,
+``record-results``, and ``install-dashboard`` -- so each agent turn runs as an independent process
+holding no session state across user prompts.
 """
 
 from __future__ import annotations
@@ -83,6 +83,8 @@ def main(argv: list[str] | None = None) -> int:
         return _run_inputs(args)
     if args.command == "workspace-paths":
         return _run_workspace_paths(args)
+    if args.command == "resolve-agentic":
+        return _run_resolve_agentic(args)
     if args.command == "record-results":
         return _run_record_results(args)
     if args.command == "install-dashboard":
@@ -91,6 +93,46 @@ def main(argv: list[str] | None = None) -> int:
     return 2
 
 
+def _run_resolve_agentic(args: argparse.Namespace) -> int:
+    """Runs the fingerprint-bound agentic resolution workflow for Airflow leaf gaps."""
+    if args.source != "airflow":
+        print("resolve-agentic is not enabled for ADF; ADF uses the legacy merge path.", file=sys.stderr)
+        return 2
+    from flowx.agentic import (
+        AgenticContractError,
+        apply_airflow_resolutions,
+        prepare_airflow_resolutions,
+        stage_airflow_resolutions,
+    )
+
+    try:
+        if args.action == "prepare":
+            if args.source_path is None or args.report is None:
+                print("resolve-agentic prepare requires --source-path and --report.", file=sys.stderr)
+                return 2
+            payload = prepare_airflow_resolutions(
+                source_path=args.source_path,
+                report_path=args.report,
+                output_dir=args.output_dir,
+                dbt_mode=args.dbt_mode,
+            )
+        elif args.action == "stage":
+            payload = stage_airflow_resolutions(output_dir=args.output_dir, candidate_paths=args.candidate)
+        else:
+            payload = apply_airflow_resolutions(
+                output_dir=args.output_dir,
+                accepted_gap_ids=args.accept_gap,
+                accept_all=args.accept_all,
+                reset=args.reset,
+                source_path=args.source_path,
+            )
+    except (AgenticContractError, OSError, json.JSONDecodeError) as error:
+        print(f"Agentic resolution failed: {error}", file=sys.stderr)
+        return 1
+    _emit_json(payload, None)
+    return 0
+
+
 def _run_record_results(args: argparse.Namespace) -> int:
     """Implements ``record-results``: write per-pipeline coverage to a UC table.
 
@@ -370,6 +412,37 @@ def _build_parser() -> argparse.ArgumentParser:
         help="Destination path for the lookup-values JSON list.",
     )
 
+    resolve_agentic = subparsers.add_parser(
+        "resolve-agentic",
+        help="Prepare, validate, and apply fingerprint-bound Airflow leaf-gap resolutions.",
+    )
+    resolve_agentic.add_argument("action", choices=("prepare", "stage", "apply"))
+    resolve_agentic.add_argument("--source", required=True, help="Must be airflow; ADF uses merge_agentic.")
+    resolve_agentic.add_argument("--output-dir", type=Path, required=True, help="Shared migration output directory.")
+    resolve_agentic.add_argument("--source-path", type=Path, default=None, help="Airflow DAG file or directory.")
+    resolve_agentic.add_argument("--report", type=Path, default=None, help="Deterministic translation report.")
+    resolve_agentic.add_argument(
+        "--candidate",
+        type=Path,
+        action="append",
+        default=[],
+        help="Provider-authored AgenticResolution JSON to validate and stage. Repeatable.",
+    )
+    resolve_agentic.add_argument(
+        "--accept-gap",
+        action="append",
+        default=[],
+        help="Prepared gap fingerprint to accept. Repeatable; the full allowlist is replayed from baseline.",
+    )
+    resolve_agentic.add_argument("--accept-all", action="store_true", help="Accept all already-staged candidates.")
+    resolve_agentic.add_argument("--reset", action="store_true", help="Restore the immutable deterministic baseline.")
+    resolve_agentic.add_argument(
+        "--dbt-mode",
+        choices=("static", "pydabs"),
+        default="static",
+        help="Airflow dbt conversion mode used to reproduce the deterministic report during prepare.",
+    )
+
     record = subparsers.add_parser(
         "record-results",
         help="Write per-pipeline migration coverage for this run to a Unity Catalog table.",
diff --git a/src/flowx/agentic.py b/src/flowx/agentic.py
new file mode 100644
index 0000000..635b3ff
--- /dev/null
+++ b/src/flowx/agentic.py
@@ -0,0 +1,943 @@
+"""Fingerprint-bound agentic resolution for source-reconciled migration gaps.
+
+Flowx remains the owner of source parsing, task identity, graph structure, policy, IR, and
+packaging. A provider may reason about one captured leaf gap and return only a constrained payload;
+this module validates that payload and applies it to an immutable deterministic baseline.
+"""
+
+from __future__ import annotations
+
+import ast
+import copy
+import hashlib
+import json
+import re
+import shutil
+import tempfile
+import textwrap
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from flowx.ir_serde import pipeline_to_dict
+from flowx.sources.airflow.loader import discover_dags, load_pipelines
+
+CONTRACT_VERSION = "1"
+PROVIDER_NAME = "airflow-to-dabs"
+PROVIDER_VERSION = "0.1.0"
+PROVIDER_REPOSITORY = "https://github.com/park-peter/airflow-to-dabs"
+
+_ALLOWED_REPLACEMENT_KINDS = ("notebook", "sql")
+_RESOLUTION_STATUSES = {"resolved", "needs_input", "deferred"}
+_DISPOSITIONS = {"consumed", "preserved_by_flowx", "ignored"}
+_COMMON_TASK_FIELDS = (
+    "name",
+    "task_key",
+    "description",
+    "timeout_seconds",
+    "max_retries",
+    "min_retry_interval_millis",
+    "depends_on",
+    "cluster",
+    "existing_cluster_id",
+    "libraries",
+    "parameter_approximations",
+    "required_parameters",
+    "compute_mode",
+    "notifications",
+)
+_FLOWX_OWNED_ARGUMENTS = {
+    "task_id",
+    "retries",
+    "retry_delay",
+    "execution_timeout",
+    "trigger_rule",
+    "depends_on_past",
+    "wait_for_downstream",
+    "pool",
+    "pool_slots",
+    "priority_weight",
+    "queue",
+}
+_NESTED_TASK_FIELDS = ("inner_activities", "if_true_activities", "if_false_activities", "default_activities")
+_AIRFLOW_TEMPLATE = re.compile(r"{{\s*([^{}]+?)\s*}}|{%\s*([^{}]+?)\s*%}")
+_DAB_TEMPLATE_PREFIXES = ("job.", "tasks.", "input", "backfill.")
+
+
+class AgenticContractError(ValueError):
+    """Raised when an agentic workspace or resolution violates the contract."""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class GapEnvelope:
+    """Versioned context for one source-reconciled leaf placeholder."""
+
+    gap_id: str
+    pipeline_name: str
+    task_key: str
+    task_path: list[str | int]
+    operator: str
+    source_file: str
+    source_sha256: str
+    baseline_report_sha256: str
+    source_span: dict[str, int]
+    raw_definition: dict[str, Any]
+    arguments: list[dict[str, Any]]
+    upstream_task_keys: list[str]
+    downstream_task_keys: list[str]
+    dag_settings: dict[str, Any]
+    reason: dict[str, str]
+
+    def as_dict(self) -> dict[str, Any]:
+        """Returns the public GapEnvelope v1 representation."""
+        return {
+            "contract_version": CONTRACT_VERSION,
+            "gap_id": self.gap_id,
+            "source": "airflow",
+            "pipeline_name": self.pipeline_name,
+            "capture_identity": self.task_key,
+            "task_key": self.task_key,
+            "task_path": self.task_path,
+            "operator": self.operator,
+            "operator_fqn": self.operator,
+            "source_file": self.source_file,
+            "source_sha256": self.source_sha256,
+            "baseline_report_sha256": self.baseline_report_sha256,
+            "source_span": self.source_span,
+            "raw_definition": self.raw_definition,
+            "arguments": self.arguments,
+            "upstream_task_keys": self.upstream_task_keys,
+            "downstream_task_keys": self.downstream_task_keys,
+            "dag_settings": self.dag_settings,
+            "reason": self.reason,
+            "allowed_replacement_kinds": list(_ALLOWED_REPLACEMENT_KINDS),
+            "knowledge_provider": {
+                "name": PROVIDER_NAME,
+                "version": PROVIDER_VERSION,
+                "repository": PROVIDER_REPOSITORY,
+            },
+        }
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class StagedResolution:
+    """A schema-validated resolution bound to one GapEnvelope."""
+
+    gap: dict[str, Any]
+    candidate: dict[str, Any]
+    sha256: str
+
+
+def prepare_airflow_resolutions(
+    *,
+    source_path: Path,
+    report_path: Path,
+    output_dir: Path,
+    dbt_mode: str = "static",
+) -> dict[str, Any]:
+    """Snapshots source and an exactly reproducible deterministic report, then emits GapEnvelope v1."""
+    source_path = source_path.resolve()
+    report_path = report_path.resolve()
+    output_dir = output_dir.resolve()
+    if not source_path.exists():
+        raise AgenticContractError(f"Airflow source path does not exist: {source_path}")
+    try:
+        baseline_bytes = report_path.read_bytes()
+        baseline = json.loads(baseline_bytes)
+    except OSError as error:
+        raise AgenticContractError(f"Could not read deterministic report: {error}") from error
+    except json.JSONDecodeError as error:
+        raise AgenticContractError(f"Deterministic report contains invalid JSON: {error}") from error
+    _require_airflow_baseline(baseline)
+
+    source_files = _source_files(source_path)
+    if not source_files:
+        raise AgenticContractError(f"No Airflow DAG files found under {source_path}")
+    source_hashes = {relative: _sha256_file(path) for relative, path in source_files}
+    baseline_hash = _sha256_bytes(baseline_bytes)
+
+    work_dir = output_dir / ".work"
+    work_dir.mkdir(parents=True, exist_ok=True)
+    target = work_dir / "agentic"
+    with tempfile.TemporaryDirectory(prefix=".agentic-prepare-", dir=work_dir) as temporary:
+        staging = Path(temporary)
+        snapshot = staging / "source"
+        for relative, path in source_files:
+            destination = snapshot / relative
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(path, destination)
+        if {relative: _sha256_file(snapshot / relative) for relative in source_hashes} != source_hashes:
+            raise AgenticContractError("Airflow source changed while the agentic snapshot was being created")
+
+        snapshot_source = snapshot / source_files[0][0] if source_path.is_file() else snapshot
+        rebuilt = _rebuild_airflow_report(snapshot_source, baseline, dbt_mode=dbt_mode)
+        if rebuilt != baseline:
+            raise AgenticContractError(
+                "Airflow source no longer reproduces the deterministic report; rerun convert before prepare"
+            )
+
+        gaps = _build_gap_envelopes(baseline, baseline_hash=baseline_hash, source_hashes=source_hashes)
+        if not gaps:
+            raise AgenticContractError("The deterministic report contains no eligible Airflow leaf gaps")
+
+        (staging / "baseline.json").write_bytes(baseline_bytes)
+        gaps_bytes = _json_bytes(gaps)
+        (staging / "gaps.json").write_bytes(gaps_bytes)
+        (staging / "candidates").mkdir()
+        _write_json(staging / "candidate_index.json", {})
+        manifest = {
+            "contract_version": CONTRACT_VERSION,
+            "source": "airflow",
+            "provider": {
+                "name": PROVIDER_NAME,
+                "version": PROVIDER_VERSION,
+                "repository": PROVIDER_REPOSITORY,
+            },
+            "source_path": str(source_path),
+            "source_kind": "file" if source_path.is_file() else "directory",
+            "source_files": [
+                {"path": relative, "sha256": source_hashes[relative]} for relative in sorted(source_hashes)
+            ],
+            "dbt_mode": dbt_mode,
+            "baseline_report_sha256": baseline_hash,
+            "gaps_sha256": _sha256_bytes(gaps_bytes),
+        }
+        _write_json(staging / "manifest.json", manifest)
+        if target.exists():
+            shutil.rmtree(target)
+        shutil.move(str(staging), target)
+
+    return {
+        "status": "prepared",
+        "contract_version": CONTRACT_VERSION,
+        "provider_version": PROVIDER_VERSION,
+        "gap_count": len(gaps),
+        "workspace": str(target),
+    }
+
+
+def stage_airflow_resolutions(*, output_dir: Path, candidate_paths: list[Path]) -> dict[str, Any]:
+    """Validates provider candidates and records their immutable hashes in the agentic workspace."""
+    workspace = _workspace(output_dir)
+    manifest, gaps = _load_workspace(workspace)
+    if not candidate_paths:
+        raise AgenticContractError("At least one --candidate path is required")
+    gap_by_id = {gap["gap_id"]: gap for gap in gaps}
+    staged: dict[str, StagedResolution] = {}
+    for path in candidate_paths:
+        try:
+            candidate = json.loads(path.read_text(encoding="utf-8"))
+        except OSError as error:
+            raise AgenticContractError(f"Could not read candidate {path}: {error}") from error
+        except json.JSONDecodeError as error:
+            raise AgenticContractError(f"Candidate {path} contains invalid JSON: {error}") from error
+        resolution = _validate_candidate(candidate, gap_by_id=gap_by_id, manifest=manifest)
+        staged[resolution.gap["gap_id"]] = resolution
+
+    candidates_dir = workspace / "candidates"
+    candidates_dir.mkdir(exist_ok=True)
+    index = _load_candidate_index(workspace, valid_gap_ids=set(gap_by_id))
+    for gap_id, resolution in staged.items():
+        destination = candidates_dir / f"{gap_id}.json"
+        destination.write_bytes(_json_bytes(resolution.candidate))
+        index[gap_id] = {
+            "sha256": resolution.sha256,
+            "status": resolution.candidate["status"],
+        }
+    _write_json(workspace / "candidate_index.json", index)
+    return {"status": "staged", "staged": sorted(staged), "candidate_count": len(index)}
+
+
+def apply_airflow_resolutions(
+    *,
+    output_dir: Path,
+    accepted_gap_ids: list[str] | None = None,
+    accept_all: bool = False,
+    reset: bool = False,
+    source_path: Path | None = None,
+) -> dict[str, Any]:
+    """Rebuilds an agentic report from the immutable baseline and the declarative acceptance set."""
+    if sum(bool(option) for option in (accepted_gap_ids, accept_all, reset)) != 1:
+        raise AgenticContractError("Choose exactly one of --accept-gap, --accept-all, or --reset")
+    output_dir = output_dir.resolve()
+    workspace = _workspace(output_dir)
+    manifest, gaps = _load_workspace(workspace)
+    baseline_path = workspace / "baseline.json"
+    baseline_bytes = baseline_path.read_bytes()
+    if _sha256_bytes(baseline_bytes) != manifest["baseline_report_sha256"]:
+        raise AgenticContractError("The immutable deterministic baseline was modified after prepare")
+    baseline = json.loads(baseline_bytes)
+
+    _verify_snapshot(workspace, manifest)
+    live_source = (source_path or Path(manifest["source_path"])).resolve()
+    if _current_source_hashes(live_source) != _manifest_source_hashes(manifest):
+        raise AgenticContractError("source changed since prepare; re-run prepare")
+    snapshot_source = _snapshot_source(workspace, manifest)
+    if _rebuild_airflow_report(snapshot_source, baseline, dbt_mode=manifest["dbt_mode"]) != baseline:
+        raise AgenticContractError("The prepared source snapshot no longer reproduces the deterministic report")
+
+    gap_by_id = {gap["gap_id"]: gap for gap in gaps}
+    index = _load_candidate_index(workspace, valid_gap_ids=set(gap_by_id))
+    selected_ids = [] if reset else sorted(index) if accept_all else list(dict.fromkeys(accepted_gap_ids or []))
+    missing = sorted(set(selected_ids) - set(index))
+    if missing:
+        raise AgenticContractError(f"No staged candidate exists for gap(s): {', '.join(missing)}")
+
+    selected: list[StagedResolution] = []
+    for gap_id in selected_ids:
+        candidate_path = workspace / "candidates" / f"{gap_id}.json"
+        candidate_bytes = candidate_path.read_bytes()
+        if _sha256_bytes(candidate_bytes) != index[gap_id]["sha256"]:
+            raise AgenticContractError(f"staged candidate was modified after validation: {gap_id}")
+        candidate = json.loads(candidate_bytes)
+        selected.append(_validate_candidate(candidate, gap_by_id=gap_by_id, manifest=manifest))
+
+    applied = _apply_to_baseline(baseline, selected)
+    report_path = output_dir / ".work" / "translation_report.agentic.json"
+    _write_json_atomic(report_path, applied)
+
+    evidence = output_dir / "metadata" / "agentic"
+    evidence.mkdir(parents=True, exist_ok=True)
+    (evidence / "baseline.json").write_bytes(baseline_bytes)
+    (evidence / "gaps.json").write_bytes((workspace / "gaps.json").read_bytes())
+    (evidence / "manifest.json").write_bytes((workspace / "manifest.json").read_bytes())
+    accepted_payload = {
+        "contract_version": CONTRACT_VERSION,
+        "candidates": [resolution.candidate for resolution in selected],
+    }
+    _write_json(evidence / "accepted_resolutions.json", accepted_payload)
+    return {
+        "status": "reset" if reset else "applied",
+        "accepted_gap_ids": selected_ids,
+        "report_path": str(report_path),
+    }
+
+
+def validate_persisted_agentic_report(report: dict[str, Any], *, evidence_dir: Path) -> list[str]:
+    """Replays accepted candidates from kept evidence and compares the exact expected report."""
+    try:
+        baseline_bytes = (evidence_dir / "baseline.json").read_bytes()
+        baseline = json.loads(baseline_bytes)
+        gaps_bytes = (evidence_dir / "gaps.json").read_bytes()
+        gaps = json.loads(gaps_bytes)
+        manifest = _read_json_object(evidence_dir / "manifest.json")
+        accepted = _read_json_object(evidence_dir / "accepted_resolutions.json")
+    except (OSError, json.JSONDecodeError, AgenticContractError) as error:
+        return [f"agentic resolution evidence is missing or invalid: {error}"]
+    if _sha256_bytes(baseline_bytes) != manifest.get("baseline_report_sha256"):
+        return ["agentic resolution baseline hash does not match its manifest"]
+    if _sha256_bytes(gaps_bytes) != manifest.get("gaps_sha256"):
+        return ["agentic gap-envelope hash does not match its manifest"]
+    expected_provider = {
+        "name": PROVIDER_NAME,
+        "version": PROVIDER_VERSION,
+        "repository": PROVIDER_REPOSITORY,
+    }
+    if (
+        manifest.get("contract_version") != CONTRACT_VERSION
+        or manifest.get("source") != "airflow"
+        or manifest.get("provider") != expected_provider
+    ):
+        return ["agentic resolution manifest has an unsupported contract, source, or provider"]
+    gap_by_id = {
+        str(gap["gap_id"]): gap for gap in gaps if isinstance(gap, dict) and isinstance(gap.get("gap_id"), str)
+    }
+    resolutions: list[StagedResolution] = []
+    try:
+        candidates = accepted.get("candidates")
+        if not isinstance(candidates, list):
+            raise AgenticContractError("accepted_resolutions.json must contain a candidates list")
+        for candidate in candidates:
+            resolutions.append(_validate_candidate(candidate, gap_by_id=gap_by_id, manifest=manifest))
+        expected = _apply_to_baseline(baseline, resolutions)
+    except AgenticContractError as error:
+        return [f"agentic resolution evidence failed validation: {error}"]
+    if expected != report:
+        return ["agentic report does not match replay from its immutable baseline and accepted resolutions"]
+    return []
+
+
+def _require_airflow_baseline(payload: Any) -> None:
+    pipelines = _pipeline_list(payload)
+    for pipeline in pipelines:
+        if not isinstance(pipeline, dict) or (pipeline.get("tags") or {}).get("source") != "airflow":
+            raise AgenticContractError("resolve-agentic requires a canonical Airflow translation report")
+        if pipeline.get("reconciliation_status") == "failed":
+            raise AgenticContractError("Agentic resolution cannot repair a failed source-reconciliation report")
+
+
+def _rebuild_airflow_report(source_path: Path, baseline: dict[str, Any], *, dbt_mode: str) -> dict[str, Any]:
+    expected = _pipeline_list(baseline)
+    expected_names = [pipeline["name"] for pipeline in expected]
+    excluded = {pipeline["name"] for pipeline in expected if pipeline.get("migration_status") == "excluded"}
+    loaded = load_pipelines(source_path, dbt_mode=dbt_mode, exclude_dags=excluded)
+    by_name = {pipeline.name: pipeline for pipeline in loaded}
+    if any(name not in by_name for name in expected_names):
+        raise AgenticContractError("Source snapshot does not contain every DAG in the deterministic report")
+    rebuilt = [pipeline_to_dict(by_name[name]) for name in expected_names]
+    return {"pipelines": rebuilt} if "pipelines" in baseline else rebuilt[0]
+
+
+def _build_gap_envelopes(
+    baseline: dict[str, Any],
+    *,
+    baseline_hash: str,
+    source_hashes: dict[str, str],
+) -> list[dict[str, Any]]:
+    envelopes: list[dict[str, Any]] = []
+    for pipeline in _pipeline_list(baseline):
+        if pipeline.get("migration_status") == "excluded":
+            continue
+        source_file = (pipeline.get("audit") or {}).get("source_file", "")
+        source_hash = source_hashes.get(source_file)
+        if source_hash is None and len(source_hashes) == 1:
+            source_hash = next(iter(source_hashes.values()))
+        if source_hash is None:
+            raise AgenticContractError(f"No source snapshot hash matches pipeline {pipeline.get('name')!r}")
+        findings = {
+            (finding.get("details") or {}).get("task_key"): finding
+            for finding in pipeline.get("not_translatable") or []
+            if isinstance(finding, dict) and finding.get("code") == "operator_placeholder"
+        }
+        tasks = list(_walk_tasks(pipeline.get("tasks") or []))
+        downstream = _downstream_index([task for _, task in tasks])
+        for task_path, task in tasks:
+            if task.get("type") != "PlaceholderActivity" or str(task.get("task_key", "")).startswith("__flowx_"):
+                continue
+            finding = findings.get(task.get("task_key"))
+            if not finding or not finding.get("fingerprint"):
+                continue
+            raw_definition = dict(task.get("raw_definition") or {})
+            operator = str(raw_definition.get("operator") or task.get("original_type") or "UnknownOperator")
+            envelope = GapEnvelope(
+                gap_id=str(finding["fingerprint"]),
+                pipeline_name=str(pipeline["name"]),
+                task_key=str(task["task_key"]),
+                task_path=list(task_path),
+                operator=operator,
+                source_file=str(source_file),
+                source_sha256=source_hash,
+                baseline_report_sha256=baseline_hash,
+                source_span={key: int(finding.get(key, 0)) for key in ("line", "column", "end_line", "end_column")},
+                raw_definition=raw_definition,
+                arguments=_extract_arguments(raw_definition, operator=operator),
+                upstream_task_keys=[str(item.get("task_key")) for item in task.get("depends_on") or []],
+                downstream_task_keys=downstream.get(str(task["task_key"]), []),
+                dag_settings={
+                    "schedule": pipeline.get("schedule"),
+                    "parameters": pipeline.get("parameters"),
+                    "tags": pipeline.get("tags"),
+                },
+                reason={
+                    "code": str(finding.get("code", "operator_placeholder")),
+                    "message": str(finding.get("message", "")),
+                },
+            )
+            envelopes.append(envelope.as_dict())
+    return sorted(envelopes, key=lambda item: (item["pipeline_name"], item["task_path"]))
+
+
+def _extract_arguments(raw_definition: dict[str, Any], *, operator: str) -> list[dict[str, Any]]:
+    source = raw_definition.get("source")
+    arguments: list[dict[str, Any]] = []
+    if isinstance(source, str) and source.strip():
+        try:
+            module = ast.parse(textwrap.dedent(source))
+        except SyntaxError:
+            module = None
+        if module is not None:
+            calls = [node for node in ast.walk(module) if isinstance(node, ast.Call)]
+            matching = [call for call in calls if _call_name(call.func) == operator]
+            call = matching[0] if matching else calls[0] if calls else None
+            if call is not None:
+                for index, value in enumerate(call.args):
+                    name = f"$star{index}" if isinstance(value, ast.Starred) else f"$arg{index}"
+                    expression = ast.unparse(value.value if isinstance(value, ast.Starred) else value)
+                    arguments.append(_argument(name, expression))
+                kwargs_index = 0
+                for keyword in call.keywords:
+                    keyword_name = keyword.arg
+                    if keyword_name is None:
+                        keyword_name = f"$kwargs{kwargs_index}"
+                        kwargs_index += 1
+                    arguments.append(_argument(keyword_name, ast.unparse(keyword.value)))
+    mapping = raw_definition.get("mapping")
+    if isinstance(mapping, str) and mapping:
+        arguments.append(_argument("$mapping", mapping))
+    return arguments
+
+
+def _argument(name: str, expression: str) -> dict[str, Any]:
+    return {
+        "name": name,
+        "source_expression": expression,
+        "preserved_by_flowx": name in _FLOWX_OWNED_ARGUMENTS,
+    }
+
+
+def _validate_candidate(
+    candidate: Any,
+    *,
+    gap_by_id: dict[str, dict[str, Any]],
+    manifest: dict[str, Any],
+) -> StagedResolution:
+    if not isinstance(candidate, dict):
+        raise AgenticContractError("Candidate must be a JSON object")
+    allowed_top = {
+        "contract_version",
+        "gap_id",
+        "status",
+        "baseline_report_sha256",
+        "source_sha256",
+        "provider",
+        "model",
+        "argument_disposition",
+        "prerequisites",
+        "warnings",
+        "semantic_deltas",
+        "replacement",
+        "generated_files",
+        "reason",
+    }
+    extra = sorted(set(candidate) - allowed_top)
+    if extra:
+        raise AgenticContractError(f"Candidate contains unsupported fields: {', '.join(extra)}")
+    if candidate.get("contract_version") != CONTRACT_VERSION:
+        raise AgenticContractError(f"Unsupported agentic contract_version: {candidate.get('contract_version')!r}")
+    gap_id = candidate.get("gap_id")
+    if not isinstance(gap_id, str):
+        raise AgenticContractError("Candidate gap_id must be a string")
+    gap = gap_by_id.get(gap_id)
+    if gap is None:
+        raise AgenticContractError(f"Candidate gap_id does not match a prepared gap: {gap_id!r}")
+    if candidate.get("baseline_report_sha256") != manifest.get("baseline_report_sha256"):
+        raise AgenticContractError("Candidate baseline_report_sha256 does not match the prepared baseline")
+    if candidate.get("source_sha256") != gap.get("source_sha256"):
+        raise AgenticContractError("Candidate source_sha256 does not match its GapEnvelope")
+    provider = candidate.get("provider")
+    expected_provider = {
+        "name": PROVIDER_NAME,
+        "version": PROVIDER_VERSION,
+        "repository": PROVIDER_REPOSITORY,
+    }
+    if provider != expected_provider:
+        raise AgenticContractError(f"Candidate provider must match pinned {PROVIDER_NAME} v{PROVIDER_VERSION}")
+    model = candidate.get("model")
+    if not isinstance(model, dict) or not isinstance(model.get("name"), str) or not model["name"].strip():
+        raise AgenticContractError("Candidate model provenance requires a non-empty model.name")
+    status = candidate.get("status")
+    if status not in _RESOLUTION_STATUSES:
+        raise AgenticContractError(f"Candidate status must be one of: {', '.join(sorted(_RESOLUTION_STATUSES))}")
+    for field in ("prerequisites", "warnings", "semantic_deltas"):
+        if not isinstance(candidate.get(field), list) or not all(isinstance(item, str) for item in candidate[field]):
+            raise AgenticContractError(f"Candidate {field} must be a list of strings")
+    _validate_argument_disposition(candidate.get("argument_disposition"), gap)
+    if status == "resolved":
+        _validate_replacement(candidate, gap)
+    else:
+        if not isinstance(candidate.get("reason"), str) or not candidate["reason"].strip():
+            raise AgenticContractError(f"{status} candidate requires a non-empty reason")
+        if "replacement" in candidate or "generated_files" in candidate:
+            raise AgenticContractError(f"{status} candidate must not contain a replacement or generated files")
+    normalized = json.loads(_json_bytes(candidate))
+    return StagedResolution(gap=gap, candidate=normalized, sha256=_sha256_bytes(_json_bytes(normalized)))
+
+
+def _validate_argument_disposition(value: Any, gap: dict[str, Any]) -> None:
+    if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
+        raise AgenticContractError("argument_disposition must be a list of objects")
+    expected = {argument["name"]: argument for argument in gap.get("arguments") or []}
+    actual_names = [item.get("name") for item in value]
+    if len(actual_names) != len(set(actual_names)) or set(actual_names) != set(expected):
+        raise AgenticContractError("argument_disposition must cover every source argument exactly once")
+    for item in value:
+        disposition = item.get("disposition")
+        if disposition not in _DISPOSITIONS:
+            raise AgenticContractError(f"Unknown argument disposition for {item.get('name')!r}: {disposition!r}")
+        rationale = item.get("rationale")
+        if not isinstance(rationale, str) or not rationale.strip():
+            qualifier = "ignored argument" if disposition == "ignored" else "argument disposition"
+            raise AgenticContractError(f"{qualifier} requires a rationale: {item.get('name')}")
+        if expected[item["name"]]["preserved_by_flowx"] and disposition != "preserved_by_flowx":
+            raise AgenticContractError(f"Flowx-owned argument must be preserved_by_flowx: {item['name']}")
+        if not expected[item["name"]]["preserved_by_flowx"] and disposition == "preserved_by_flowx":
+            raise AgenticContractError(f"Provider argument is not preserved by Flowx: {item['name']}")
+
+
+def _validate_replacement(candidate: dict[str, Any], gap: dict[str, Any]) -> None:
+    replacement = candidate.get("replacement")
+    if not isinstance(replacement, dict):
+        raise AgenticContractError("Resolved candidate requires a replacement object")
+    kind = replacement.get("kind")
+    if kind not in gap.get("allowed_replacement_kinds", []):
+        raise AgenticContractError(f"Replacement kind is not allowed for this gap: {kind!r}")
+    allowed = {"kind", "file", "base_parameters"} if kind == "notebook" else {"kind", "file", "parameters"}
+    extra = sorted(set(replacement) - allowed)
+    if extra:
+        raise AgenticContractError(f"replacement contains unsupported fields: {', '.join(extra)}")
+    file_name = replacement.get("file")
+    if not isinstance(file_name, str) or not _safe_relative_path(file_name):
+        raise AgenticContractError("Replacement file must be a safe relative path")
+    parameters_field = "base_parameters" if kind == "notebook" else "parameters"
+    parameters = replacement.get(parameters_field, {})
+    if not isinstance(parameters, dict) or not all(
+        isinstance(key, str) and isinstance(val, str) for key, val in parameters.items()
+    ):
+        raise AgenticContractError(f"Replacement {parameters_field} must be a string-to-string object")
+    files = candidate.get("generated_files")
+    if not isinstance(files, list) or len(files) != 1 or not isinstance(files[0], dict):
+        raise AgenticContractError("Resolved v1 candidate requires exactly one inline generated file")
+    generated = files[0]
+    allowed_file_fields = {"path", "language", "content", "sha256"}
+    if set(generated) - allowed_file_fields:
+        raise AgenticContractError("Generated file contains unsupported fields")
+    if generated.get("path") != file_name:
+        raise AgenticContractError("Replacement file does not match generated_files.path")
+    expected_language = "python" if kind == "notebook" else "sql"
+    if generated.get("language") != expected_language:
+        raise AgenticContractError(f"Generated file language must be {expected_language!r}")
+    content = generated.get("content")
+    if not isinstance(content, str) or not content.strip():
+        raise AgenticContractError("Generated file content must be non-empty")
+    if generated.get("sha256") != _sha256_bytes(content.encode("utf-8")):
+        raise AgenticContractError("Generated file sha256 does not match its content")
+    _reject_unresolved_templates({"replacement": replacement, "generated_files": files})
+    if kind == "notebook":
+        try:
+            module = ast.parse(content)
+        except SyntaxError as error:
+            raise AgenticContractError(f"Generated notebook is not valid Python: {error}") from error
+        for node in ast.walk(module):
+            if isinstance(node, ast.Import) and any(
+                alias.name == "airflow" or alias.name.startswith("airflow.") for alias in node.names
+            ):
+                raise AgenticContractError("Generated notebook must not import Airflow")
+            if isinstance(node, ast.ImportFrom) and (
+                node.module == "airflow" or str(node.module).startswith("airflow.")
+            ):
+                raise AgenticContractError("Generated notebook must not import Airflow")
+
+
+def _apply_to_baseline(baseline: dict[str, Any], resolutions: list[StagedResolution]) -> dict[str, Any]:
+    applied = copy.deepcopy(baseline)
+    baseline_pipelines = {pipeline["name"]: pipeline for pipeline in _pipeline_list(baseline)}
+    applied_pipelines = {pipeline["name"]: pipeline for pipeline in _pipeline_list(applied)}
+    by_pipeline: dict[str, list[StagedResolution]] = {}
+    for resolution in resolutions:
+        by_pipeline.setdefault(resolution.gap["pipeline_name"], []).append(resolution)
+
+    for pipeline_name, selected in by_pipeline.items():
+        original_pipeline = baseline_pipelines[pipeline_name]
+        pipeline = applied_pipelines[pipeline_name]
+        resolved_count = 0
+        accepted_proof: list[dict[str, Any]] = []
+        resolved_paths: set[tuple[str | int, ...]] = set()
+        for resolution in selected:
+            path = tuple(resolution.gap["task_path"])
+            task = _get_path(pipeline, path)
+            if not isinstance(task, dict) or task.get("type") != "PlaceholderActivity":
+                raise AgenticContractError(f"Gap no longer points to a placeholder: {resolution.gap['gap_id']}")
+            status = resolution.candidate["status"]
+            if status == "resolved":
+                _set_path(pipeline, path, _build_replacement(task, resolution.candidate))
+                resolved_count += 1
+                resolved_paths.add(path)
+            _annotate_finding(pipeline, resolution)
+            accepted_proof.append(
+                {
+                    "gap_id": resolution.gap["gap_id"],
+                    "task_key": resolution.gap["task_key"],
+                    "status": status,
+                    "candidate_sha256": resolution.sha256,
+                }
+            )
+        _assert_task_invariants(original_pipeline, pipeline, resolved_paths=resolved_paths)
+        baseline_graph = _graph_hash(original_pipeline)
+        merged_graph = _graph_hash(pipeline)
+        if baseline_graph != merged_graph:
+            raise AgenticContractError(
+                f"Agentic resolution changed graph or task policy for pipeline {pipeline_name!r}"
+            )
+        pipeline.setdefault("audit", {})["agentic_resolution"] = {
+            "contract_version": CONTRACT_VERSION,
+            "provider_version": PROVIDER_VERSION,
+            "validation_status": "verified",
+            "baseline_graph_sha256": baseline_graph,
+            "merged_graph_sha256": merged_graph,
+            "accepted": sorted(accepted_proof, key=lambda item: item["gap_id"]),
+            "resolved_count": resolved_count,
+        }
+        if resolved_count:
+            pipeline["reconciliation_status"] = "verified_with_reviewed_resolutions"
+    return applied
+
+
+def _build_replacement(placeholder: dict[str, Any], candidate: dict[str, Any]) -> dict[str, Any]:
+    replacement = candidate["replacement"]
+    generated = candidate["generated_files"][0]
+    task = {field: copy.deepcopy(placeholder[field]) for field in _COMMON_TASK_FIELDS if field in placeholder}
+    if replacement["kind"] == "notebook":
+        task.update(
+            {
+                "type": "NotebookActivity",
+                "notebook_path": f"notebooks/{placeholder['task_key']}.py",
+                "generated_source": generated["content"],
+            }
+        )
+        if replacement.get("base_parameters"):
+            task["base_parameters"] = dict(replacement["base_parameters"])
+    else:
+        task.update({"type": "SqlActivity", "sql": generated["content"], "warehouse_ref": "${var.warehouse_id}"})
+        if replacement.get("parameters"):
+            task["parameters"] = dict(replacement["parameters"])
+    return task
+
+
+def _annotate_finding(pipeline: dict[str, Any], resolution: StagedResolution) -> None:
+    for finding in pipeline.get("not_translatable") or []:
+        if isinstance(finding, dict) and finding.get("fingerprint") == resolution.gap["gap_id"]:
+            finding["resolution"] = {
+                "status": resolution.candidate["status"],
+                "provider": resolution.candidate["provider"],
+                "model": resolution.candidate["model"],
+                "argument_disposition": resolution.candidate["argument_disposition"],
+                "prerequisites": resolution.candidate["prerequisites"],
+                "warnings": resolution.candidate["warnings"],
+                "semantic_deltas": resolution.candidate["semantic_deltas"],
+                "candidate_sha256": resolution.sha256,
+            }
+            if resolution.candidate["status"] == "resolved":
+                finding["severity"] = "resolved"
+            elif resolution.candidate.get("reason"):
+                finding["resolution"]["reason"] = resolution.candidate["reason"]
+            return
+    raise AgenticContractError(f"Gap finding is missing from pipeline report: {resolution.gap['gap_id']}")
+
+
+def _assert_task_invariants(
+    baseline_pipeline: dict[str, Any],
+    applied_pipeline: dict[str, Any],
+    *,
+    resolved_paths: set[tuple[str | int, ...]],
+) -> None:
+    baseline_tasks = {path: task for path, task in _walk_tasks(baseline_pipeline.get("tasks") or [])}
+    applied_tasks = {path: task for path, task in _walk_tasks(applied_pipeline.get("tasks") or [])}
+    if set(baseline_tasks) != set(applied_tasks):
+        raise AgenticContractError("Agentic resolution changed task count or enclosing control-flow structure")
+    for path, baseline_task in baseline_tasks.items():
+        applied_task = applied_tasks[path]
+        if path not in resolved_paths and _task_shell(baseline_task) != _task_shell(applied_task):
+            raise AgenticContractError(f"Agentic resolution changed an unaccepted task at path {list(path)}")
+        if path in resolved_paths and _task_policy(baseline_task) != _task_policy(applied_task):
+            raise AgenticContractError(
+                f"Agentic resolution changed task identity, dependencies, or policy at {list(path)}"
+            )
+
+
+def _graph_hash(pipeline: dict[str, Any]) -> str:
+    projection = [{"path": list(path), **_task_policy(task)} for path, task in _walk_tasks(pipeline.get("tasks") or [])]
+    return _sha256_bytes(json.dumps(projection, sort_keys=True, separators=(",", ":")).encode("utf-8"))
+
+
+def _task_policy(task: dict[str, Any]) -> dict[str, Any]:
+    return {field: copy.deepcopy(task.get(field)) for field in _COMMON_TASK_FIELDS}
+
+
+def _task_shell(task: dict[str, Any]) -> dict[str, Any]:
+    """Returns one task without descendant lists so a nested leaf may change independently."""
+    shell = {key: copy.deepcopy(value) for key, value in task.items() if key not in _NESTED_TASK_FIELDS}
+    cases = shell.get("cases")
+    if isinstance(cases, list):
+        shell["cases"] = [
+            {key: value for key, value in case.items() if key != "activities"} if isinstance(case, dict) else case
+            for case in cases
+        ]
+    return shell
+
+
+def _walk_tasks(
+    tasks: list[Any], path: tuple[str | int, ...] = ("tasks",)
+) -> list[tuple[tuple[str | int, ...], dict[str, Any]]]:
+    walked: list[tuple[tuple[str | int, ...], dict[str, Any]]] = []
+    for index, task in enumerate(tasks):
+        if not isinstance(task, dict):
+            continue
+        task_path = (*path, index)
+        walked.append((task_path, task))
+        for field in _NESTED_TASK_FIELDS:
+            child = task.get(field)
+            if isinstance(child, list):
+                walked.extend(_walk_tasks(child, (*task_path, field)))
+        cases = task.get("cases")
+        if isinstance(cases, list):
+            for case_index, case in enumerate(cases):
+                if isinstance(case, dict) and isinstance(case.get("activities"), list):
+                    walked.extend(_walk_tasks(case["activities"], (*task_path, "cases", case_index, "activities")))
+    return walked
+
+
+def _get_path(root: dict[str, Any], path: tuple[str | int, ...]) -> Any:
+    current: Any = root
+    for part in path:
+        current = current[part]
+    return current
+
+
+def _set_path(root: dict[str, Any], path: tuple[str | int, ...], value: Any) -> None:
+    parent = _get_path(root, path[:-1])
+    parent[path[-1]] = value
+
+
+def _downstream_index(tasks: list[dict[str, Any]]) -> dict[str, list[str]]:
+    downstream: dict[str, list[str]] = {}
+    for task in tasks:
+        for dependency in task.get("depends_on") or []:
+            if isinstance(dependency, dict) and dependency.get("task_key"):
+                downstream.setdefault(str(dependency["task_key"]), []).append(str(task.get("task_key")))
+    return {key: sorted(value) for key, value in downstream.items()}
+
+
+def _reject_unresolved_templates(value: Any) -> None:
+    if isinstance(value, str):
+        for match in _AIRFLOW_TEMPLATE.finditer(value):
+            expression = (match.group(1) or match.group(2) or "").strip()
+            if not expression.startswith(_DAB_TEMPLATE_PREFIXES):
+                raise AgenticContractError(f"Generated payload contains unresolved Airflow Jinja: {match.group(0)}")
+    elif isinstance(value, dict):
+        for item in value.values():
+            _reject_unresolved_templates(item)
+    elif isinstance(value, list):
+        for item in value:
+            _reject_unresolved_templates(item)
+
+
+def _pipeline_list(payload: Any) -> list[dict[str, Any]]:
+    if not isinstance(payload, dict):
+        raise AgenticContractError("Translation report must be a JSON object")
+    pipelines = payload.get("pipelines") if "pipelines" in payload else [payload]
+    if not isinstance(pipelines, list) or not pipelines or not all(isinstance(item, dict) for item in pipelines):
+        raise AgenticContractError("Translation report does not contain canonical pipelines")
+    return pipelines
+
+
+def _source_files(source_path: Path) -> list[tuple[str, Path]]:
+    paths = discover_dags(source_path)
+    root = source_path.parent if source_path.is_file() else source_path
+    return sorted((path.resolve().relative_to(root.resolve()).as_posix(), path.resolve()) for path in paths)
+
+
+def _current_source_hashes(source_path: Path) -> dict[str, str]:
+    if not source_path.exists():
+        return {}
+    return {relative: _sha256_file(path) for relative, path in _source_files(source_path)}
+
+
+def _manifest_source_hashes(manifest: dict[str, Any]) -> dict[str, str]:
+    return {str(item["path"]): str(item["sha256"]) for item in manifest.get("source_files") or []}
+
+
+def _snapshot_source(workspace: Path, manifest: dict[str, Any]) -> Path:
+    snapshot = workspace / "source"
+    if manifest.get("source_kind") == "file":
+        files = manifest.get("source_files") or []
+        if len(files) != 1:
+            raise AgenticContractError("Prepared single-file source manifest is invalid")
+        return snapshot / files[0]["path"]
+    return snapshot
+
+
+def _verify_snapshot(workspace: Path, manifest: dict[str, Any]) -> None:
+    snapshot = workspace / "source"
+    actual = {
+        str(item["path"]): _sha256_file(snapshot / str(item["path"])) for item in manifest.get("source_files") or []
+    }
+    if actual != _manifest_source_hashes(manifest):
+        raise AgenticContractError("The prepared Airflow source snapshot was modified after prepare")
+
+
+def _load_workspace(workspace: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    if not workspace.is_dir():
+        raise AgenticContractError(f"Agentic workspace not found: {workspace}; run prepare first")
+    manifest = _read_json_object(workspace / "manifest.json")
+    if manifest.get("contract_version") != CONTRACT_VERSION or manifest.get("source") != "airflow":
+        raise AgenticContractError("Agentic workspace has an unsupported contract or source")
+    expected_provider = {
+        "name": PROVIDER_NAME,
+        "version": PROVIDER_VERSION,
+        "repository": PROVIDER_REPOSITORY,
+    }
+    if manifest.get("provider") != expected_provider:
+        raise AgenticContractError(f"Agentic workspace provider must be pinned to {PROVIDER_NAME} v{PROVIDER_VERSION}")
+    gaps_bytes = (workspace / "gaps.json").read_bytes()
+    if _sha256_bytes(gaps_bytes) != manifest.get("gaps_sha256"):
+        raise AgenticContractError("Prepared GapEnvelope file was modified after prepare")
+    gaps = json.loads(gaps_bytes)
+    if not isinstance(gaps, list):
+        raise AgenticContractError("Prepared gaps.json must be a list")
+    return manifest, gaps
+
+
+def _load_candidate_index(workspace: Path, *, valid_gap_ids: set[str]) -> dict[str, dict[str, str]]:
+    index = _read_json_object(workspace / "candidate_index.json")
+    validated: dict[str, dict[str, str]] = {}
+    for gap_id, entry in index.items():
+        if gap_id not in valid_gap_ids:
+            raise AgenticContractError(f"Candidate index contains an unknown gap_id: {gap_id!r}")
+        if not isinstance(entry, dict) or set(entry) != {"sha256", "status"}:
+            raise AgenticContractError(f"Candidate index entry is invalid for gap: {gap_id}")
+        digest = entry.get("sha256")
+        if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None:
+            raise AgenticContractError(f"Candidate index sha256 is invalid for gap: {gap_id}")
+        status = entry.get("status")
+        if not isinstance(status, str) or status not in _RESOLUTION_STATUSES:
+            raise AgenticContractError(f"Candidate index status is invalid for gap: {gap_id}")
+        validated[gap_id] = {"sha256": digest, "status": status}
+    return validated
+
+
+def _workspace(output_dir: Path) -> Path:
+    return output_dir.resolve() / ".work" / "agentic"
+
+
+def _safe_relative_path(value: str) -> bool:
+    path = Path(value)
+    return bool(value) and not path.is_absolute() and ".." not in path.parts and value == path.as_posix()
+
+
+def _call_name(node: ast.expr) -> str:
+    if isinstance(node, ast.Name):
+        return node.id
+    if isinstance(node, ast.Attribute):
+        return node.attr
+    return ""
+
+
+def _json_bytes(value: Any) -> bytes:
+    return (json.dumps(value, indent=2, sort_keys=True, default=str) + "\n").encode("utf-8")
+
+
+def _write_json(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(_json_bytes(value))
+
+
+def _write_json_atomic(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(f".{path.name}.tmp")
+    temporary.write_bytes(_json_bytes(value))
+    temporary.replace(path)
+
+
+def _read_json_object(path: Path) -> dict[str, Any]:
+    value = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(value, dict):
+        raise AgenticContractError(f"Expected a JSON object in {path}")
+    return value
+
+
+def _sha256_file(path: Path) -> str:
+    return _sha256_bytes(path.read_bytes())
+
+
+def _sha256_bytes(value: bytes) -> str:
+    return hashlib.sha256(value).hexdigest()
diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py
index 81c0c5a..cbab4d2 100644
--- a/src/flowx/bundler/dab_writer.py
+++ b/src/flowx/bundler/dab_writer.py
@@ -1759,8 +1759,29 @@ def _report_reconciliation_failures(report_path: Path) -> list[str]:
                 return [f"legacy ADF translation at index {index} is missing pipeline or IR data"]
         return []
 
+    if any(
+        isinstance(pipeline, dict)
+        and (
+            pipeline.get("reconciliation_status") == "verified_with_reviewed_resolutions"
+            or (
+                isinstance(pipeline.get("audit"), dict)
+                and isinstance(pipeline["audit"].get("agentic_resolution"), dict)
+            )
+        )
+        for pipeline in pipelines
+    ):
+        from flowx.agentic import validate_persisted_agentic_report
+
+        output_dir = report_path.parent.parent if report_path.parent.name == ".work" else report_path.parent
+        agentic_failures = validate_persisted_agentic_report(
+            report,
+            evidence_dir=output_dir / "metadata" / "agentic",
+        )
+        if agentic_failures:
+            return agentic_failures
+
     failures: list[str] = []
-    airflow_statuses = {"verified", "verified_with_gaps", "failed"}
+    airflow_statuses = {"verified", "verified_with_gaps", "verified_with_reviewed_resolutions", "failed"}
     required_airflow_audit_fields = {"source_file", "audited_activity_count", "transformations"}
     for index, pipeline in enumerate(pipelines):
         label = f"pipeline[{index}]"
diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py
index 9ef808a..79e113f 100644
--- a/src/flowx/mcp/server.py
+++ b/src/flowx/mcp/server.py
@@ -6,7 +6,9 @@
 
 from __future__ import annotations
 
+import json
 import os
+import tempfile
 from collections.abc import Callable
 from pathlib import Path
 from typing import Any
@@ -53,6 +55,8 @@ def _transport_security() -> TransportSecuritySettings:
   flowx("inspect", {"report_path": "/.work/translation_report.json"})
   flowx("apply_answers", {"report_path": "...", "answers": ["id=value"], "output_dir": "..."})
   flowx("package", {"output_dir": "...", "catalog": "main", "schema": "default"})
+For a reviewed Airflow leaf gap, call `resolve_agentic` with action `prepare`, then `stage` with
+provider-authored candidates, then `apply` with an explicit `accept_gap` allowlist.
 Or run it all at once:
   flowx("migrate", {"source": "airflow", "airflow_source_path": "...", "output_dir": "...",
                     "catalog": "...", "schema": "..."})
@@ -218,7 +222,7 @@ def _cmd_merge_agentic(p: dict[str, Any]) -> dict[str, Any]:
     if source_name == "airflow":
         return {
             "ok": False,
-            "error": "Airflow agentic merge is disabled until the fingerprint-bound resolution workflow is available.",
+            "error": "Airflow agentic merge is disabled; use the fingerprint-bound resolve_agentic workflow.",
         }
     args = [
         "convert",
@@ -236,6 +240,51 @@ def _cmd_merge_agentic(p: dict[str, Any]) -> dict[str, Any]:
     return {"ok": result.ok, "process": result.as_dict()}
 
 
+def _cmd_resolve_agentic(p: dict[str, Any]) -> dict[str, Any]:
+    source_name = _source_name(p)
+    if source_name != "airflow":
+        return {"ok": False, "error": "resolve_agentic is not enabled for ADF; ADF uses the legacy merge path."}
+    action = p["action"]
+    if action not in {"prepare", "stage", "apply"}:
+        return {"ok": False, "error": "resolve_agentic action must be prepare, stage, or apply."}
+    output_dir = Path(p.get("output_dir", "./flowx_output"))
+    args: list[Any] = ["resolve-agentic", action, "--source", "airflow", "--output-dir", output_dir]
+    if p.get("airflow_source_path"):
+        args += ["--source-path", p["airflow_source_path"]]
+    if p.get("report_path"):
+        args += ["--report", p["report_path"]]
+    if p.get("dbt_mode"):
+        args += ["--dbt-mode", p["dbt_mode"]]
+    accepted_gaps = p.get("accept_gap") or p.get("accept_gaps") or []
+    if isinstance(accepted_gaps, str):
+        accepted_gaps = [accepted_gaps]
+    for gap_id in accepted_gaps:
+        args += ["--accept-gap", gap_id]
+    if p.get("accept_all"):
+        args.append("--accept-all")
+    if p.get("reset"):
+        args.append("--reset")
+
+    raw_candidate_paths = p.get("candidate_paths") or []
+    candidate_paths = [raw_candidate_paths] if isinstance(raw_candidate_paths, str) else list(raw_candidate_paths)
+    inline_candidates = p.get("candidates") or []
+    if isinstance(inline_candidates, dict):
+        inline_candidates = [inline_candidates]
+    with tempfile.TemporaryDirectory(prefix="flowx-agentic-candidates-") as temporary:
+        for index, candidate in enumerate(inline_candidates):
+            inline_path = Path(temporary) / f"candidate-{index}.json"
+            inline_path.write_text(json.dumps(candidate, indent=2), encoding="utf-8")
+            candidate_paths.append(str(inline_path))
+        for candidate_path in candidate_paths:
+            args += ["--candidate", candidate_path]
+        result = runner.run_adapter(args)
+    payload = runner.parse_stdout_json(result)
+    extra: dict[str, Any] = {"result": payload}
+    if action == "prepare":
+        extra["gaps"] = runner.read_json(output_dir / ".work" / "agentic" / "gaps.json")
+    return {"ok": result.ok, "process": result.as_dict(), **extra}
+
+
 def _cmd_inspect(p: dict[str, Any]) -> dict[str, Any]:
     args: list[Any] = ["inspect", p["report_path"]]
     for answer in p.get("answers") or []:
@@ -437,6 +486,7 @@ def _cmd_install_dashboard(p: dict[str, Any]) -> dict[str, Any]:
     "discover": _cmd_discover,
     "convert": _cmd_convert,
     "merge_agentic": _cmd_merge_agentic,
+    "resolve_agentic": _cmd_resolve_agentic,
     "inspect": _cmd_inspect,
     "apply_answers": _cmd_apply_answers,
     "materialize_lookup": _cmd_materialize_lookup,
@@ -485,8 +535,10 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A
         - "convert": source(req), (one ADF source key | airflow_source_path), output_dir, pipeline,
           exclude_dag | exclude_dags (Airflow, repeatable list).
         - "merge_agentic": source(req: "adf"), report_path(req), agentic_results_dir(req), output_path —
-          merge ADF agent results. Airflow's legacy name-based merge is disabled pending the
-          fingerprint-bound resolution workflow.
+          merge ADF agent results. Airflow's legacy name-based merge is disabled; use resolve_agentic.
+        - "resolve_agentic": source(req: "airflow"), action(req: prepare | stage | apply), output_dir,
+          airflow_source_path, report_path, candidates, accept_gap | accept_gaps, accept_all, reset —
+          prepare, stage, and explicitly apply fingerprint-bound Airflow leaf-gap resolutions.
         - "inspect": report_path(req) — return the full translation-option schema (every option with
           a `show_when` condition) for the agent to walk locally. See "Collecting options" below.
         - "apply_answers": report_path(req), answers(req, list of "ID=VALUE"), output_dir, lookup_csv.
diff --git a/src/flowx/sources/airflow/convert.py b/src/flowx/sources/airflow/convert.py
index 9ae29cb..71455ad 100644
--- a/src/flowx/sources/airflow/convert.py
+++ b/src/flowx/sources/airflow/convert.py
@@ -59,7 +59,7 @@ def main(argv: list[str] | None = None) -> int:
     logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
 
     if args.merge_agentic:
-        logger.error("Airflow agentic merge is disabled until the fingerprint-bound resolution workflow is available.")
+        logger.error("Airflow agentic merge is disabled; use the fingerprint-bound resolve-agentic workflow.")
         return 2
 
     if not args.source_dir:
diff --git a/tests/unit/test_airflow_agentic_resolution.py b/tests/unit/test_airflow_agentic_resolution.py
new file mode 100644
index 0000000..8c7ea9f
--- /dev/null
+++ b/tests/unit/test_airflow_agentic_resolution.py
@@ -0,0 +1,565 @@
+"""Tests for the fingerprint-bound Airflow agentic resolution workflow."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+import pytest
+
+from flowx.adapter.__main__ import main as adapter_main
+from flowx.bundler.dab_writer import main as package_main
+from flowx.sources.airflow.convert import main as airflow_convert
+
+
+def _write_source(tmp_path: Path, *, two_tasks: bool = False) -> Path:
+    source = tmp_path / "dag.py"
+    second = (
+        "    second = KubernetesPodOperator(task_id='second', image='python:3.12')\n    pod >> second\n"
+        if two_tasks
+        else ""
+    )
+    source.write_text(
+        "from airflow import DAG\n"
+        "with DAG(dag_id='agentic') as dag:\n"
+        "    pod = KubernetesPodOperator(task_id='pod', image='python:3.11', retries=2)\n"
+        f"{second}",
+        encoding="utf-8",
+    )
+    return source
+
+
+def _prepare(tmp_path: Path, *, two_tasks: bool = False) -> tuple[Path, Path, dict]:
+    source = _write_source(tmp_path, two_tasks=two_tasks)
+    output = tmp_path / "output"
+    assert airflow_convert(["--source-dir", str(source), "--output-dir", str(output)]) == 0
+    report = output / ".work" / "translation_report.json"
+    original = report.read_bytes()
+
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "prepare",
+                "--source",
+                "airflow",
+                "--source-path",
+                str(source),
+                "--report",
+                str(report),
+                "--output-dir",
+                str(output),
+            ]
+        )
+        == 0
+    )
+    assert report.read_bytes() == original
+    gaps = json.loads((output / ".work" / "agentic" / "gaps.json").read_text(encoding="utf-8"))
+    return source, output, gaps
+
+
+def _candidate(gap: dict, *, source: str = "print('Migrated from Airflow')\n", status: str = "resolved") -> dict:
+    generated_file = {
+        "path": "task.py",
+        "language": "python",
+        "content": source,
+        "sha256": hashlib.sha256(source.encode("utf-8")).hexdigest(),
+    }
+    dispositions = [
+        {
+            "name": argument["name"],
+            "disposition": "preserved_by_flowx" if argument["preserved_by_flowx"] else "consumed",
+            "rationale": "Flowx preserves task policy." if argument["preserved_by_flowx"] else "Used by notebook code.",
+        }
+        for argument in gap["arguments"]
+    ]
+    candidate = {
+        "contract_version": "1",
+        "gap_id": gap["gap_id"],
+        "status": status,
+        "baseline_report_sha256": gap["baseline_report_sha256"],
+        "source_sha256": gap["source_sha256"],
+        "provider": {
+            "name": "airflow-to-dabs",
+            "version": "0.1.0",
+            "repository": "https://github.com/park-peter/airflow-to-dabs",
+        },
+        "model": {"name": "test-model"},
+        "argument_disposition": dispositions,
+        "prerequisites": [],
+        "warnings": [],
+        "semantic_deltas": [],
+    }
+    if status == "resolved":
+        candidate["replacement"] = {"kind": "notebook", "file": "task.py"}
+        candidate["generated_files"] = [generated_file]
+    else:
+        candidate["reason"] = "More deployment information is required."
+    return candidate
+
+
+def _stage(output: Path, candidate: dict, *, name: str = "candidate.json") -> int:
+    candidate_path = output / name
+    candidate_path.write_text(json.dumps(candidate, indent=2), encoding="utf-8")
+    return adapter_main(
+        [
+            "resolve-agentic",
+            "stage",
+            "--source",
+            "airflow",
+            "--output-dir",
+            str(output),
+            "--candidate",
+            str(candidate_path),
+        ]
+    )
+
+
+def _load_tasks(report: Path) -> dict[str, dict]:
+    payload = json.loads(report.read_text(encoding="utf-8"))
+    pipeline = payload["pipelines"][0] if "pipelines" in payload else payload
+    return {task["task_key"]: task for task in pipeline["tasks"]}
+
+
+def test_prepare_writes_versioned_fingerprint_bound_gap_without_changing_report(tmp_path: Path):
+    source, output, gaps = _prepare(tmp_path)
+
+    assert len(gaps) == 1
+    gap = gaps[0]
+    assert gap["contract_version"] == "1"
+    assert gap["gap_id"]
+    assert gap["pipeline_name"] == "agentic"
+    assert gap["task_key"] == "pod"
+    assert gap["operator"] == "KubernetesPodOperator"
+    assert gap["source_sha256"] == hashlib.sha256(source.read_bytes()).hexdigest()
+    assert {argument["name"] for argument in gap["arguments"]} == {"task_id", "image", "retries"}
+    assert (output / ".work" / "agentic" / "baseline.json").exists()
+    assert (output / ".work" / "agentic" / "source" / "dag.py").read_bytes() == source.read_bytes()
+
+
+def test_resolve_agentic_is_explicitly_airflow_only(tmp_path: Path, capsys):
+    exit_code = adapter_main(
+        [
+            "resolve-agentic",
+            "prepare",
+            "--source",
+            "adf",
+            "--source-path",
+            str(tmp_path),
+            "--report",
+            str(tmp_path / "report.json"),
+            "--output-dir",
+            str(tmp_path / "output"),
+        ]
+    )
+
+    assert exit_code == 2
+    assert "not enabled for ADF; ADF uses the legacy merge path" in capsys.readouterr().err
+
+
+def test_stage_rejects_graph_identity_fields(tmp_path: Path, capsys):
+    _, output, gaps = _prepare(tmp_path)
+    candidate = _candidate(gaps[0])
+    candidate["replacement"]["task_key"] = "HIJACKED"
+
+    assert _stage(output, candidate) == 1
+    assert "replacement contains unsupported fields" in capsys.readouterr().err
+    assert not list((output / ".work" / "agentic" / "candidates").glob("*.json"))
+
+
+def test_stage_rejects_airflow_import_but_allows_airflow_in_comments(tmp_path: Path, capsys):
+    _, output, gaps = _prepare(tmp_path)
+    bad = _candidate(gaps[0], source="# Airflow provenance\nfrom airflow import DAG\n")
+
+    assert _stage(output, bad) == 1
+    assert "must not import Airflow" in capsys.readouterr().err
+
+    good = _candidate(gaps[0], source="# Migrated from Airflow\nprint('ok')\n")
+    assert _stage(output, good) == 0
+
+
+def test_stage_requires_complete_argument_disposition_and_ignored_rationale(tmp_path: Path, capsys):
+    _, output, gaps = _prepare(tmp_path)
+    missing = _candidate(gaps[0])
+    missing["argument_disposition"].pop()
+
+    assert _stage(output, missing) == 1
+    assert "argument_disposition must cover every source argument" in capsys.readouterr().err
+
+    ignored = _candidate(gaps[0])
+    ignored["argument_disposition"][1] = {
+        "name": ignored["argument_disposition"][1]["name"],
+        "disposition": "ignored",
+        "rationale": "",
+    }
+    assert _stage(output, ignored) == 1
+    assert "ignored argument requires a rationale" in capsys.readouterr().err
+
+
+def test_stage_rejects_unresolved_jinja_provider_drift_and_file_hash_mismatch(tmp_path: Path, capsys):
+    _, output, gaps = _prepare(tmp_path)
+
+    unresolved = _candidate(gaps[0], source="print('{{ ds }}')\n")
+    assert _stage(output, unresolved) == 1
+    assert "unresolved Airflow Jinja" in capsys.readouterr().err
+
+    wrong_provider = _candidate(gaps[0])
+    wrong_provider["provider"]["version"] = "0.2.0"
+    assert _stage(output, wrong_provider) == 1
+    assert "pinned airflow-to-dabs v0.1.0" in capsys.readouterr().err
+
+    bad_hash = _candidate(gaps[0])
+    bad_hash["generated_files"][0]["sha256"] = "0" * 64
+    assert _stage(output, bad_hash) == 1
+    assert "sha256 does not match" in capsys.readouterr().err
+
+
+def test_apply_rebuilds_from_baseline_and_preserves_graph_policy(tmp_path: Path):
+    _, output, gaps = _prepare(tmp_path)
+    assert _stage(output, _candidate(gaps[0])) == 0
+    baseline_report = output / ".work" / "agentic" / "baseline.json"
+    baseline_bytes = baseline_report.read_bytes()
+    baseline_task = _load_tasks(baseline_report)["pod"]
+
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+
+    applied_report = output / ".work" / "translation_report.agentic.json"
+    applied = json.loads(applied_report.read_text(encoding="utf-8"))
+    applied_task = _load_tasks(applied_report)["pod"]
+    for field in ("name", "task_key", "depends_on", "max_retries", "timeout_seconds", "min_retry_interval_millis"):
+        assert applied_task.get(field) == baseline_task.get(field)
+    assert applied_task["type"] == "NotebookActivity"
+    assert "Migrated from Airflow" in applied_task["generated_source"]
+    assert applied["reconciliation_status"] == "verified_with_reviewed_resolutions"
+    assert applied["audit"]["agentic_resolution"]["validation_status"] == "verified"
+    assert baseline_report.read_bytes() == baseline_bytes
+    assert (output / "metadata" / "agentic" / "accepted_resolutions.json").exists()
+
+
+def test_apply_supports_sql_leaf_payload(tmp_path: Path):
+    _, output, gaps = _prepare(tmp_path)
+    candidate = _candidate(gaps[0])
+    sql = "SELECT 1 AS resolved\n"
+    candidate["replacement"] = {"kind": "sql", "file": "task.sql", "parameters": {}}
+    candidate["generated_files"] = [
+        {
+            "path": "task.sql",
+            "language": "sql",
+            "content": sql,
+            "sha256": hashlib.sha256(sql.encode("utf-8")).hexdigest(),
+        }
+    ]
+    assert _stage(output, candidate) == 0
+
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+
+    task = _load_tasks(output / ".work" / "translation_report.agentic.json")["pod"]
+    assert task["type"] == "SqlActivity"
+    assert task["sql"] == sql
+    assert task["warehouse_ref"] == "${var.warehouse_id}"
+
+
+def test_nested_for_each_resolution_preserves_enclosing_control_flow(tmp_path: Path):
+    fixture = Path(__file__).resolve().parents[1] / "resources" / "airflow" / "review_repros" / "a8_classic_mapping.py"
+    source = tmp_path / "a8_classic_mapping.py"
+    source.write_bytes(fixture.read_bytes())
+    output = tmp_path / "output"
+    assert airflow_convert(["--source-dir", str(source), "--output-dir", str(output)]) == 0
+    report = output / ".work" / "translation_report.json"
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "prepare",
+                "--source",
+                "airflow",
+                "--source-path",
+                str(source),
+                "--report",
+                str(report),
+                "--output-dir",
+                str(output),
+            ]
+        )
+        == 0
+    )
+    gaps = json.loads((output / ".work" / "agentic" / "gaps.json").read_text(encoding="utf-8"))
+    assert len(gaps) == 1
+    assert _stage(output, _candidate(gaps[0], source="print(dbutils.widgets.get('env'))\n")) == 0
+    baseline = json.loads((output / ".work" / "agentic" / "baseline.json").read_text(encoding="utf-8"))
+    baseline_outer = baseline["tasks"][0]
+
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+
+    applied = json.loads((output / ".work" / "translation_report.agentic.json").read_text(encoding="utf-8"))
+    applied_outer = applied["tasks"][0]
+    assert applied_outer["type"] == baseline_outer["type"] == "ForEachActivity"
+    assert applied_outer["task_key"] == baseline_outer["task_key"]
+    assert applied_outer["items_expression"] == baseline_outer["items_expression"]
+    assert applied_outer["inner_activities"][0]["type"] == "NotebookActivity"
+
+
+def test_apply_rejects_staged_candidate_tampering(tmp_path: Path, capsys):
+    _, output, gaps = _prepare(tmp_path)
+    assert _stage(output, _candidate(gaps[0])) == 0
+    staged = output / ".work" / "agentic" / "candidates" / f"{gaps[0]['gap_id']}.json"
+    payload = json.loads(staged.read_text(encoding="utf-8"))
+    payload["replacement"]["kind"] = "sql"
+    staged.write_text(json.dumps(payload), encoding="utf-8")
+
+    exit_code = adapter_main(
+        [
+            "resolve-agentic",
+            "apply",
+            "--source",
+            "airflow",
+            "--output-dir",
+            str(output),
+            "--accept-gap",
+            gaps[0]["gap_id"],
+        ]
+    )
+
+    assert exit_code == 1
+    assert "staged candidate was modified after validation" in capsys.readouterr().err
+    assert not (output / ".work" / "translation_report.agentic.json").exists()
+
+
+def test_apply_rejects_malformed_candidate_index(tmp_path: Path, capsys):
+    _, output, _ = _prepare(tmp_path)
+    index = output / ".work" / "agentic" / "candidate_index.json"
+    index.write_text(
+        json.dumps({"../../outside": {"sha256": "0" * 64, "status": "resolved"}}),
+        encoding="utf-8",
+    )
+
+    exit_code = adapter_main(
+        ["resolve-agentic", "apply", "--source", "airflow", "--output-dir", str(output), "--reset"]
+    )
+
+    assert exit_code == 1
+    assert "Candidate index contains an unknown gap_id" in capsys.readouterr().err
+    assert not (output / ".work" / "translation_report.agentic.json").exists()
+
+
+def test_apply_rejects_live_source_changes(tmp_path: Path, capsys):
+    source, output, gaps = _prepare(tmp_path)
+    assert _stage(output, _candidate(gaps[0])) == 0
+    source.write_text(source.read_text(encoding="utf-8") + "# changed\n", encoding="utf-8")
+
+    exit_code = adapter_main(
+        [
+            "resolve-agentic",
+            "apply",
+            "--source",
+            "airflow",
+            "--output-dir",
+            str(output),
+            "--accept-gap",
+            gaps[0]["gap_id"],
+        ]
+    )
+
+    assert exit_code == 1
+    assert "source changed since prepare; re-run prepare" in capsys.readouterr().err
+
+
+def test_reduced_allowlist_restores_unaccepted_placeholder_and_reset_restores_all(tmp_path: Path):
+    _, output, gaps = _prepare(tmp_path, two_tasks=True)
+    for index, gap in enumerate(gaps):
+        assert _stage(output, _candidate(gap), name=f"candidate-{index}.json") == 0
+
+    assert (
+        adapter_main(["resolve-agentic", "apply", "--source", "airflow", "--output-dir", str(output), "--accept-all"])
+        == 0
+    )
+    applied_report = output / ".work" / "translation_report.agentic.json"
+    assert {task["type"] for task in _load_tasks(applied_report).values()} == {"NotebookActivity"}
+
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+    types = {task_key: task["type"] for task_key, task in _load_tasks(applied_report).items()}
+    assert types[gaps[0]["task_key"]] == "NotebookActivity"
+    assert types[gaps[1]["task_key"]] == "PlaceholderActivity"
+
+    assert (
+        adapter_main(["resolve-agentic", "apply", "--source", "airflow", "--output-dir", str(output), "--reset"]) == 0
+    )
+    assert {task["type"] for task in _load_tasks(applied_report).values()} == {"PlaceholderActivity"}
+
+
+@pytest.mark.parametrize("status", ["needs_input", "deferred"])
+def test_unresolved_outcome_is_terminal_and_keeps_the_linked_placeholder(tmp_path: Path, status: str):
+    _, output, gaps = _prepare(tmp_path)
+    assert _stage(output, _candidate(gaps[0], status=status)) == 0
+
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+
+    report = output / ".work" / "translation_report.agentic.json"
+    assert _load_tasks(report)["pod"]["type"] == "PlaceholderActivity"
+    payload = json.loads(report.read_text(encoding="utf-8"))
+    finding = next(item for item in payload["not_translatable"] if item["fingerprint"] == gaps[0]["gap_id"])
+    assert finding["resolution"]["status"] == status
+
+
+def test_package_accepts_only_flowx_verified_agentic_report(tmp_path: Path):
+    _, output, gaps = _prepare(tmp_path)
+    assert _stage(output, _candidate(gaps[0])) == 0
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+
+    report = output / ".work" / "translation_report.agentic.json"
+    assert (
+        package_main(
+            [
+                "--report",
+                str(report),
+                "--output-dir",
+                str(output),
+                "--no-download-workspace-files",
+            ]
+        )
+        == 0
+    )
+    assert not (output / ".work").exists()
+    assert (output / "metadata" / "agentic" / "accepted_resolutions.json").exists()
+
+
+def test_package_replays_terminal_needs_input_evidence(tmp_path: Path):
+    _, output, gaps = _prepare(tmp_path)
+    assert _stage(output, _candidate(gaps[0], status="needs_input")) == 0
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+    report = output / ".work" / "translation_report.agentic.json"
+    payload = json.loads(report.read_text(encoding="utf-8"))
+    assert payload["reconciliation_status"] == "verified_with_gaps"
+    accepted = output / "metadata" / "agentic" / "accepted_resolutions.json"
+    accepted.write_text(json.dumps({"contract_version": "1", "candidates": []}), encoding="utf-8")
+    bundle = tmp_path / "bundle"
+
+    assert package_main(["--report", str(report), "--output-dir", str(bundle)]) == 1
+    assert not (bundle / "databricks.yml").exists()
+
+
+def test_package_rejects_agentic_report_tampering_before_bundle_writes(tmp_path: Path):
+    _, output, gaps = _prepare(tmp_path)
+    assert _stage(output, _candidate(gaps[0])) == 0
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+    report = output / ".work" / "translation_report.agentic.json"
+    payload = json.loads(report.read_text(encoding="utf-8"))
+    payload["tasks"][0]["task_key"] = "HIJACKED"
+    report.write_text(json.dumps(payload), encoding="utf-8")
+    bundle = tmp_path / "bundle"
+
+    assert package_main(["--report", str(report), "--output-dir", str(bundle)]) == 1
+    assert not (bundle / "databricks.yml").exists()
diff --git a/tests/unit/test_mcp_source_routing.py b/tests/unit/test_mcp_source_routing.py
index 54877d1..ee265eb 100644
--- a/tests/unit/test_mcp_source_routing.py
+++ b/tests/unit/test_mcp_source_routing.py
@@ -120,7 +120,50 @@ def test_merge_agentic_rejects_airflow_without_invoking_adapter(captured):
 
     assert result == {
         "ok": False,
-        "error": "Airflow agentic merge is disabled until the fingerprint-bound resolution workflow is available.",
+        "error": "Airflow agentic merge is disabled; use the fingerprint-bound resolve_agentic workflow.",
+    }
+    assert captured == []
+
+
+def test_resolve_agentic_prepare_routes_airflow_contract(captured):
+    result = server._cmd_resolve_agentic(
+        {
+            "source": "airflow",
+            "action": "prepare",
+            "airflow_source_path": "/tmp/dags",
+            "report_path": "/tmp/out/.work/translation_report.json",
+            "output_dir": "/tmp/out",
+        }
+    )
+
+    argv = _argv(captured, "resolve-agentic")
+    assert argv[:4] == ["resolve-agentic", "prepare", "--source", "airflow"]
+    assert argv[argv.index("--source-path") + 1] == "/tmp/dags"
+    assert argv[argv.index("--report") + 1] == "/tmp/out/.work/translation_report.json"
+    assert result["ok"] is True
+
+
+def test_resolve_agentic_stage_materializes_inline_candidate(captured):
+    result = server._cmd_resolve_agentic(
+        {
+            "source": "airflow",
+            "action": "stage",
+            "output_dir": "/tmp/out",
+            "candidates": [{"gap_id": "abc"}],
+        }
+    )
+
+    argv = _argv(captured, "resolve-agentic")
+    assert "--candidate" in argv
+    assert result["ok"] is True
+
+
+def test_resolve_agentic_rejects_adf_without_invoking_adapter(captured):
+    result = server._cmd_resolve_agentic({"source": "adf", "action": "prepare", "output_dir": "/tmp/out"})
+
+    assert result == {
+        "ok": False,
+        "error": "resolve_agentic is not enabled for ADF; ADF uses the legacy merge path.",
     }
     assert captured == []
 
diff --git a/tests/unit/test_package_invariants.py b/tests/unit/test_package_invariants.py
index 5420273..869551b 100644
--- a/tests/unit/test_package_invariants.py
+++ b/tests/unit/test_package_invariants.py
@@ -214,7 +214,7 @@ def test_report_preflight_rejects_unknown_reconciliation_status(tmp_path: Path):
     assert any("unknown reconciliation_status" in failure for failure in failures)
 
 
-def test_report_preflight_rejects_reviewed_resolution_status_until_validator_exists(tmp_path: Path):
+def test_report_preflight_rejects_reviewed_resolution_status_without_replay_evidence(tmp_path: Path):
     report = _airflow_pipeline(
         "premature_resolution",
         [_notebook_task("a", "a")],
@@ -223,7 +223,7 @@ def test_report_preflight_rejects_reviewed_resolution_status_until_validator_exi
 
     failures = _preflight_failures(tmp_path, report)
 
-    assert any("unknown reconciliation_status" in failure for failure in failures)
+    assert any("agentic resolution evidence" in failure for failure in failures)
 
 
 def test_report_preflight_rejects_excluded_status_for_included_dag(tmp_path: Path):

From 81df2a157711c336724e54f84de83310bfc0e327 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Sat, 8 Aug 2026 23:38:22 -0700
Subject: [PATCH 56/77] Report reviewed Airflow gap resolutions

---
 README.md                                     |   6 +-
 docs/content/docs/architecture.mdx            |   2 +-
 docs/content/docs/guide.mdx                   |   4 +-
 docs/content/docs/options.mdx                 |  14 +-
 .../flowx-convert/sources/airflow-coverage.md |   6 +
 skills/flowx-resolve-airflow-gaps/SKILL.md    |  10 +-
 .../airflow-to-dabs-v0.2.0/PROFILE.md         |  84 ++++++++++++
 .../fixtures/gap-deferred.json                |  37 +++++
 .../fixtures/gap-needs-input.json             |  43 ++++++
 .../fixtures/gap-notebook.json                |  43 ++++++
 .../fixtures/gap-sql.json                     |  42 ++++++
 .../fixtures/resolution-deferred.json         |  34 +++++
 .../fixtures/resolution-needs-input.json      |  44 ++++++
 .../fixtures/resolution-notebook.json         |  52 ++++++++
 .../fixtures/resolution-sql.json              |  47 +++++++
 .../airflow-to-dabs-v0.2.0/provider.json      |  52 ++++++++
 .../references/contract-v1.md                 |   4 +-
 src/flowx/agentic.py                          | 126 +++++++++++++++---
 src/flowx/reporting/coverage.py               |  62 ++++++++-
 src/flowx/reporting/dashboard_template.json   |  68 +++++++---
 src/flowx/reporting/results.py                |  16 ++-
 tests/unit/test_airflow_agentic_resolution.py | 114 +++++++++++++++-
 tests/unit/test_reporting_coverage.py         |  19 ++-
 tests/unit/test_reporting_dashboard.py        |  17 +++
 tests/unit/test_reporting_results.py          |  16 +++
 25 files changed, 907 insertions(+), 55 deletions(-)
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/PROFILE.md
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-deferred.json
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-needs-input.json
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-notebook.json
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-sql.json
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-deferred.json
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-needs-input.json
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-notebook.json
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-sql.json
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/provider.json

diff --git a/README.md b/README.md
index 4ee120a..33205ee 100644
--- a/README.md
+++ b/README.md
@@ -166,7 +166,9 @@ execution) and maps ~35 operator/sensor families to the shared IR. Highlights:
   `params={...}` → job parameters, `>>` / `<<` / `set_upstream` / TaskGroup edges.
 
 Operators without a deterministic mapping become a failing placeholder and are recorded in
-`gaps.json` for review. Airflow agentic resolution is not a supported workflow yet. Full matrix:
+`gaps.json` for review. Eligible leaf gaps can use the fingerprint-bound resolver backed by the
+pinned [`airflow-to-dabs` v0.2.0](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.0)
+provider profile; flowx retains ownership of parsing, graph identity, policy, IR, and packaging. Full matrix:
 [`skills/flowx-convert/sources/airflow-coverage.md`](skills/flowx-convert/sources/airflow-coverage.md).
 
 Airflow discovery independently audits DAG declarations, task candidates, dependency declarations,
@@ -184,7 +186,7 @@ to the supported static subset; flowx never imports or executes DAG modules.
 Parses the source into typed nodes and classifies each activity/operator as deterministic, agentic, or unsupported — ADF JSON from Unity Catalog volumes (or a `/Workspace` Git folder, normalizing ARM template format), or Airflow DAG `.py` modules read statically with `ast`. Airflow inventory includes audited/deterministic/agentic/failed/excluded counts, reconciliation status, stable finding fingerprints, translation-path coverage, and deterministic coverage. Produces `metadata/inventory.json` and a per-pipeline complexity report at `metadata/profile_report.csv`.
 
 ### Phase 2: Convert
-Applies deterministic translators (ADF activity registry / Airflow operator mapping), resolves dependencies, and records unresolved gaps. ADF supports the guided agentic translation workflow; Airflow currently retains failing placeholders for explicit review. Produces the shared Pipeline IR consumed unchanged by the package phase.
+Applies deterministic translators (ADF activity registry / Airflow operator mapping), resolves dependencies, and records unresolved gaps. ADF supports its guided agentic translation workflow. Airflow supports a fingerprint-bound, explicitly reviewed leaf-gap workflow whose constrained provider output is replayed against an immutable deterministic baseline before packaging. Produces the shared Pipeline IR consumed unchanged by the package phase.
 
 ### Phase 3: Package
 Converts Pipeline IR into a deployable DABs project: `databricks.yml`, per-job YAML resource files, generated Python notebooks, and setup scripts for UC volumes, secrets, and connections.
diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx
index 99d6b9b..ce327bb 100644
--- a/docs/content/docs/architecture.mdx
+++ b/docs/content/docs/architecture.mdx
@@ -28,7 +28,7 @@ Each activity is classified with a `TranslationStrategy`:
 * `AGENTIC` (LLM-assisted gaps)
 * `UNSUPPORTED`
 
-The reporting layer (`reporting/`) can write per-run coverage to a Unity Catalog table and publish an AI/BI dashboard. Airflow rows use independently audited candidates as the denominator and persist reconciliation status, failed/excluded counts, stable finding fingerprints, translation-path coverage, and deterministic coverage.
+The reporting layer (`reporting/`) can write per-run coverage to a Unity Catalog table and publish an AI/BI dashboard. Airflow rows use independently audited candidates as the denominator and persist reconciliation status, failed/excluded counts, stable finding fingerprints, translation-path coverage, deterministic coverage, reviewed agentic outcomes, and mechanically validated code-attached coverage. Provider-authored code remains distinct from deterministic translation and requires human review.
 
 ## Two surfaces over one core
 
diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx
index e263f02..ffd7fb1 100644
--- a/docs/content/docs/guide.mdx
+++ b/docs/content/docs/guide.mdx
@@ -85,7 +85,9 @@ and `run_by` (`record-results`) — and install a published AI/BI coverage dashb
 
 For Airflow, `activities` is the independent source-audit count rather than the number of tasks the
 translator happened to emit. Reporting distinguishes deterministic, agentic, failed, and excluded
-candidates and carries reconciliation status plus both translation-path and deterministic coverage.
+candidates and carries reconciliation status, translation-path coverage, deterministic coverage,
+unresolved agentic outcomes, and mechanically validated code-attached coverage. Code attachment is
+not a certification that provider-authored code is semantically correct.
 
 
 
diff --git a/docs/content/docs/options.mdx b/docs/content/docs/options.mdx
index cc4f1e7..51584a7 100644
--- a/docs/content/docs/options.mdx
+++ b/docs/content/docs/options.mdx
@@ -115,14 +115,18 @@ phase surfaces three optional inputs — `results_table`, `results_warehouse_id`
 - **`record-results`** writes one row **per pipeline per run** to the supplied Unity Catalog
   table (`catalog.schema.table`), combining the complexity columns above with the
   audited/deterministic/agentic/failed/excluded coverage breakdown, reconciliation and migration
-  status, finding fingerprints, translation-path coverage, and deterministic coverage. Airflow's
-  audited count remains the denominator even for failed or excluded candidates. Every row is stamped with a shared
+  status, finding fingerprints, translation-path coverage, deterministic coverage, unresolved
+  agentic count, reviewed-resolution outcomes/provider version, and code-attached coverage.
+  Airflow's audited count remains the denominator even for failed or excluded candidates.
+  Code-attached coverage counts deterministic tasks plus accepted `resolved` provider candidates;
+  it means the generated code passed mechanical contract validation, not that its semantics were
+  certified. Every row is stamped with a shared
   **`run_id`** (UUID), **`run_date`** (`CURRENT_TIMESTAMP()`), and **`run_by`**
   (`CURRENT_USER()`), so coverage is trackable across runs and users.
 - **`install-dashboard`** creates and publishes an AI/BI (Lakeview) dashboard over that table —
-  KPI counters (pipelines, audited activities, and coverage), failed/excluded totals, a
-  pipelines-by-complexity bar chart, a coverage-over-runs line, and a per-pipeline coverage
-  table.
+  KPI counters (pipelines, audited activities, and mechanically validated code-attached coverage),
+  failed/excluded totals, a pipelines-by-complexity bar chart, a code-attached-coverage trend, and a
+  per-pipeline table that retains translation-path and deterministic coverage.
 
 The SQL warehouse is auto-detected (preferring a running serverless warehouse) when
 `results_warehouse_id` is left blank. Both run via the Databricks SDK and degrade gracefully
diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md
index 8af3c08..c07b64b 100644
--- a/skills/flowx-convert/sources/airflow-coverage.md
+++ b/skills/flowx-convert/sources/airflow-coverage.md
@@ -46,6 +46,12 @@ safe fallback is a flagged, failing task rather than a silent omission. Callable
 (`**context` / `ti`) or XCom, and runtime-branching decorators, take the same route rather than
 emitting code that fails at runtime.
 
+The resolver consumes the pinned `airflow-to-dabs` v0.2.0 Flowx provider profile. It receives one
+flowx-produced gap envelope and cannot express graph or task-policy changes. Accepted `resolved`
+candidates contribute to mechanically validated code-attached coverage, but remain agentic and do
+not increase deterministic coverage. `needs_input`, `deferred`, and unreviewed candidates remain
+linked failing placeholders.
+
 ## Not yet supported
 
 These are absent but fail safely — routed to a linked placeholder notebook that raises
diff --git a/skills/flowx-resolve-airflow-gaps/SKILL.md b/skills/flowx-resolve-airflow-gaps/SKILL.md
index cca896d..3cacf6f 100644
--- a/skills/flowx-resolve-airflow-gaps/SKILL.md
+++ b/skills/flowx-resolve-airflow-gaps/SKILL.md
@@ -10,10 +10,16 @@ description: >
 Use this workflow only for Airflow reports whose deterministic conversion succeeded with gaps.
 Flowx owns source parsing, task identity, dependencies, task policy, IR, and packaging. This skill
 reasons about one prepared gap at a time using the migration knowledge from
-[`park-peter/airflow-to-dabs` v0.1.0](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.1.0).
+[`park-peter/airflow-to-dabs` v0.2.0](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.0).
 It must not parse the DAG independently or generate a second bundle.
 
-Read [`references/contract-v1.md`](references/contract-v1.md) before authoring a resolution.
+Read [`references/contract-v1.md`](references/contract-v1.md) and the pinned
+[`airflow-to-dabs-v0.2.0/PROFILE.md`](references/airflow-to-dabs-v0.2.0/PROFILE.md) before authoring a
+resolution. The profile's `../../references/*.md` knowledge paths are relative to the upstream
+v0.2.0 release. Resolve them against
+`https://github.com/park-peter/airflow-to-dabs/tree/v0.2.0/references` or an exact local checkout of
+that tag. If required knowledge is unavailable, return `needs_input` or `deferred`; never infer
+missing operator semantics.
 
 ## 1. Prepare immutable gap envelopes
 
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/PROFILE.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/PROFILE.md
new file mode 100644
index 0000000..e321507
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/PROFILE.md
@@ -0,0 +1,84 @@
+# Flowx Airflow Gap Resolver Profile
+
+Resolve exactly one source-reconciled Airflow leaf gap supplied by Flowx. Flowx owns DAG parsing,
+capture identity, task keys, dependencies, task policy, control flow, IR, and bundle packaging. Do
+not reopen or parse the original DAG, construct another task graph, or generate a bundle.
+
+This profile implements Flowx Airflow agentic gap contract `1` with the pinned provider identity:
+
+```json
+{
+  "name": "airflow-to-dabs",
+  "version": "0.2.0",
+  "repository": "https://github.com/park-peter/airflow-to-dabs"
+}
+```
+
+## Inputs
+
+Accept one `GapEnvelope` JSON object. Use only the captured source, arguments, surrounding task-key
+context, DAG settings, and finding reason in that envelope. Reject an envelope when:
+
+- `contract_version` is not `"1"`;
+- `source` is not `"airflow"`;
+- `knowledge_provider` does not match the pinned provider identity;
+- the requested behavior cannot be determined without source or deployment information absent from
+  the envelope.
+
+Read `../../references/operator-mapping.md` for operator semantics. Read another knowledge file
+listed in `provider.json` only when the gap involves that domain. These references inform a leaf
+resolution; they do not grant authority to emit jobs, triggers, clusters, pipelines, or graph edits.
+
+## Resolution procedure
+
+1. Classify the operator's intent from `operator_fqn`, `raw_definition`, and `arguments`.
+2. Decide the terminal status:
+   - `resolved`: one self-contained Python notebook or SQL file preserves the represented behavior.
+   - `needs_input`: a concrete deployment fact, credential mapping, runtime dependency, or semantic
+     choice is required before a safe leaf implementation can be written.
+   - `deferred`: a faithful migration requires graph, control-flow, schedule, compute, resource, or
+     other changes outside the leaf-only contract.
+3. Account for every envelope argument exactly once in `argument_disposition`:
+   - `consumed`: the generated payload or resolution decision uses it;
+   - `preserved_by_flowx`: Flowx retains it as task identity or policy;
+   - `ignored`: the resolution intentionally omits it and states the exact behavioral loss.
+4. Enumerate prerequisites, warnings, and semantic deltas. Never hide a dropped behavior in prose or
+   omit an argument from the disposition list.
+5. Return one `AgenticResolution` JSON object and no bundle files or graph patches.
+
+## Resolved payload rules
+
+- Emit exactly one replacement with `kind` equal to `notebook` or `sql`.
+- Emit exactly one inline generated file whose `path` matches `replacement.file` and whose `sha256`
+  is the lowercase SHA-256 of the UTF-8 content bytes.
+- For a notebook, emit syntactically valid Python with no `import airflow` or `from airflow ...`
+  statements. Airflow may be named in comments.
+- For SQL, use Databricks SQL syntax. Put dynamic values in named parameter markers and declare the
+  corresponding string values in `replacement.parameters`.
+- Do not emit unresolved Airflow Jinja. Databricks dynamic references such as
+  `{{job.parameters.x}}`, `{{tasks.upstream.values.x}}`, `{{input}}`, and `{{backfill.iso_date}}` are
+  allowed when valid for the captured context.
+- Keep the replacement self-contained. Record required libraries, secrets, UC objects, network
+  access, or user decisions in `prerequisites`; do not invent them.
+
+## Forbidden authority
+
+Never include task names, task keys, dependencies, retries, timeouts, clusters, libraries,
+schedules, triggers, notifications, control-flow bodies, or graph mutations in `replacement`.
+Return `deferred` when those changes are required. Return `needs_input` when a safe leaf result might
+be possible after the user supplies missing information.
+
+## Output shape
+
+Use only these top-level fields:
+
+- always: `contract_version`, `gap_id`, `status`, `baseline_report_sha256`, `source_sha256`,
+  `provider`, `model`, `argument_disposition`, `prerequisites`, `warnings`, `semantic_deltas`;
+- `resolved`: add `replacement` and `generated_files`;
+- `needs_input` or `deferred`: add `reason` and omit `replacement` and `generated_files`.
+
+Copy the gap, baseline, and source hashes verbatim from the envelope. Set `model.name` to the actual
+model identifier. Do not retry `needs_input` or `deferred` automatically.
+
+Use the paired files under `fixtures/` as contract examples. They are interoperability fixtures,
+not permission to substitute their assumptions into another gap.
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-deferred.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-deferred.json
new file mode 100644
index 0000000..894737e
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-deferred.json
@@ -0,0 +1,37 @@
+{
+  "contract_version": "1",
+  "gap_id": "4444444444444444",
+  "source": "airflow",
+  "pipeline_name": "branching",
+  "capture_identity": "choose_path",
+  "task_key": "choose_path",
+  "task_path": ["tasks", 1],
+  "operator": "BranchPythonOperator",
+  "operator_fqn": "airflow.operators.python.BranchPythonOperator",
+  "source_file": "branching.py",
+  "source_sha256": "9999999999999999999999999999999999999999999999999999999999999999",
+  "baseline_report_sha256": "8888888888888888888888888888888888888888888888888888888888888888",
+  "source_span": {"line": 21, "column": 4, "end_line": 25, "end_column": 5},
+  "raw_definition": {
+    "operator": "BranchPythonOperator",
+    "source": "choose = BranchPythonOperator(task_id=\"choose_path\", python_callable=choose_target, trigger_rule=\"none_failed\")"
+  },
+  "arguments": [
+    {"name": "task_id", "source_expression": "'choose_path'", "preserved_by_flowx": true},
+    {"name": "python_callable", "source_expression": "choose_target", "preserved_by_flowx": false},
+    {"name": "trigger_rule", "source_expression": "'none_failed'", "preserved_by_flowx": true}
+  ],
+  "upstream_task_keys": ["read_config"],
+  "downstream_task_keys": ["full_load", "incremental_load"],
+  "dag_settings": {"schedule": null, "parameters": [], "tags": {"source": "airflow"}},
+  "reason": {
+    "code": "operator_placeholder",
+    "message": "BranchPythonOperator requires a graph-aware branch conversion"
+  },
+  "allowed_replacement_kinds": ["notebook", "sql"],
+  "knowledge_provider": {
+    "name": "airflow-to-dabs",
+    "version": "0.2.0",
+    "repository": "https://github.com/park-peter/airflow-to-dabs"
+  }
+}
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-needs-input.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-needs-input.json
new file mode 100644
index 0000000..9c6e6aa
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-needs-input.json
@@ -0,0 +1,43 @@
+{
+  "contract_version": "1",
+  "gap_id": "3333333333333333",
+  "source": "airflow",
+  "pipeline_name": "container_workload",
+  "capture_identity": "run_container",
+  "task_key": "run_container",
+  "task_path": ["tasks", 1],
+  "operator": "KubernetesPodOperator",
+  "operator_fqn": "airflow.providers.cncf.kubernetes.operators.pod.KubernetesPodOperator",
+  "source_file": "container_workload.py",
+  "source_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+  "baseline_report_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+  "source_span": {"line": 14, "column": 4, "end_line": 22, "end_column": 5},
+  "raw_definition": {
+    "operator": "KubernetesPodOperator",
+    "source": "run = KubernetesPodOperator(task_id=\"run_container\", image=\"registry.example.com/orders:7\", cmds=[\"python\", \"/app/run.py\"], namespace=\"data\", secrets=[orders_secret])"
+  },
+  "arguments": [
+    {"name": "task_id", "source_expression": "'run_container'", "preserved_by_flowx": true},
+    {
+      "name": "image",
+      "source_expression": "'registry.example.com/orders:7'",
+      "preserved_by_flowx": false
+    },
+    {"name": "cmds", "source_expression": "['python', '/app/run.py']", "preserved_by_flowx": false},
+    {"name": "namespace", "source_expression": "'data'", "preserved_by_flowx": false},
+    {"name": "secrets", "source_expression": "[orders_secret]", "preserved_by_flowx": false}
+  ],
+  "upstream_task_keys": ["build_inputs"],
+  "downstream_task_keys": ["publish_results"],
+  "dag_settings": {"schedule": null, "parameters": [], "tags": {"source": "airflow"}},
+  "reason": {
+    "code": "operator_placeholder",
+    "message": "KubernetesPodOperator requires deployment-specific migration decisions"
+  },
+  "allowed_replacement_kinds": ["notebook", "sql"],
+  "knowledge_provider": {
+    "name": "airflow-to-dabs",
+    "version": "0.2.0",
+    "repository": "https://github.com/park-peter/airflow-to-dabs"
+  }
+}
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-notebook.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-notebook.json
new file mode 100644
index 0000000..bdb2196
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-notebook.json
@@ -0,0 +1,43 @@
+{
+  "contract_version": "1",
+  "gap_id": "1111111111111111",
+  "source": "airflow",
+  "pipeline_name": "orders",
+  "capture_identity": "notify_orders",
+  "task_key": "notify_orders",
+  "task_path": ["tasks", 2],
+  "operator": "SimpleHttpOperator",
+  "operator_fqn": "airflow.providers.http.operators.http.SimpleHttpOperator",
+  "source_file": "orders.py",
+  "source_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  "baseline_report_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+  "source_span": {"line": 18, "column": 4, "end_line": 24, "end_column": 5},
+  "raw_definition": {
+    "operator": "SimpleHttpOperator",
+    "source": "notify = SimpleHttpOperator(task_id=\"notify_orders\", endpoint=\"https://example.com/hooks/orders\", method=\"POST\", data={\"event\": \"orders_ready\"}, retries=2)"
+  },
+  "arguments": [
+    {"name": "task_id", "source_expression": "'notify_orders'", "preserved_by_flowx": true},
+    {
+      "name": "endpoint",
+      "source_expression": "'https://example.com/hooks/orders'",
+      "preserved_by_flowx": false
+    },
+    {"name": "method", "source_expression": "'POST'", "preserved_by_flowx": false},
+    {"name": "data", "source_expression": "{'event': 'orders_ready'}", "preserved_by_flowx": false},
+    {"name": "retries", "source_expression": "2", "preserved_by_flowx": true}
+  ],
+  "upstream_task_keys": ["publish_orders"],
+  "downstream_task_keys": [],
+  "dag_settings": {"schedule": null, "parameters": [], "tags": {"source": "airflow"}},
+  "reason": {
+    "code": "operator_placeholder",
+    "message": "SimpleHttpOperator requires a provider-authored leaf implementation"
+  },
+  "allowed_replacement_kinds": ["notebook", "sql"],
+  "knowledge_provider": {
+    "name": "airflow-to-dabs",
+    "version": "0.2.0",
+    "repository": "https://github.com/park-peter/airflow-to-dabs"
+  }
+}
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-sql.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-sql.json
new file mode 100644
index 0000000..c810269
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-sql.json
@@ -0,0 +1,42 @@
+{
+  "contract_version": "1",
+  "gap_id": "2222222222222222",
+  "source": "airflow",
+  "pipeline_name": "retention",
+  "capture_identity": "cleanup_events",
+  "task_key": "cleanup_events",
+  "task_path": ["tasks", 0],
+  "operator": "SQLExecuteQueryOperator",
+  "operator_fqn": "airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator",
+  "source_file": "retention.py",
+  "source_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
+  "baseline_report_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
+  "source_span": {"line": 9, "column": 4, "end_line": 15, "end_column": 5},
+  "raw_definition": {
+    "operator": "SQLExecuteQueryOperator",
+    "source": "cleanup = SQLExecuteQueryOperator(task_id=\"cleanup_events\", conn_id=\"databricks_default\", sql=\"DELETE FROM main.ops.events WHERE processed_at < current_date() - INTERVAL 30 DAYS\", autocommit=True)"
+  },
+  "arguments": [
+    {"name": "task_id", "source_expression": "'cleanup_events'", "preserved_by_flowx": true},
+    {"name": "conn_id", "source_expression": "'databricks_default'", "preserved_by_flowx": false},
+    {
+      "name": "sql",
+      "source_expression": "'DELETE FROM main.ops.events WHERE processed_at < current_date() - INTERVAL 30 DAYS'",
+      "preserved_by_flowx": false
+    },
+    {"name": "autocommit", "source_expression": "True", "preserved_by_flowx": false}
+  ],
+  "upstream_task_keys": [],
+  "downstream_task_keys": ["vacuum_events"],
+  "dag_settings": {"schedule": null, "parameters": [], "tags": {"source": "airflow"}},
+  "reason": {
+    "code": "operator_placeholder",
+    "message": "SQLExecuteQueryOperator requires a provider-authored leaf implementation"
+  },
+  "allowed_replacement_kinds": ["notebook", "sql"],
+  "knowledge_provider": {
+    "name": "airflow-to-dabs",
+    "version": "0.2.0",
+    "repository": "https://github.com/park-peter/airflow-to-dabs"
+  }
+}
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-deferred.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-deferred.json
new file mode 100644
index 0000000..944d268
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-deferred.json
@@ -0,0 +1,34 @@
+{
+  "contract_version": "1",
+  "gap_id": "4444444444444444",
+  "status": "deferred",
+  "baseline_report_sha256": "8888888888888888888888888888888888888888888888888888888888888888",
+  "source_sha256": "9999999999999999999999999999999999999999999999999999999999999999",
+  "provider": {
+    "name": "airflow-to-dabs",
+    "version": "0.2.0",
+    "repository": "https://github.com/park-peter/airflow-to-dabs"
+  },
+  "model": {"name": "fixture-model"},
+  "argument_disposition": [
+    {
+      "name": "task_id",
+      "disposition": "preserved_by_flowx",
+      "rationale": "Flowx preserves the collision-safe task identity."
+    },
+    {
+      "name": "python_callable",
+      "disposition": "consumed",
+      "rationale": "The callable is recognized as selecting downstream task identities."
+    },
+    {
+      "name": "trigger_rule",
+      "disposition": "preserved_by_flowx",
+      "rationale": "Flowx preserves supported task-run policy independently of the provider."
+    }
+  ],
+  "prerequisites": [],
+  "warnings": [],
+  "semantic_deltas": [],
+  "reason": "A faithful branch migration requires condition tasks and downstream dependency rewrites, which are outside the leaf-only provider contract."
+}
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-needs-input.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-needs-input.json
new file mode 100644
index 0000000..c57751d
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-needs-input.json
@@ -0,0 +1,44 @@
+{
+  "contract_version": "1",
+  "gap_id": "3333333333333333",
+  "status": "needs_input",
+  "baseline_report_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+  "source_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+  "provider": {
+    "name": "airflow-to-dabs",
+    "version": "0.2.0",
+    "repository": "https://github.com/park-peter/airflow-to-dabs"
+  },
+  "model": {"name": "fixture-model"},
+  "argument_disposition": [
+    {
+      "name": "task_id",
+      "disposition": "preserved_by_flowx",
+      "rationale": "Flowx preserves the collision-safe task identity."
+    },
+    {
+      "name": "image",
+      "disposition": "consumed",
+      "rationale": "The image identifies the runtime whose dependencies must be assessed."
+    },
+    {
+      "name": "cmds",
+      "disposition": "consumed",
+      "rationale": "The command identifies the container entrypoint that must be migrated."
+    },
+    {
+      "name": "namespace",
+      "disposition": "consumed",
+      "rationale": "The namespace is deployment context needed to locate Kubernetes dependencies."
+    },
+    {
+      "name": "secrets",
+      "disposition": "consumed",
+      "rationale": "The secret reference must be mapped to Databricks secrets or Unity Catalog."
+    }
+  ],
+  "prerequisites": [],
+  "warnings": [],
+  "semantic_deltas": [],
+  "reason": "Provide the container source or packaged application, required Python/system dependencies, registry access requirements, and the Databricks secret or Unity Catalog mappings for orders_secret."
+}
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-notebook.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-notebook.json
new file mode 100644
index 0000000..ab673d8
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-notebook.json
@@ -0,0 +1,52 @@
+{
+  "contract_version": "1",
+  "gap_id": "1111111111111111",
+  "status": "resolved",
+  "baseline_report_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+  "source_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  "provider": {
+    "name": "airflow-to-dabs",
+    "version": "0.2.0",
+    "repository": "https://github.com/park-peter/airflow-to-dabs"
+  },
+  "model": {"name": "fixture-model"},
+  "argument_disposition": [
+    {
+      "name": "task_id",
+      "disposition": "preserved_by_flowx",
+      "rationale": "Flowx preserves the collision-safe task identity."
+    },
+    {
+      "name": "endpoint",
+      "disposition": "consumed",
+      "rationale": "The endpoint is embedded in the generated HTTP request."
+    },
+    {
+      "name": "method",
+      "disposition": "consumed",
+      "rationale": "The generated notebook issues the captured POST request."
+    },
+    {
+      "name": "data",
+      "disposition": "consumed",
+      "rationale": "The captured payload is passed as the JSON request body."
+    },
+    {
+      "name": "retries",
+      "disposition": "preserved_by_flowx",
+      "rationale": "Flowx preserves retry policy on the enclosing job task."
+    }
+  ],
+  "prerequisites": ["The Databricks task must have outbound network access to example.com."],
+  "warnings": [],
+  "semantic_deltas": ["The HTTP request runs in a Databricks notebook instead of an Airflow worker."],
+  "replacement": {"kind": "notebook", "file": "notify_orders.py", "base_parameters": {}},
+  "generated_files": [
+    {
+      "path": "notify_orders.py",
+      "language": "python",
+      "content": "# Databricks notebook source\nimport requests\n\nresponse = requests.post(\n    \"https://example.com/hooks/orders\",\n    json={\"event\": \"orders_ready\"},\n    timeout=30,\n)\nresponse.raise_for_status()\n",
+      "sha256": "07d382c58a610ecfabf55c0e3d2097988c4db1b1fd664d0e8cdc60dce9eaddec"
+    }
+  ]
+}
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-sql.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-sql.json
new file mode 100644
index 0000000..146b209
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-sql.json
@@ -0,0 +1,47 @@
+{
+  "contract_version": "1",
+  "gap_id": "2222222222222222",
+  "status": "resolved",
+  "baseline_report_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
+  "source_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
+  "provider": {
+    "name": "airflow-to-dabs",
+    "version": "0.2.0",
+    "repository": "https://github.com/park-peter/airflow-to-dabs"
+  },
+  "model": {"name": "fixture-model"},
+  "argument_disposition": [
+    {
+      "name": "task_id",
+      "disposition": "preserved_by_flowx",
+      "rationale": "Flowx preserves the collision-safe task identity."
+    },
+    {
+      "name": "conn_id",
+      "disposition": "consumed",
+      "rationale": "The connection identifies Databricks SQL as the execution target."
+    },
+    {
+      "name": "sql",
+      "disposition": "consumed",
+      "rationale": "The captured statement is emitted as the SQL file content."
+    },
+    {
+      "name": "autocommit",
+      "disposition": "ignored",
+      "rationale": "Databricks SQL task execution does not expose the Airflow autocommit toggle."
+    }
+  ],
+  "prerequisites": ["Configure the Flowx warehouse_id bundle variable for the target workspace."],
+  "warnings": ["Airflow autocommit behavior is not reproduced by the Databricks SQL task."],
+  "semantic_deltas": ["The statement runs on the configured Databricks SQL warehouse."],
+  "replacement": {"kind": "sql", "file": "cleanup_events.sql", "parameters": {}},
+  "generated_files": [
+    {
+      "path": "cleanup_events.sql",
+      "language": "sql",
+      "content": "DELETE FROM main.ops.events\nWHERE processed_at < current_date() - INTERVAL 30 DAYS\n",
+      "sha256": "89a5ec5ce7c02bfdffa57473908407ef620918b53840d8b8ce848179790546a7"
+    }
+  ]
+}
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/provider.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/provider.json
new file mode 100644
index 0000000..6c64a60
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/provider.json
@@ -0,0 +1,52 @@
+{
+  "profile_schema_version": "1",
+  "provider": {
+    "name": "airflow-to-dabs",
+    "version": "0.2.0",
+    "repository": "https://github.com/park-peter/airflow-to-dabs"
+  },
+  "interface": {
+    "name": "flowx-gap-resolver",
+    "source": "airflow",
+    "contract_versions": ["1"],
+    "entrypoint": "PROFILE.md",
+    "statuses": ["resolved", "needs_input", "deferred"],
+    "replacement_kinds": ["notebook", "sql"]
+  },
+  "knowledge": [
+    {
+      "path": "../../references/operator-mapping.md",
+      "purpose": "Operator intent, semantic mappings, and unsupported boundaries"
+    },
+    {
+      "path": "../../references/dab-schema-reference.md",
+      "purpose": "Notebook and SQL task runtime constraints"
+    },
+    {
+      "path": "../../references/schedule-trigger-mapping.md",
+      "purpose": "Trigger-rule, schedule, sensor, and template semantics"
+    },
+    {
+      "path": "../../references/airflow3-migration.md",
+      "purpose": "Airflow 3 operator and import-path semantics"
+    },
+    {
+      "path": "../../references/lakeflow-connect.md",
+      "purpose": "Detect resolutions that require resources outside the leaf-only contract"
+    },
+    {
+      "path": "../../references/hadoop-migration-guide.md",
+      "purpose": "Spark-submit, HDFS, Hive, and Hadoop semantic guidance"
+    }
+  ],
+  "fixtures": [
+    "fixtures/gap-notebook.json",
+    "fixtures/resolution-notebook.json",
+    "fixtures/gap-sql.json",
+    "fixtures/resolution-sql.json",
+    "fixtures/gap-needs-input.json",
+    "fixtures/resolution-needs-input.json",
+    "fixtures/gap-deferred.json",
+    "fixtures/resolution-deferred.json"
+  ]
+}
diff --git a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
index a773d71..5e76844 100644
--- a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
+++ b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
@@ -14,7 +14,7 @@ captured graph.
   "source_sha256": "copied from the envelope",
   "provider": {
     "name": "airflow-to-dabs",
-    "version": "0.1.0",
+    "version": "0.2.0",
     "repository": "https://github.com/park-peter/airflow-to-dabs"
   },
   "model": {"name": "model identifier"},
@@ -59,4 +59,4 @@ automatic retry occurs.
 - Airflow Jinja is rejected. Databricks dynamic references such as `{{job.parameters.x}}`,
   `{{tasks.upstream.values.x}}`, and `{{input}}` remain valid.
 - Every source argument in the envelope has exactly one disposition and a non-empty rationale.
-- The provider identity must match the pinned `airflow-to-dabs` v0.1.0 knowledge release.
+- The provider identity must match the pinned `airflow-to-dabs` v0.2.0 knowledge release.
diff --git a/src/flowx/agentic.py b/src/flowx/agentic.py
index 635b3ff..21dbb7c 100644
--- a/src/flowx/agentic.py
+++ b/src/flowx/agentic.py
@@ -24,7 +24,7 @@
 
 CONTRACT_VERSION = "1"
 PROVIDER_NAME = "airflow-to-dabs"
-PROVIDER_VERSION = "0.1.0"
+PROVIDER_VERSION = "0.2.0"
 PROVIDER_REPOSITORY = "https://github.com/park-peter/airflow-to-dabs"
 
 _ALLOWED_REPLACEMENT_KINDS = ("notebook", "sql")
@@ -128,6 +128,16 @@ class StagedResolution:
     sha256: str
 
 
+@dataclass(frozen=True, slots=True, kw_only=True)
+class PersistedResolutionEvidence:
+    """Validated kept evidence for one reviewed Airflow resolution run."""
+
+    provider_version: str
+    gaps: list[dict[str, Any]]
+    resolutions: list[StagedResolution]
+    expected_report: dict[str, Any]
+
+
 def prepare_airflow_resolutions(
     *,
     source_path: Path,
@@ -315,6 +325,40 @@ def apply_airflow_resolutions(
 
 def validate_persisted_agentic_report(report: dict[str, Any], *, evidence_dir: Path) -> list[str]:
     """Replays accepted candidates from kept evidence and compares the exact expected report."""
+    try:
+        evidence = _load_persisted_agentic_evidence(evidence_dir)
+    except AgenticContractError as error:
+        return [f"agentic resolution evidence is missing or invalid: {error}"]
+    if evidence.expected_report != report:
+        return ["agentic report does not match replay from its immutable baseline and accepted resolutions"]
+    return []
+
+
+def summarize_persisted_agentic_resolutions(evidence_dir: Path) -> dict[str, Any]:
+    """Returns validated per-pipeline outcomes from kept Airflow resolution evidence.
+
+    An absent evidence directory represents a deterministic-only run. Once evidence exists, every
+    file is contract- and hash-validated before any reporting metric may consume it.
+    """
+    if not evidence_dir.exists():
+        return {}
+    evidence = _load_persisted_agentic_evidence(evidence_dir)
+    pipeline_outcomes: dict[str, dict[str, int]] = {}
+    for gap in evidence.gaps:
+        pipeline = str(gap["pipeline_name"])
+        pipeline_outcomes.setdefault(pipeline, _empty_resolution_outcomes())["unreviewed"] += 1
+    for resolution in evidence.resolutions:
+        pipeline = str(resolution.gap["pipeline_name"])
+        outcomes = pipeline_outcomes.setdefault(pipeline, _empty_resolution_outcomes())
+        outcomes["unreviewed"] -= 1
+        outcomes[str(resolution.candidate["status"])] += 1
+    return {
+        "provider_version": evidence.provider_version,
+        "pipelines": pipeline_outcomes,
+    }
+
+
+def _load_persisted_agentic_evidence(evidence_dir: Path) -> PersistedResolutionEvidence:
     try:
         baseline_bytes = (evidence_dir / "baseline.json").read_bytes()
         baseline = json.loads(baseline_bytes)
@@ -323,11 +367,13 @@ def validate_persisted_agentic_report(report: dict[str, Any], *, evidence_dir: P
         manifest = _read_json_object(evidence_dir / "manifest.json")
         accepted = _read_json_object(evidence_dir / "accepted_resolutions.json")
     except (OSError, json.JSONDecodeError, AgenticContractError) as error:
-        return [f"agentic resolution evidence is missing or invalid: {error}"]
+        raise AgenticContractError(str(error)) from error
+
+    _require_airflow_baseline(baseline)
     if _sha256_bytes(baseline_bytes) != manifest.get("baseline_report_sha256"):
-        return ["agentic resolution baseline hash does not match its manifest"]
+        raise AgenticContractError("agentic resolution baseline hash does not match its manifest")
     if _sha256_bytes(gaps_bytes) != manifest.get("gaps_sha256"):
-        return ["agentic gap-envelope hash does not match its manifest"]
+        raise AgenticContractError("agentic gap-envelope hash does not match its manifest")
     expected_provider = {
         "name": PROVIDER_NAME,
         "version": PROVIDER_VERSION,
@@ -338,23 +384,67 @@ def validate_persisted_agentic_report(report: dict[str, Any], *, evidence_dir: P
         or manifest.get("source") != "airflow"
         or manifest.get("provider") != expected_provider
     ):
-        return ["agentic resolution manifest has an unsupported contract, source, or provider"]
-    gap_by_id = {
-        str(gap["gap_id"]): gap for gap in gaps if isinstance(gap, dict) and isinstance(gap.get("gap_id"), str)
-    }
+        raise AgenticContractError("agentic resolution manifest has an unsupported contract, source, or provider")
+    if accepted.get("contract_version") != CONTRACT_VERSION:
+        raise AgenticContractError("accepted_resolutions.json has an unsupported contract_version")
+    if not isinstance(gaps, list):
+        raise AgenticContractError("gaps.json must contain a list")
+
+    gap_by_id: dict[str, dict[str, Any]] = {}
+    for gap in gaps:
+        if not isinstance(gap, dict):
+            raise AgenticContractError("every persisted gap must be an object")
+        gap_id = gap.get("gap_id")
+        pipeline_name = gap.get("pipeline_name")
+        if not isinstance(gap_id, str) or not gap_id:
+            raise AgenticContractError("every persisted gap requires a non-empty gap_id")
+        if not isinstance(pipeline_name, str) or not pipeline_name:
+            raise AgenticContractError(f"persisted gap {gap_id!r} requires a non-empty pipeline_name")
+        if gap_id in gap_by_id:
+            raise AgenticContractError(f"duplicate persisted gap_id: {gap_id}")
+        gap_by_id[gap_id] = gap
+
+    try:
+        source_hashes = _manifest_source_hashes(manifest)
+    except (KeyError, TypeError) as error:
+        raise AgenticContractError("agentic resolution manifest has invalid source_files") from error
+    if not source_hashes:
+        raise AgenticContractError("agentic resolution manifest has no source_files")
+    expected_gaps = _build_gap_envelopes(
+        baseline,
+        baseline_hash=str(manifest["baseline_report_sha256"]),
+        source_hashes=source_hashes,
+    )
+    if gaps != expected_gaps:
+        raise AgenticContractError("persisted gap envelopes do not match the immutable baseline")
+
+    candidates = accepted.get("candidates")
+    if not isinstance(candidates, list):
+        raise AgenticContractError("accepted_resolutions.json must contain a candidates list")
     resolutions: list[StagedResolution] = []
+    accepted_ids: set[str] = set()
+    for candidate in candidates:
+        resolution = _validate_candidate(candidate, gap_by_id=gap_by_id, manifest=manifest)
+        gap_id = str(resolution.gap["gap_id"])
+        if gap_id in accepted_ids:
+            raise AgenticContractError(f"duplicate accepted resolution for gap_id: {gap_id}")
+        accepted_ids.add(gap_id)
+        resolutions.append(resolution)
+
     try:
-        candidates = accepted.get("candidates")
-        if not isinstance(candidates, list):
-            raise AgenticContractError("accepted_resolutions.json must contain a candidates list")
-        for candidate in candidates:
-            resolutions.append(_validate_candidate(candidate, gap_by_id=gap_by_id, manifest=manifest))
-        expected = _apply_to_baseline(baseline, resolutions)
+        expected_report = _apply_to_baseline(baseline, resolutions)
     except AgenticContractError as error:
-        return [f"agentic resolution evidence failed validation: {error}"]
-    if expected != report:
-        return ["agentic report does not match replay from its immutable baseline and accepted resolutions"]
-    return []
+        raise AgenticContractError(f"agentic resolution evidence failed validation: {error}") from error
+    return PersistedResolutionEvidence(
+        provider_version=PROVIDER_VERSION,
+        gaps=gaps,
+        resolutions=resolutions,
+        expected_report=expected_report,
+    )
+
+
+def _empty_resolution_outcomes() -> dict[str, int]:
+    return {"resolved": 0, "needs_input": 0, "deferred": 0, "unreviewed": 0}
 
 
 def _require_airflow_baseline(payload: Any) -> None:
diff --git a/src/flowx/reporting/coverage.py b/src/flowx/reporting/coverage.py
index ca41d23..5b0dcf9 100644
--- a/src/flowx/reporting/coverage.py
+++ b/src/flowx/reporting/coverage.py
@@ -18,6 +18,8 @@
 from pathlib import Path
 from typing import Any
 
+from flowx.agentic import AgenticContractError, summarize_persisted_agentic_resolutions
+
 # Metric columns (order matters: it drives the results-table column order).
 COVERAGE_METRIC_COLUMNS: tuple[str, ...] = (
     "pipeline",
@@ -31,6 +33,9 @@
     "other_activities",
     "deterministic_activities",
     "agentic_activities",
+    "unresolved_agentic_activities",
+    "agentic_resolution_outcomes",
+    "agentic_provider_version",
     "unsupported_activities",
     "failed_activities",
     "excluded_activities",
@@ -38,6 +43,7 @@
     "migration_status",
     "coverage_pct",
     "deterministic_coverage_pct",
+    "runnable_coverage_pct",
     "finding_count",
     "finding_fingerprints",
     "complexity_score",
@@ -69,6 +75,13 @@ def _deterministic_coverage_pct(deterministic: int, total: int) -> float:
     return round(deterministic / total * 100, 1)
 
 
+def _runnable_coverage_pct(deterministic: int, resolved: int, total: int) -> float:
+    """Mechanically code-attached coverage over audited activity candidates."""
+    if total <= 0:
+        return 0.0
+    return round((deterministic + resolved) / total * 100, 1)
+
+
 def build_coverage_rows(metadata_dir: Path) -> list[dict[str, Any]]:
     """Builds per-pipeline coverage rows from a migration ``metadata/`` directory.
 
@@ -86,6 +99,20 @@ def build_coverage_rows(metadata_dir: Path) -> list[dict[str, Any]]:
     """
     inventory_path = metadata_dir / "inventory.json"
     inventory = json.loads(inventory_path.read_text(encoding="utf-8"))
+    is_airflow = inventory.get("source") == "airflow"
+    agentic_summary = summarize_persisted_agentic_resolutions(metadata_dir / "agentic") if is_airflow else {}
+    provider_version = str(agentic_summary.get("provider_version", ""))
+    resolution_pipelines = agentic_summary.get("pipelines", {})
+    if not isinstance(resolution_pipelines, dict):
+        raise AgenticContractError("agentic resolution summary pipelines must be an object")
+    inventory_names = {
+        str(pipeline.get("name", "")) for pipeline in inventory.get("pipelines", []) if isinstance(pipeline, dict)
+    }
+    unknown_pipelines = sorted(set(resolution_pipelines) - inventory_names)
+    if unknown_pipelines:
+        raise AgenticContractError(
+            "agentic resolution evidence references unknown inventory pipeline(s): " + ", ".join(unknown_pipelines)
+        )
 
     csv_by_pipeline: dict[str, dict[str, str]] = {}
     csv_path = metadata_dir / "profile_report.csv"
@@ -105,6 +132,31 @@ def build_coverage_rows(metadata_dir: Path) -> list[dict[str, Any]]:
         failed = int(pipeline.get("failed_count", 0)) if has_audit else 0
         excluded = int(pipeline.get("excluded_count", 0)) if has_audit else 0
         total = int(pipeline.get("audited_activity_count", 0)) if has_audit else len(strategies)
+        if is_airflow:
+            if agentic_summary:
+                outcomes = resolution_pipelines.get(
+                    name,
+                    {"resolved": 0, "needs_input": 0, "deferred": 0, "unreviewed": 0},
+                )
+                if not isinstance(outcomes, dict) or any(
+                    not isinstance(outcomes.get(key), int)
+                    for key in ("resolved", "needs_input", "deferred", "unreviewed")
+                ):
+                    raise AgenticContractError(f"invalid agentic resolution outcomes for pipeline {name!r}")
+                if sum(outcomes.values()) != agentic:
+                    raise AgenticContractError(
+                        f"agentic resolution evidence accounts for {sum(outcomes.values())} of "
+                        f"{agentic} agentic activities in pipeline {name!r}"
+                    )
+            else:
+                outcomes = {"resolved": 0, "needs_input": 0, "deferred": 0, "unreviewed": agentic}
+            resolved_agentic = outcomes["resolved"]
+            unresolved_agentic = agentic - resolved_agentic
+            runnable_coverage = _runnable_coverage_pct(deterministic, resolved_agentic, total)
+        else:
+            outcomes = {}
+            unresolved_agentic = 0
+            runnable_coverage = _coverage_pct(deterministic, agentic, total)
         findings = pipeline.get("findings", [])
         fingerprints = [
             finding["fingerprint"]
@@ -132,13 +184,21 @@ def _csv_int(col: str, _csv_row: dict[str, str] = csv_row) -> int:
                 "other_activities": _csv_int("other_activities"),
                 "deterministic_activities": deterministic,
                 "agentic_activities": agentic,
+                "unresolved_agentic_activities": unresolved_agentic,
+                "agentic_resolution_outcomes": json.dumps(outcomes, sort_keys=True, separators=(",", ":")),
+                "agentic_provider_version": provider_version if is_airflow else "",
                 "unsupported_activities": unsupported,
                 "failed_activities": failed,
                 "excluded_activities": excluded,
-                "reconciliation_status": pipeline.get("reconciliation_status", "not_applicable"),
+                "reconciliation_status": (
+                    "verified_with_reviewed_resolutions"
+                    if is_airflow and outcomes.get("resolved", 0) > 0
+                    else pipeline.get("reconciliation_status", "not_applicable")
+                ),
                 "migration_status": pipeline.get("migration_status", "included"),
                 "coverage_pct": _coverage_pct(deterministic, agentic, total),
                 "deterministic_coverage_pct": _deterministic_coverage_pct(deterministic, total),
+                "runnable_coverage_pct": runnable_coverage,
                 "finding_count": len(findings),
                 "finding_fingerprints": json.dumps(fingerprints, separators=(",", ":")),
                 "complexity_score": _csv_int("complexity_score"),
diff --git a/src/flowx/reporting/dashboard_template.json b/src/flowx/reporting/dashboard_template.json
index d6ca7ee..39f3f72 100644
--- a/src/flowx/reporting/dashboard_template.json
+++ b/src/flowx/reporting/dashboard_template.json
@@ -8,11 +8,13 @@
         "SUM(audited_activities) AS audited_activities, ",
         "SUM(deterministic_activities) AS deterministic_activities, ",
         "SUM(agentic_activities) AS agentic_activities, ",
+        "SUM(unresolved_agentic_activities) AS unresolved_agentic_activities, ",
         "SUM(unsupported_activities) AS unsupported_activities, ",
         "SUM(failed_activities) AS failed_activities, ",
         "SUM(excluded_activities) AS excluded_activities, ",
         "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(audited_activities),0),1) AS coverage_pct, ",
-        "ROUND(100.0*SUM(deterministic_activities)/NULLIF(SUM(audited_activities),0),1) AS deterministic_coverage_pct ",
+        "ROUND(100.0*SUM(deterministic_activities)/NULLIF(SUM(audited_activities),0),1) AS deterministic_coverage_pct, ",
+        "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities)-SUM(unresolved_agentic_activities))/NULLIF(SUM(audited_activities),0),1) AS runnable_coverage_pct ",
         "FROM {{RESULTS_TABLE}} ",
         "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1)"
       ]
@@ -31,12 +33,13 @@
       "name": "latest_pipelines",
       "displayName": "Pipeline coverage (latest run)",
       "queryLines": [
-        "SELECT pipeline, audited_activities, deterministic_activities, agentic_activities, ",
+        "SELECT pipeline, audited_activities, deterministic_activities, agentic_activities, unresolved_agentic_activities, ",
         "unsupported_activities, failed_activities, excluded_activities, reconciliation_status, ",
-        "migration_status, coverage_pct, deterministic_coverage_pct, finding_count, collapsible_patterns, complexity_size ",
+        "migration_status, coverage_pct, deterministic_coverage_pct, runnable_coverage_pct, ",
+        "agentic_resolution_outcomes, agentic_provider_version, finding_count, collapsible_patterns, complexity_size ",
         "FROM {{RESULTS_TABLE}} ",
         "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1) ",
-        "ORDER BY coverage_pct ASC, audited_activities DESC"
+        "ORDER BY runnable_coverage_pct ASC, audited_activities DESC"
       ]
     },
     {
@@ -46,6 +49,7 @@
         "SELECT DATE_TRUNC('SECOND', run_date) AS run_ts, ",
         "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(audited_activities),0),1) AS coverage_pct, ",
         "ROUND(100.0*SUM(deterministic_activities)/NULLIF(SUM(audited_activities),0),1) AS deterministic_coverage_pct, ",
+        "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities)-SUM(unresolved_agentic_activities))/NULLIF(SUM(audited_activities),0),1) AS runnable_coverage_pct, ",
         "SUM(audited_activities) AS audited_activities, SUM(failed_activities) AS failed_activities, ",
         "SUM(excluded_activities) AS excluded_activities ",
         "FROM {{RESULTS_TABLE}} ",
@@ -81,7 +85,7 @@
             "name": "subtitle",
             "multilineTextboxSpec": {
               "lines": [
-                "Per-pipeline translation coverage from the latest flowx run. Coverage % = (deterministic + agentic) / audited activities."
+                "Code-attached coverage counts deterministic and reviewed provider code that passed mechanical validation. It does not certify semantic correctness."
               ]
             }
           },
@@ -142,8 +146,8 @@
                   "datasetName": "latest_summary",
                   "fields": [
                     {
-                      "name": "coverage_pct",
-                      "expression": "`coverage_pct`"
+                      "name": "runnable_coverage_pct",
+                      "expression": "`runnable_coverage_pct`"
                     }
                   ],
                   "disaggregated": true
@@ -155,12 +159,12 @@
               "widgetType": "counter",
               "encodings": {
                 "value": {
-                  "fieldName": "coverage_pct",
-                  "displayName": "Coverage %"
+                  "fieldName": "runnable_coverage_pct",
+                  "displayName": "Code-attached %"
                 }
               },
               "frame": {
-                "title": "Coverage %",
+                "title": "Code-attached % (mechanically validated)",
                 "showTitle": true
               }
             }
@@ -400,8 +404,8 @@
                       "expression": "`run_ts`"
                     },
                     {
-                      "name": "coverage_pct",
-                      "expression": "`coverage_pct`"
+                      "name": "runnable_coverage_pct",
+                      "expression": "`runnable_coverage_pct`"
                     }
                   ],
                   "disaggregated": true
@@ -420,15 +424,15 @@
                   "displayName": "Run"
                 },
                 "y": {
-                  "fieldName": "coverage_pct",
+                  "fieldName": "runnable_coverage_pct",
                   "scale": {
                     "type": "quantitative"
                   },
-                  "displayName": "Coverage %"
+                  "displayName": "Code-attached %"
                 }
               },
               "frame": {
-                "title": "Coverage over runs",
+                "title": "Code-attached coverage over runs (mechanically validated)",
                 "showTitle": true
               }
             }
@@ -465,6 +469,10 @@
                       "name": "agentic_activities",
                       "expression": "`agentic_activities`"
                     },
+                    {
+                      "name": "unresolved_agentic_activities",
+                      "expression": "`unresolved_agentic_activities`"
+                    },
                     {
                       "name": "failed_activities",
                       "expression": "`failed_activities`"
@@ -485,6 +493,18 @@
                       "name": "coverage_pct",
                       "expression": "`coverage_pct`"
                     },
+                    {
+                      "name": "runnable_coverage_pct",
+                      "expression": "`runnable_coverage_pct`"
+                    },
+                    {
+                      "name": "agentic_resolution_outcomes",
+                      "expression": "`agentic_resolution_outcomes`"
+                    },
+                    {
+                      "name": "agentic_provider_version",
+                      "expression": "`agentic_provider_version`"
+                    },
                     {
                       "name": "collapsible_patterns",
                       "expression": "`collapsible_patterns`"
@@ -519,6 +539,10 @@
                     "fieldName": "agentic_activities",
                     "displayName": "Agentic"
                   },
+                  {
+                    "fieldName": "unresolved_agentic_activities",
+                    "displayName": "Unresolved agentic"
+                  },
                   {
                     "fieldName": "failed_activities",
                     "displayName": "Failed"
@@ -537,7 +561,19 @@
                   },
                   {
                     "fieldName": "coverage_pct",
-                    "displayName": "Coverage %"
+                    "displayName": "Translation path %"
+                  },
+                  {
+                    "fieldName": "runnable_coverage_pct",
+                    "displayName": "Code-attached %"
+                  },
+                  {
+                    "fieldName": "agentic_resolution_outcomes",
+                    "displayName": "Agentic outcomes"
+                  },
+                  {
+                    "fieldName": "agentic_provider_version",
+                    "displayName": "Provider version"
                   },
                   {
                     "fieldName": "collapsible_patterns",
diff --git a/src/flowx/reporting/results.py b/src/flowx/reporting/results.py
index f9d7d4a..294ab62 100644
--- a/src/flowx/reporting/results.py
+++ b/src/flowx/reporting/results.py
@@ -32,6 +32,9 @@
     "other_activities": "INT",
     "deterministic_activities": "INT",
     "agentic_activities": "INT",
+    "unresolved_agentic_activities": "INT",
+    "agentic_resolution_outcomes": "STRING",
+    "agentic_provider_version": "STRING",
     "unsupported_activities": "INT",
     "failed_activities": "INT",
     "excluded_activities": "INT",
@@ -39,6 +42,7 @@
     "migration_status": "STRING",
     "coverage_pct": "DOUBLE",
     "deterministic_coverage_pct": "DOUBLE",
+    "runnable_coverage_pct": "DOUBLE",
     "finding_count": "INT",
     "finding_fingerprints": "STRING",
     "complexity_score": "INT",
@@ -53,9 +57,17 @@
 )
 
 _STRING_METRICS: frozenset[str] = frozenset(
-    {"pipeline", "reconciliation_status", "migration_status", "finding_fingerprints", "complexity_size"}
+    {
+        "pipeline",
+        "agentic_resolution_outcomes",
+        "agentic_provider_version",
+        "reconciliation_status",
+        "migration_status",
+        "finding_fingerprints",
+        "complexity_size",
+    }
 )
-_FLOAT_METRICS: frozenset[str] = frozenset({"coverage_pct", "deterministic_coverage_pct"})
+_FLOAT_METRICS: frozenset[str] = frozenset({"coverage_pct", "deterministic_coverage_pct", "runnable_coverage_pct"})
 
 
 def _sql_str(value: Any) -> str:
diff --git a/tests/unit/test_airflow_agentic_resolution.py b/tests/unit/test_airflow_agentic_resolution.py
index 8c7ea9f..1f96285 100644
--- a/tests/unit/test_airflow_agentic_resolution.py
+++ b/tests/unit/test_airflow_agentic_resolution.py
@@ -9,7 +9,9 @@
 import pytest
 
 from flowx.adapter.__main__ import main as adapter_main
+from flowx.agentic import AgenticContractError, _validate_candidate, summarize_persisted_agentic_resolutions
 from flowx.bundler.dab_writer import main as package_main
+from flowx.reporting.coverage import build_coverage_rows
 from flowx.sources.airflow.convert import main as airflow_convert
 
 
@@ -82,7 +84,7 @@ def _candidate(gap: dict, *, source: str = "print('Migrated from Airflow')\n", s
         "source_sha256": gap["source_sha256"],
         "provider": {
             "name": "airflow-to-dabs",
-            "version": "0.1.0",
+            "version": "0.2.0",
             "repository": "https://github.com/park-peter/airflow-to-dabs",
         },
         "model": {"name": "test-model"},
@@ -205,9 +207,9 @@ def test_stage_rejects_unresolved_jinja_provider_drift_and_file_hash_mismatch(tm
     assert "unresolved Airflow Jinja" in capsys.readouterr().err
 
     wrong_provider = _candidate(gaps[0])
-    wrong_provider["provider"]["version"] = "0.2.0"
+    wrong_provider["provider"]["version"] = "0.1.0"
     assert _stage(output, wrong_provider) == 1
-    assert "pinned airflow-to-dabs v0.1.0" in capsys.readouterr().err
+    assert "pinned airflow-to-dabs v0.2.0" in capsys.readouterr().err
 
     bad_hash = _candidate(gaps[0])
     bad_hash["generated_files"][0]["sha256"] = "0" * 64
@@ -215,6 +217,21 @@ def test_stage_rejects_unresolved_jinja_provider_drift_and_file_hash_mismatch(tm
     assert "sha256 does not match" in capsys.readouterr().err
 
 
+def test_pinned_v020_provider_fixtures_satisfy_the_flowx_contract() -> None:
+    root = Path(__file__).parents[2] / "skills" / "flowx-resolve-airflow-gaps" / "references" / "airflow-to-dabs-v0.2.0"
+    provider = json.loads((root / "provider.json").read_text(encoding="utf-8"))
+
+    assert provider["provider"]["version"] == "0.2.0"
+    for outcome in ("notebook", "sql", "needs-input", "deferred"):
+        gap = json.loads((root / "fixtures" / f"gap-{outcome}.json").read_text(encoding="utf-8"))
+        candidate = json.loads((root / "fixtures" / f"resolution-{outcome}.json").read_text(encoding="utf-8"))
+        manifest = {"baseline_report_sha256": gap["baseline_report_sha256"]}
+
+        resolution = _validate_candidate(candidate, gap_by_id={gap["gap_id"]: gap}, manifest=manifest)
+
+        assert resolution.gap["gap_id"] == gap["gap_id"]
+
+
 def test_apply_rebuilds_from_baseline_and_preserves_graph_policy(tmp_path: Path):
     _, output, gaps = _prepare(tmp_path)
     assert _stage(output, _candidate(gaps[0])) == 0
@@ -563,3 +580,94 @@ def test_package_rejects_agentic_report_tampering_before_bundle_writes(tmp_path:
 
     assert package_main(["--report", str(report), "--output-dir", str(bundle)]) == 1
     assert not (bundle / "databricks.yml").exists()
+
+
+def test_reviewed_resolution_evidence_drives_honest_runnable_coverage(tmp_path: Path) -> None:
+    _, output, gaps = _prepare(tmp_path, two_tasks=True)
+    assert _stage(output, _candidate(gaps[0]), name="resolved.json") == 0
+    assert _stage(output, _candidate(gaps[1], status="needs_input"), name="needs-input.json") == 0
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+                "--accept-gap",
+                gaps[1]["gap_id"],
+            ]
+        )
+        == 0
+    )
+    metadata = output / "metadata"
+    (metadata / "inventory.json").write_text(
+        json.dumps(
+            {
+                "source": "airflow",
+                "pipelines": [
+                    {
+                        "name": "agentic",
+                        "activities": [],
+                        "audited_activity_count": 2,
+                        "deterministic_count": 0,
+                        "agentic_count": 2,
+                        "failed_count": 0,
+                        "excluded_count": 0,
+                        "reconciliation_status": "verified_with_gaps",
+                        "migration_status": "included",
+                        "findings": [],
+                    }
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    summary = summarize_persisted_agentic_resolutions(metadata / "agentic")
+    row = build_coverage_rows(metadata)[0]
+
+    assert summary == {
+        "provider_version": "0.2.0",
+        "pipelines": {"agentic": {"resolved": 1, "needs_input": 1, "deferred": 0, "unreviewed": 0}},
+    }
+    assert row["coverage_pct"] == 100.0
+    assert row["deterministic_coverage_pct"] == 0.0
+    assert row["runnable_coverage_pct"] == 50.0
+    assert row["unresolved_agentic_activities"] == 1
+    assert row["agentic_provider_version"] == "0.2.0"
+    assert row["reconciliation_status"] == "verified_with_reviewed_resolutions"
+
+
+def test_reporting_rejects_duplicate_hash_valid_agentic_evidence(tmp_path: Path) -> None:
+    _, output, gaps = _prepare(tmp_path)
+    assert _stage(output, _candidate(gaps[0])) == 0
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+    evidence = output / "metadata" / "agentic"
+    duplicated_gaps = [gaps[0], gaps[0]]
+    gaps_bytes = (json.dumps(duplicated_gaps, sort_keys=True, separators=(",", ":")) + "\n").encode()
+    (evidence / "gaps.json").write_bytes(gaps_bytes)
+    manifest_path = evidence / "manifest.json"
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    manifest["gaps_sha256"] = hashlib.sha256(gaps_bytes).hexdigest()
+    manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
+
+    with pytest.raises(AgenticContractError, match="duplicate persisted gap_id"):
+        summarize_persisted_agentic_resolutions(evidence)
diff --git a/tests/unit/test_reporting_coverage.py b/tests/unit/test_reporting_coverage.py
index a9b6237..906c1d0 100644
--- a/tests/unit/test_reporting_coverage.py
+++ b/tests/unit/test_reporting_coverage.py
@@ -76,6 +76,9 @@ def test_build_coverage_rows_joins_inventory_and_csv(tmp_path: Path):
     assert alpha["unsupported_activities"] == 1
     # coverage = (det + agentic) / total = 3/4 = 75.0
     assert alpha["coverage_pct"] == 75.0
+    assert alpha["runnable_coverage_pct"] == 75.0
+    assert alpha["unresolved_agentic_activities"] == 0
+    assert alpha["agentic_resolution_outcomes"] == "{}"
     # complexity columns come from the CSV
     assert alpha["datasets"] == 2 and alpha["linked_services"] == 1
     assert alpha["collapsible_patterns"] == 1 and alpha["complexity_size"] == "M"
@@ -94,6 +97,7 @@ def test_audited_counts_drive_translation_and_deterministic_coverage(tmp_path: P
     metadata = tmp_path / "metadata"
     metadata.mkdir()
     inventory = {
+        "source": "airflow",
         "pipelines": [
             {
                 "name": "verified_with_gap",
@@ -119,7 +123,7 @@ def test_audited_counts_drive_translation_and_deterministic_coverage(tmp_path: P
                 "migration_status": "included",
                 "findings": [{"fingerprint": "def456", "severity": "failed"}],
             },
-        ]
+        ],
     }
     (metadata / "inventory.json").write_text(json.dumps(inventory), encoding="utf-8")
 
@@ -130,6 +134,14 @@ def test_audited_counts_drive_translation_and_deterministic_coverage(tmp_path: P
     assert verified["audited_activities"] == 8
     assert verified["coverage_pct"] == 100.0
     assert verified["deterministic_coverage_pct"] == 87.5
+    assert verified["runnable_coverage_pct"] == 87.5
+    assert verified["unresolved_agentic_activities"] == 1
+    assert json.loads(verified["agentic_resolution_outcomes"]) == {
+        "resolved": 0,
+        "needs_input": 0,
+        "deferred": 0,
+        "unreviewed": 1,
+    }
     assert verified["finding_count"] == 1
     assert json.loads(verified["finding_fingerprints"]) == ["abc123"]
 
@@ -138,6 +150,7 @@ def test_audited_counts_drive_translation_and_deterministic_coverage(tmp_path: P
     assert failed["failed_activities"] == 1
     assert failed["coverage_pct"] == 88.9
     assert failed["deterministic_coverage_pct"] == 77.8
+    assert failed["runnable_coverage_pct"] == 77.8
     assert failed["reconciliation_status"] == "failed"
 
 
@@ -145,6 +158,7 @@ def test_excluded_activities_remain_in_coverage_denominator(tmp_path: Path) -> N
     metadata = tmp_path / "metadata"
     metadata.mkdir()
     inventory = {
+        "source": "airflow",
         "pipelines": [
             {
                 "name": "excluded",
@@ -158,7 +172,7 @@ def test_excluded_activities_remain_in_coverage_denominator(tmp_path: Path) -> N
                 "migration_status": "excluded",
                 "findings": [],
             }
-        ]
+        ],
     }
     (metadata / "inventory.json").write_text(json.dumps(inventory), encoding="utf-8")
 
@@ -168,4 +182,5 @@ def test_excluded_activities_remain_in_coverage_denominator(tmp_path: Path) -> N
     assert row["excluded_activities"] == 3
     assert row["coverage_pct"] == 0.0
     assert row["deterministic_coverage_pct"] == 0.0
+    assert row["runnable_coverage_pct"] == 0.0
     assert row["migration_status"] == "excluded"
diff --git a/tests/unit/test_reporting_dashboard.py b/tests/unit/test_reporting_dashboard.py
index 4b7ce17..cf80aa2 100644
--- a/tests/unit/test_reporting_dashboard.py
+++ b/tests/unit/test_reporting_dashboard.py
@@ -5,6 +5,7 @@
 import json
 
 import pytest
+import sqlglot
 
 from flowx.reporting import dashboard as D
 
@@ -21,10 +22,26 @@ def test_build_serialized_dashboard_injects_table_and_is_valid_json():
     assert "excluded_activities" in joined
     assert "reconciliation_status" in joined
     assert "deterministic_coverage_pct" in joined
+    assert "runnable_coverage_pct" in joined
+    assert "unresolved_agentic_activities" in joined
+    assert "agentic_resolution_outcomes" in joined
+    assert "agentic_provider_version" in joined
     assert spec["pages"][0]["pageType"] == "PAGE_TYPE_CANVAS"
     # widget field names match their dataset fields (counter references a real column)
     widget_names = {w["widget"]["name"] for w in spec["pages"][0]["layout"]}
     assert {"kpi-coverage", "by-size", "coverage-trend", "pipeline-table"} <= widget_names
+    assert "mechanically validated" in serialized
+
+    dataset_fields = {}
+    for dataset in spec["datasets"]:
+        query = "".join(dataset["queryLines"])
+        expression = sqlglot.parse_one(query, dialect="databricks")
+        dataset_fields[dataset["name"]] = {projection.alias_or_name for projection in expression.expressions}
+    for layout_item in spec["pages"][0]["layout"]:
+        for query in layout_item["widget"].get("queries", []):
+            dataset_name = query["query"]["datasetName"]
+            for field in query["query"]["fields"]:
+                assert field["name"] in dataset_fields[dataset_name]
 
 
 def test_build_serialized_dashboard_requires_table():
diff --git a/tests/unit/test_reporting_results.py b/tests/unit/test_reporting_results.py
index 7e39655..5a9ca00 100644
--- a/tests/unit/test_reporting_results.py
+++ b/tests/unit/test_reporting_results.py
@@ -24,6 +24,10 @@ def test_create_table_sql_has_run_metadata_and_all_columns():
     assert "excluded_activities INT" in sql
     assert "reconciliation_status STRING" in sql
     assert "deterministic_coverage_pct DOUBLE" in sql
+    assert "runnable_coverage_pct DOUBLE" in sql
+    assert "unresolved_agentic_activities INT" in sql
+    assert "agentic_resolution_outcomes STRING" in sql
+    assert "agentic_provider_version STRING" in sql
     assert "finding_fingerprints STRING" in sql
     assert "complexity_size STRING" in sql
 
@@ -37,6 +41,7 @@ def test_schema_evolution_sql_adds_only_missing_metric_columns() -> None:
     assert sql.startswith("ALTER TABLE cat.sch.tbl ADD COLUMNS")
     assert "audited_activities INT" in sql
     assert "deterministic_coverage_pct DOUBLE" in sql
+    assert "runnable_coverage_pct DOUBLE" in sql
     assert "pipeline STRING" not in sql
     assert "\n  coverage_pct DOUBLE" not in sql
 
@@ -55,6 +60,9 @@ def test_insert_sql_stamps_run_metadata_and_escapes():
             "other_activities": 2,
             "deterministic_activities": 2,
             "agentic_activities": 1,
+            "unresolved_agentic_activities": 1,
+            "agentic_resolution_outcomes": '{"unreviewed":1}',
+            "agentic_provider_version": "0.2.0",
             "unsupported_activities": 0,
             "failed_activities": 0,
             "excluded_activities": 0,
@@ -62,6 +70,7 @@ def test_insert_sql_stamps_run_metadata_and_escapes():
             "migration_status": "included",
             "coverage_pct": 100.0,
             "deterministic_coverage_pct": 66.7,
+            "runnable_coverage_pct": 66.7,
             "finding_count": 1,
             "finding_fingerprints": '["abc"]',
             "complexity_score": 7,
@@ -79,6 +88,9 @@ def test_insert_sql_stamps_run_metadata_and_escapes():
             "other_activities": 1,
             "deterministic_activities": 0,
             "agentic_activities": 0,
+            "unresolved_agentic_activities": 0,
+            "agentic_resolution_outcomes": "{}",
+            "agentic_provider_version": "",
             "unsupported_activities": 1,
             "failed_activities": 0,
             "excluded_activities": 0,
@@ -86,6 +98,7 @@ def test_insert_sql_stamps_run_metadata_and_escapes():
             "migration_status": "included",
             "coverage_pct": 0.0,
             "deterministic_coverage_pct": 0.0,
+            "runnable_coverage_pct": 0.0,
             "finding_count": 0,
             "finding_fingerprints": "[]",
             "complexity_score": 3,
@@ -105,6 +118,7 @@ def test_insert_sql_stamps_run_metadata_and_escapes():
     assert "100.0" in sql
     assert "'verified_with_gaps'" in sql
     assert "'[\"abc\"]'" in sql
+    assert "'{\"unreviewed\":1}'" in sql
 
 
 class _FakeWarehouse:
@@ -255,4 +269,6 @@ def test_write_results_evolves_an_existing_legacy_schema_before_insert(tmp_path:
     assert statements[2].startswith("ALTER TABLE cat.sch.tbl ADD COLUMNS")
     assert "audited_activities INT" in statements[2]
     assert "deterministic_coverage_pct DOUBLE" in statements[2]
+    assert "runnable_coverage_pct DOUBLE" in statements[2]
+    assert "agentic_resolution_outcomes STRING" in statements[2]
     assert statements[3].startswith("INSERT INTO cat.sch.tbl")

From d9ea3af0d69b92729ed759444d2708912963a423 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Sat, 8 Aug 2026 23:52:12 -0700
Subject: [PATCH 57/77] Bind Airflow gaps to captured source spans

---
 src/flowx/sources/airflow/loader.py           | 142 +++++++++++++-----
 tests/unit/test_airflow_agentic_resolution.py | 105 +++++++++++++
 2 files changed, 206 insertions(+), 41 deletions(-)

diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py
index b9da56c..dfbd5ef 100644
--- a/src/flowx/sources/airflow/loader.py
+++ b/src/flowx/sources/airflow/loader.py
@@ -24,7 +24,6 @@
 import copy
 import json
 import re
-from collections import Counter
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
@@ -579,6 +578,7 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None = None
         self._list_bindings: dict[str, list[str]] = {}
         self._capture_sequence = 0
         self.task_captures: dict[str, TaskCapture] = {}
+        self.capture_source_nodes: dict[str, ast.Call] = {}
         self.edge_captures: list[EdgeCapture] = []
         self.unclaimed_task_calls: list[ast.Call] = []
         self.unclaimed_statements: list[ast.stmt] = []
@@ -789,6 +789,7 @@ def _register_operator_call(self, node: ast.Call, var: str, *, binding: str | No
             call=call,
             span=_span(node),
         )
+        self.capture_source_nodes[var] = node
         self._claimed_task_call_ids.add(id(node))
         callable_node = kwargs.get("python_callable")
         if isinstance(callable_node, ast.Name):
@@ -850,6 +851,7 @@ def _register_helper_factory_call(self, node: ast.Call, var: str, *, binding: st
             factory_call, var, binding=binding
         )
         if registered:
+            self.capture_source_nodes[var] = node
             self._claimed_task_call_ids.add(id(node))
             self.helper_expansions.append(
                 {
@@ -963,6 +965,7 @@ def _register_taskflow_call(self, call: ast.Call, var: str) -> bool:
         task = _TaskFlowTask(task_id=override_id or var, def_name=def_name, decorator=decorator)
         self.taskflow_tasks[var] = task
         self.calls[var] = call
+        self.capture_source_nodes[var] = call
         self._claimed_task_call_ids.add(id(call))
         if mapped:
             self.mapped.add(var)
@@ -1056,6 +1059,7 @@ def _register_taskgroup_call(self, call: ast.Call, var: str | None) -> bool:
             self._taskgroup_counter += 1
             var = f"{def_name}__tg{self._taskgroup_counter}"
         self.taskgroup_calls[var] = (var, def_name, mapped)
+        self.capture_source_nodes[var] = call
         self._claimed_task_call_ids.add(id(call))
         if self._group_stack:
             self.groups[var] = "__".join(self._group_stack)
@@ -2596,20 +2600,35 @@ def _reconcile_pipeline(
                 )
             )
 
-    helper_claims = {
-        ("helper_factory_task", int(item["invocation_line"]), str(item["helper"])): 1
-        for item in visitor.helper_expansions
-    }
     helper_capture_ids = {str(item["capture_id"]) for item in visitor.helper_expansions}
-    capture_claims: Counter[tuple[str, int, str]] = Counter(helper_claims)
+    capture_claims: dict[tuple[str, int, int, int, int, str], list[str]] = {}
+
+    def add_capture_claim(code: str, node: ast.AST, discriminator: str, capture_id: str) -> None:
+        span = _span(node)
+        key = (code, span.line, span.column, span.end_line, span.end_column, discriminator)
+        capture_claims.setdefault(key, []).append(capture_id)
+
+    for item in visitor.helper_expansions:
+        capture_id = str(item["capture_id"])
+        add_capture_claim(
+            "helper_factory_task",
+            visitor.capture_source_nodes[capture_id],
+            str(item["helper"]),
+            capture_id,
+        )
     for capture in visitor.task_captures.values():
         if capture.capture_id not in helper_capture_ids:
-            capture_claims[("operator_task", capture.span.line, capture.operator)] += 1
+            add_capture_claim(
+                "operator_task",
+                visitor.capture_source_nodes[capture.capture_id],
+                capture.operator,
+                capture.capture_id,
+            )
     for var, taskflow_task in visitor.taskflow_tasks.items():
-        call = visitor.calls.get(var)
-        capture_claims[("taskflow_task", getattr(call, "lineno", 0), taskflow_task.def_name)] += 1
+        add_capture_claim("taskflow_task", visitor.capture_source_nodes[var], taskflow_task.def_name, var)
 
     unmatched_audit_tasks: list[source_audit.AuditCandidate] = []
+    audit_candidate_by_capture: dict[str, source_audit.AuditCandidate] = {}
     for candidate in audit.tasks:
         discriminator = str(
             candidate.details.get("operator")
@@ -2617,11 +2636,19 @@ def _reconcile_pipeline(
             or candidate.details.get("callable")
             or ""
         )
-        key = (candidate.code, candidate.line, discriminator)
-        if capture_claims[key]:
-            capture_claims[key] -= 1
-        else:
+        key = (
+            candidate.code,
+            candidate.line,
+            candidate.column,
+            candidate.end_line,
+            candidate.end_column,
+            discriminator,
+        )
+        capture_ids = capture_claims.get(key)
+        if not capture_ids:
             unmatched_audit_tasks.append(candidate)
+            continue
+        audit_candidate_by_capture[capture_ids.pop(0)] = candidate
 
     for candidate in unmatched_audit_tasks:
         findings.append(
@@ -2752,21 +2779,12 @@ def source_reference(capture_id: str) -> str:
             )
         )
 
-    capture_by_location: dict[tuple[int, str], list[TaskCapture]] = {}
-    for capture in visitor.task_captures.values():
-        capture_by_location.setdefault((capture.span.line, capture.operator), []).append(capture)
     argument_failure_keys: set[str] = set()
-    audit_candidate_by_capture: dict[str, source_audit.AuditCandidate] = {}
-    for candidate in audit.tasks:
-        if candidate.code != "operator_task":
-            continue
-        operator = str(candidate.details.get("operator", ""))
-        matches = capture_by_location.get((candidate.line, operator), [])
-        if not matches:
+    for capture in visitor.task_captures.values():
+        argument_candidate = audit_candidate_by_capture.get(capture.capture_id)
+        if argument_candidate is None or argument_candidate.code != "operator_task":
             continue
-        capture = matches.pop(0)
-        audit_candidate_by_capture[capture.capture_id] = candidate
-        expected = set(candidate.details.get("kwargs", []))
+        expected = set(argument_candidate.details.get("kwargs", []))
         actual = set(visitor.operators[capture.capture_id][2])
         if expected == actual:
             continue
@@ -2781,7 +2799,7 @@ def source_reference(capture_id: str) -> str:
                     f"Airflow task {capture.task_id!r} audited argument(s) {sorted(expected)}, "
                     f"but capture retained {sorted(actual)}."
                 ),
-                candidate=candidate,
+                candidate=argument_candidate,
                 details={
                     "task_key": task_key,
                     "missing": sorted(expected - actual),
@@ -2871,14 +2889,17 @@ def source_reference(capture_id: str) -> str:
             )
         )
 
-    for var, task_key in var_to_task_key.items():
-        task_id = (
-            visitor.operators[var][0]
-            if var in visitor.operators
-            else visitor.taskflow_tasks[var].task_id
-            if var in visitor.taskflow_tasks
-            else visitor.taskgroup_calls[var][0]
+    def source_task_id(capture_id: str) -> str:
+        return (
+            visitor.operators[capture_id][0]
+            if capture_id in visitor.operators
+            else visitor.taskflow_tasks[capture_id].task_id
+            if capture_id in visitor.taskflow_tasks
+            else visitor.taskgroup_calls[capture_id][1]
         )
+
+    for var, task_key in var_to_task_key.items():
+        task_id = source_task_id(var)
         base = _sanitize_task_key(task_id)
         if var in visitor.groups:
             base = f"{visitor.groups[var]}__{base}"
@@ -2917,13 +2938,47 @@ def source_reference(capture_id: str) -> str:
             }
         )
 
-    placeholder_by_key = {
-        placeholder.task_key: placeholder
+    placeholders = [
+        placeholder
         for placeholder in _iter_placeholders(pipeline.tasks)
         if not placeholder.task_key.startswith("__flowx_")
-    }
-    for index, placeholder in enumerate(placeholder_by_key.values()):
-        placeholder_candidate = audit.tasks[index] if index < len(audit.tasks) else None
+    ]
+    capture_by_placeholder_key: dict[str, str] = {}
+    for capture_id, task_key in expected_key_by_capture.items():
+        capture_by_placeholder_key[task_key] = capture_id
+        if capture_id in visitor.mapped:
+            capture_by_placeholder_key[f"{task_key}_iteration"] = capture_id
+
+    for placeholder in placeholders:
+        placeholder_capture_id = capture_by_placeholder_key.get(placeholder.task_key)
+        if placeholder_capture_id is None:
+            findings.append(
+                source_audit.finding(
+                    source_file=source_file,
+                    code="operator_placeholder_capture_mismatch",
+                    severity="failed",
+                    message=(f"Placeholder task {placeholder.task_key!r} has no captured Airflow task identity."),
+                    details={"task_key": placeholder.task_key, "operator": placeholder.original_type},
+                )
+            )
+            continue
+        placeholder_candidate = audit_candidate_by_capture.get(placeholder_capture_id)
+        if placeholder_candidate is None:
+            node = visitor.capture_source_nodes[placeholder_capture_id]
+            span = _span(node)
+            placeholder_candidate = source_audit.AuditCandidate(
+                kind="task",
+                code="captured_task",
+                line=span.line,
+                column=span.column,
+                occurrence=1,
+                end_line=span.end_line,
+                end_column=span.end_column,
+                details={
+                    "task_id": source_task_id(placeholder_capture_id),
+                    "operator": placeholder.original_type,
+                },
+            )
         findings.append(
             source_audit.finding(
                 source_file=source_file,
@@ -2933,7 +2988,12 @@ def source_reference(capture_id: str) -> str:
                     f"Airflow task {placeholder.name!r} ({placeholder.original_type}) requires explicit migration."
                 ),
                 candidate=placeholder_candidate,
-                details={"task_key": placeholder.task_key, "operator": placeholder.original_type},
+                details={
+                    "task_key": placeholder.task_key,
+                    "operator": placeholder.original_type,
+                    "capture_id": placeholder_capture_id,
+                    "source_task_id": source_task_id(placeholder_capture_id),
+                },
             )
         )
 
@@ -2956,7 +3016,7 @@ def source_reference(capture_id: str) -> str:
     gap_findings = [item for item in findings if item["severity"] == "gap"]
     status = "failed" if failed_findings else "verified_with_gaps" if gap_findings else "verified"
     failed_capture_keys = argument_failure_keys | set(missing_task_keys)
-    agentic_captured_count = len(placeholder_by_key)
+    agentic_captured_count = len(placeholders)
     deterministic_count = captured_task_count - len(failed_capture_keys) - agentic_captured_count
     agentic_count = agentic_captured_count + len(unresolved)
     failed_count = (
diff --git a/tests/unit/test_airflow_agentic_resolution.py b/tests/unit/test_airflow_agentic_resolution.py
index 1f96285..6b251f7 100644
--- a/tests/unit/test_airflow_agentic_resolution.py
+++ b/tests/unit/test_airflow_agentic_resolution.py
@@ -13,6 +13,7 @@
 from flowx.bundler.dab_writer import main as package_main
 from flowx.reporting.coverage import build_coverage_rows
 from flowx.sources.airflow.convert import main as airflow_convert
+from flowx.sources.airflow.loader import load_airflow_dag
 
 
 def _write_source(tmp_path: Path, *, two_tasks: bool = False) -> Path:
@@ -140,6 +141,110 @@ def test_prepare_writes_versioned_fingerprint_bound_gap_without_changing_report(
     assert (output / ".work" / "agentic" / "source" / "dag.py").read_bytes() == source.read_bytes()
 
 
+def test_gap_fingerprints_use_each_placeholder_own_source_span(tmp_path: Path) -> None:
+    source = tmp_path / "interleaved.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "with DAG(dag_id='interleaved') as dag:\n"
+        "    a_ok = BashOperator(task_id='a_ok', bash_command='echo a')\n"
+        "    b_ok = BashOperator(task_id='b_ok', bash_command='echo b')\n"
+        "    c_gap = KubernetesPodOperator(task_id='c_gap', image='python:3.12')\n"
+        "    d_ok = BashOperator(task_id='d_ok', bash_command='echo d')\n"
+        "    e_gap = KubernetesPodOperator(task_id='e_gap', image='python:3.12')\n",
+        encoding="utf-8",
+    )
+
+    pipeline = load_airflow_dag(source)
+    findings = {
+        item["details"]["source_task_id"]: item
+        for item in pipeline.not_translatable
+        if item["code"] == "operator_placeholder"
+    }
+
+    assert {task_id: finding["line"] for task_id, finding in findings.items()} == {"c_gap": 13, "e_gap": 15}
+    assert findings["c_gap"]["fingerprint"] != findings["e_gap"]["fingerprint"]
+
+
+@pytest.mark.parametrize(
+    ("source_text", "expected"),
+    [
+        (
+            "from airflow import DAG\n"
+            "with DAG(dag_id='collision') as dag:\n"
+            "    first = KubernetesPodOperator(task_id='load.data', image='python:3.12')\n"
+            "    second = KubernetesPodOperator(task_id='load_data', image='python:3.12')\n",
+            [("load_data", "load.data", 3), ("load_data__2", "load_data", 4)],
+        ),
+        (
+            "from airflow import DAG\n"
+            "with DAG(dag_id='mapped') as dag:\n"
+            "    pod = KubernetesPodOperator.partial(task_id='pod', image='python:3.12').expand(env=['a'])\n",
+            [("pod_iteration", "pod", 3)],
+        ),
+        (
+            "from airflow import DAG\n"
+            "with DAG(dag_id='bare') as dag:\n"
+            "    KubernetesPodOperator(task_id='bare_pod', image='python:3.12')\n",
+            [("bare_pod", "bare_pod", 3)],
+        ),
+        (
+            "from airflow import DAG\n"
+            "def make(task_id):\n"
+            "    return KubernetesPodOperator(task_id=task_id, image='python:3.12')\n"
+            "with DAG(dag_id='helper') as dag:\n"
+            "    pod = make('helper_pod')\n",
+            [("helper_pod", "helper_pod", 5)],
+        ),
+        (
+            "from airflow.decorators import dag, task\n"
+            "@task.branch\n"
+            "def choose():\n"
+            "    return 'next'\n"
+            "@dag(dag_id='taskflow')\n"
+            "def workflow():\n"
+            "    choose()\n"
+            "workflow()\n",
+            [("choose", "choose", 7)],
+        ),
+        (
+            "from airflow.decorators import dag, task_group\n"
+            "@task_group\n"
+            "def grouped():\n"
+            "    pass\n"
+            "@dag(dag_id='task_group')\n"
+            "def workflow():\n"
+            "    grouped()\n"
+            "workflow()\n",
+            [("grouped_tg1", "grouped", 7)],
+        ),
+    ],
+    ids=("collision-safe-keys", "classic-mapped-inner", "bare-operator", "helper-factory", "taskflow", "task-group"),
+)
+def test_placeholder_findings_bind_capture_identity_to_source(
+    tmp_path: Path,
+    source_text: str,
+    expected: list[tuple[str, str, int]],
+) -> None:
+    source = tmp_path / "dag.py"
+    source.write_text(source_text, encoding="utf-8")
+
+    pipeline = load_airflow_dag(source)
+    findings = [item for item in pipeline.not_translatable if item["code"] == "operator_placeholder"]
+
+    assert [
+        (item["details"]["task_key"], item["details"]["source_task_id"], item["line"]) for item in findings
+    ] == expected
+    assert all(item["details"]["capture_id"] for item in findings)
+
+
 def test_resolve_agentic_is_explicitly_airflow_only(tmp_path: Path, capsys):
     exit_code = adapter_main(
         [

From 2fc6569890a825f5a6feecfd39e7c39a315a129e Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Sun, 9 Aug 2026 00:22:17 -0700
Subject: [PATCH 58/77] Harden Airflow agentic gap contract

---
 docs/content/docs/architecture.mdx            |   2 +-
 skills/flowx-resolve-airflow-gaps/SKILL.md    |   4 +-
 .../references/contract-v1.md                 |  10 +-
 src/flowx/agentic.py                          | 150 +++++--
 src/flowx/sources/airflow/audit.py            |   3 +
 src/flowx/sources/airflow/loader.py           | 147 +++++--
 src/flowx/sources/airflow/templating.py       |  42 +-
 tests/unit/test_airflow_agentic_resolution.py | 387 ++++++++++++++++++
 .../unit/test_airflow_production_readiness.py |  34 ++
 9 files changed, 695 insertions(+), 84 deletions(-)

diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx
index ce327bb..d50b88f 100644
--- a/docs/content/docs/architecture.mdx
+++ b/docs/content/docs/architecture.mdx
@@ -59,7 +59,7 @@ The unified `flowx.adapter` CLI is the single contract. Both surfaces go through
 | `mcp/runner.py`   | Subprocess bridge to `flowx.adapter` with artifact summarizers (for running translation without the `mcp` dependency) |
 | `mcp/__main__.py` | `python -m flowx.mcp` entry point (stdio default, `--http` for hosting)                                               |
 
-The `flowx` tool's `command` selects the adapter operation: `inputs`, `discover`, `convert`, `merge_agentic`, `inspect`, `apply_answers`, `materialize_lookup`, `workspace_paths`, `package`, `migrate`, `record_results`, and `install_dashboard` (with `parameters` carrying that command's arguments).
+The `flowx` tool's `command` selects the adapter operation: `inputs`, `discover`, `convert`, `merge_agentic` (ADF only), `resolve_agentic` (Airflow only), `inspect`, `apply_answers`, `materialize_lookup`, `workspace_paths`, `package`, `migrate`, `record_results`, and `install_dashboard` (with `parameters` carrying that command's arguments).
 
 ## Deployment topology
 
diff --git a/skills/flowx-resolve-airflow-gaps/SKILL.md b/skills/flowx-resolve-airflow-gaps/SKILL.md
index 3cacf6f..f027fd5 100644
--- a/skills/flowx-resolve-airflow-gaps/SKILL.md
+++ b/skills/flowx-resolve-airflow-gaps/SKILL.md
@@ -56,7 +56,9 @@ state the specific semantic loss. Never include task names, task keys, dependenc
 timeouts, clusters, schedules, or other graph/policy fields in the replacement.
 
 Generated code must be self-contained, contain no Airflow import statements, and contain no
-unresolved Airflow Jinja. Comments may mention Airflow for provenance.
+template expressions. Python notebooks must start with `# Databricks notebook source`. Put
+Databricks dynamic references in replacement parameters and read them through notebook widgets or
+SQL named parameters. Comments may mention Airflow for provenance.
 
 ## 3. Stage candidates
 
diff --git a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
index 5e76844..7f6b7ad 100644
--- a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
+++ b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
@@ -3,6 +3,10 @@
 The provider receives a `GapEnvelope` produced by flowx. It does not receive authority to alter the
 captured graph.
 
+`capture_identity` is flowx's source-capture identity and may differ from the collision-safe
+Databricks `task_key`. `task_path` identifies the exact placeholder location, including a nested
+`for_each` body; providers must copy neither field into the replacement payload.
+
 ## Resolution shape
 
 ```json
@@ -55,8 +59,10 @@ automatic retry occurs.
 - Generated file paths are relative and cannot contain `..`.
 - Every generated file is inline and hash-bound; external workspace paths are not accepted.
 - Python payloads may mention Airflow in comments but may not contain `import airflow` or
-  `from airflow ...` statements.
+  `from airflow ...` statements. They must start with `# Databricks notebook source` so the bundle
+  imports them as notebooks rather than ordinary Python files.
 - Airflow Jinja is rejected. Databricks dynamic references such as `{{job.parameters.x}}`,
-  `{{tasks.upstream.values.x}}`, and `{{input}}` remain valid.
+  `{{tasks.upstream.values.x}}`, and `{{input}}` are valid in replacement parameter values, not in
+  uploaded notebook or SQL source files; source files must read widgets or SQL named parameters.
 - Every source argument in the envelope has exactly one disposition and a non-empty rationale.
 - The provider identity must match the pinned `airflow-to-dabs` v0.2.0 knowledge release.
diff --git a/src/flowx/agentic.py b/src/flowx/agentic.py
index 21dbb7c..ffbc64a 100644
--- a/src/flowx/agentic.py
+++ b/src/flowx/agentic.py
@@ -48,20 +48,10 @@
 )
 _FLOWX_OWNED_ARGUMENTS = {
     "task_id",
-    "retries",
-    "retry_delay",
-    "execution_timeout",
-    "trigger_rule",
-    "depends_on_past",
-    "wait_for_downstream",
-    "pool",
-    "pool_slots",
-    "priority_weight",
-    "queue",
 }
 _NESTED_TASK_FIELDS = ("inner_activities", "if_true_activities", "if_false_activities", "default_activities")
 _AIRFLOW_TEMPLATE = re.compile(r"{{\s*([^{}]+?)\s*}}|{%\s*([^{}]+?)\s*%}")
-_DAB_TEMPLATE_PREFIXES = ("job.", "tasks.", "input", "backfill.")
+_DAB_TEMPLATE_PREFIXES = ("job.", "tasks.", "input.", "backfill.")
 
 
 class AgenticContractError(ValueError):
@@ -74,6 +64,7 @@ class GapEnvelope:
 
     gap_id: str
     pipeline_name: str
+    capture_identity: str
     task_key: str
     task_path: list[str | int]
     operator: str
@@ -95,7 +86,7 @@ def as_dict(self) -> dict[str, Any]:
             "gap_id": self.gap_id,
             "source": "airflow",
             "pipeline_name": self.pipeline_name,
-            "capture_identity": self.task_key,
+            "capture_identity": self.capture_identity,
             "task_key": self.task_key,
             "task_path": self.task_path,
             "operator": self.operator,
@@ -242,7 +233,10 @@ def stage_airflow_resolutions(*, output_dir: Path, candidate_paths: list[Path])
         except json.JSONDecodeError as error:
             raise AgenticContractError(f"Candidate {path} contains invalid JSON: {error}") from error
         resolution = _validate_candidate(candidate, gap_by_id=gap_by_id, manifest=manifest)
-        staged[resolution.gap["gap_id"]] = resolution
+        gap_id = str(resolution.gap["gap_id"])
+        if gap_id in staged:
+            raise AgenticContractError(f"Duplicate candidate supplied for gap_id: {gap_id}")
+        staged[gap_id] = resolution
 
     candidates_dir = workspace / "candidates"
     candidates_dir.mkdir(exist_ok=True)
@@ -344,12 +338,17 @@ def summarize_persisted_agentic_resolutions(evidence_dir: Path) -> dict[str, Any
         return {}
     evidence = _load_persisted_agentic_evidence(evidence_dir)
     pipeline_outcomes: dict[str, dict[str, int]] = {}
-    for gap in evidence.gaps:
-        pipeline = str(gap["pipeline_name"])
-        pipeline_outcomes.setdefault(pipeline, _empty_resolution_outcomes())["unreviewed"] += 1
+    for pipeline in _pipeline_list(evidence.expected_report):
+        name = str(pipeline["name"])
+        audit = pipeline.get("audit") or {}
+        outcomes = _empty_resolution_outcomes()
+        outcomes["unreviewed"] = int(audit.get("agentic_count", 0))
+        pipeline_outcomes[name] = outcomes
     for resolution in evidence.resolutions:
-        pipeline = str(resolution.gap["pipeline_name"])
-        outcomes = pipeline_outcomes.setdefault(pipeline, _empty_resolution_outcomes())
+        pipeline_name = str(resolution.gap["pipeline_name"])
+        outcomes = pipeline_outcomes.setdefault(pipeline_name, _empty_resolution_outcomes())
+        if outcomes["unreviewed"] <= 0:
+            raise AgenticContractError(f"agentic resolution over-accounts pipeline {pipeline_name!r}")
         outcomes["unreviewed"] -= 1
         outcomes[str(resolution.candidate["status"])] += 1
     return {
@@ -484,33 +483,63 @@ def _build_gap_envelopes(
             source_hash = next(iter(source_hashes.values()))
         if source_hash is None:
             raise AgenticContractError(f"No source snapshot hash matches pipeline {pipeline.get('name')!r}")
-        findings = {
-            (finding.get("details") or {}).get("task_key"): finding
-            for finding in pipeline.get("not_translatable") or []
-            if isinstance(finding, dict) and finding.get("code") == "operator_placeholder"
-        }
+        pipeline_findings = [finding for finding in pipeline.get("not_translatable") or [] if isinstance(finding, dict)]
+        findings: dict[tuple[str | int, ...], dict[str, Any]] = {}
+        for finding in pipeline_findings:
+            if finding.get("code") != "operator_placeholder":
+                continue
+            task_path = (finding.get("details") or {}).get("task_path")
+            if not isinstance(task_path, list) or not all(isinstance(item, (str, int)) for item in task_path):
+                raise AgenticContractError("Operator placeholder finding is missing its captured task path")
+            path_key = tuple(task_path)
+            if path_key in findings:
+                raise AgenticContractError(f"Duplicate operator placeholder finding at task path: {task_path}")
+            findings[path_key] = finding
         tasks = list(_walk_tasks(pipeline.get("tasks") or []))
         downstream = _downstream_index([task for _, task in tasks])
         for task_path, task in tasks:
-            if task.get("type") != "PlaceholderActivity" or str(task.get("task_key", "")).startswith("__flowx_"):
-                continue
-            finding = findings.get(task.get("task_key"))
-            if not finding or not finding.get("fingerprint"):
+            if task.get("type") != "PlaceholderActivity" or task.get("original_type") == "AirflowSourceSemantics":
                 continue
+            matched_finding = findings.get(task_path)
+            if not matched_finding or not matched_finding.get("fingerprint"):
+                raise AgenticContractError(f"Placeholder at task path {list(task_path)} has no bound finding")
             raw_definition = dict(task.get("raw_definition") or {})
             operator = str(raw_definition.get("operator") or task.get("original_type") or "UnknownOperator")
+            finding_details = matched_finding.get("details") or {}
+            capture_identity = finding_details.get("capture_id")
+            if not isinstance(capture_identity, str) or not capture_identity:
+                raise AgenticContractError(f"Placeholder at task path {list(task_path)} has no capture identity")
+            flowx_owned_arguments = set(_FLOWX_OWNED_ARGUMENTS)
+            if task.get("max_retries") is not None:
+                flowx_owned_arguments.add("retries")
+            if task.get("min_retry_interval_millis") is not None:
+                flowx_owned_arguments.add("retry_delay")
+            if task.get("timeout_seconds") is not None:
+                flowx_owned_arguments.add("execution_timeout")
+            related_findings = [
+                item for item in pipeline_findings if (item.get("details") or {}).get("capture_id") == capture_identity
+            ]
+            if not any(item.get("code") == "unsupported_trigger_rule" for item in related_findings):
+                flowx_owned_arguments.add("trigger_rule")
             envelope = GapEnvelope(
-                gap_id=str(finding["fingerprint"]),
+                gap_id=str(matched_finding["fingerprint"]),
                 pipeline_name=str(pipeline["name"]),
+                capture_identity=capture_identity,
                 task_key=str(task["task_key"]),
                 task_path=list(task_path),
                 operator=operator,
                 source_file=str(source_file),
                 source_sha256=source_hash,
                 baseline_report_sha256=baseline_hash,
-                source_span={key: int(finding.get(key, 0)) for key in ("line", "column", "end_line", "end_column")},
+                source_span={
+                    key: int(matched_finding.get(key, 0)) for key in ("line", "column", "end_line", "end_column")
+                },
                 raw_definition=raw_definition,
-                arguments=_extract_arguments(raw_definition, operator=operator),
+                arguments=_extract_arguments(
+                    raw_definition,
+                    operator=operator,
+                    flowx_owned_arguments=flowx_owned_arguments,
+                ),
                 upstream_task_keys=[str(item.get("task_key")) for item in task.get("depends_on") or []],
                 downstream_task_keys=downstream.get(str(task["task_key"]), []),
                 dag_settings={
@@ -519,16 +548,36 @@ def _build_gap_envelopes(
                     "tags": pipeline.get("tags"),
                 },
                 reason={
-                    "code": str(finding.get("code", "operator_placeholder")),
-                    "message": str(finding.get("message", "")),
+                    "code": str(matched_finding.get("code", "operator_placeholder")),
+                    "message": str(task.get("comment") or matched_finding.get("message", "")),
                 },
             )
             envelopes.append(envelope.as_dict())
-    return sorted(envelopes, key=lambda item: (item["pipeline_name"], item["task_path"]))
+    ordered = sorted(envelopes, key=lambda item: (item["pipeline_name"], item["task_path"]))
+    gap_ids = [item["gap_id"] for item in ordered]
+    if len(gap_ids) != len(set(gap_ids)):
+        raise AgenticContractError("Prepared Airflow gaps contain duplicate fingerprints")
+    return ordered
 
 
-def _extract_arguments(raw_definition: dict[str, Any], *, operator: str) -> list[dict[str, Any]]:
-    source = raw_definition.get("source")
+def _extract_arguments(
+    raw_definition: dict[str, Any],
+    *,
+    operator: str,
+    flowx_owned_arguments: set[str] | None = None,
+) -> list[dict[str, Any]]:
+    owned = _FLOWX_OWNED_ARGUMENTS if flowx_owned_arguments is None else flowx_owned_arguments
+    bound_source = raw_definition.get("bound_source")
+    invocation = raw_definition.get("invocation")
+    source = (
+        bound_source
+        if isinstance(bound_source, str) and bound_source.strip()
+        else invocation
+        if isinstance(invocation, str) and invocation.strip()
+        else raw_definition.get("source")
+    )
+    if operator.startswith("@") and not (isinstance(invocation, str) and invocation.strip()):
+        source = None
     arguments: list[dict[str, Any]] = []
     if isinstance(source, str) and source.strip():
         try:
@@ -543,25 +592,25 @@ def _extract_arguments(raw_definition: dict[str, Any], *, operator: str) -> list
                 for index, value in enumerate(call.args):
                     name = f"$star{index}" if isinstance(value, ast.Starred) else f"$arg{index}"
                     expression = ast.unparse(value.value if isinstance(value, ast.Starred) else value)
-                    arguments.append(_argument(name, expression))
+                    arguments.append(_argument(name, expression, owned))
                 kwargs_index = 0
                 for keyword in call.keywords:
                     keyword_name = keyword.arg
                     if keyword_name is None:
                         keyword_name = f"$kwargs{kwargs_index}"
                         kwargs_index += 1
-                    arguments.append(_argument(keyword_name, ast.unparse(keyword.value)))
+                    arguments.append(_argument(keyword_name, ast.unparse(keyword.value), owned))
     mapping = raw_definition.get("mapping")
     if isinstance(mapping, str) and mapping:
-        arguments.append(_argument("$mapping", mapping))
+        arguments.append(_argument("$mapping", mapping, owned))
     return arguments
 
 
-def _argument(name: str, expression: str) -> dict[str, Any]:
+def _argument(name: str, expression: str, flowx_owned_arguments: set[str]) -> dict[str, Any]:
     return {
         "name": name,
         "source_expression": expression,
-        "preserved_by_flowx": name in _FLOWX_OWNED_ARGUMENTS,
+        "preserved_by_flowx": name in flowx_owned_arguments,
     }
 
 
@@ -691,8 +740,13 @@ def _validate_replacement(candidate: dict[str, Any], gap: dict[str, Any]) -> Non
         raise AgenticContractError("Generated file content must be non-empty")
     if generated.get("sha256") != _sha256_bytes(content.encode("utf-8")):
         raise AgenticContractError("Generated file sha256 does not match its content")
-    _reject_unresolved_templates({"replacement": replacement, "generated_files": files})
+    _reject_unresolved_templates(replacement)
+    _reject_generated_file_templates(content)
     if kind == "notebook":
+        lines = content.splitlines()
+        first_line = lines[0] if lines else ""
+        if first_line != "# Databricks notebook source":
+            raise AgenticContractError("Generated notebook requires the Databricks notebook source marker")
         try:
             module = ast.parse(content)
         except SyntaxError as error:
@@ -891,7 +945,7 @@ def _reject_unresolved_templates(value: Any) -> None:
     if isinstance(value, str):
         for match in _AIRFLOW_TEMPLATE.finditer(value):
             expression = (match.group(1) or match.group(2) or "").strip()
-            if not expression.startswith(_DAB_TEMPLATE_PREFIXES):
+            if expression != "input" and not expression.startswith(_DAB_TEMPLATE_PREFIXES):
                 raise AgenticContractError(f"Generated payload contains unresolved Airflow Jinja: {match.group(0)}")
     elif isinstance(value, dict):
         for item in value.values():
@@ -901,6 +955,18 @@ def _reject_unresolved_templates(value: Any) -> None:
             _reject_unresolved_templates(item)
 
 
+def _reject_generated_file_templates(content: str) -> None:
+    """Rejects templates in uploaded source files, where Jobs cannot interpolate them."""
+    for match in _AIRFLOW_TEMPLATE.finditer(content):
+        expression = (match.group(1) or match.group(2) or "").strip()
+        if expression == "input" or expression.startswith(_DAB_TEMPLATE_PREFIXES):
+            raise AgenticContractError(
+                "Generated file content cannot contain Databricks dynamic references; "
+                "pass them through replacement parameters"
+            )
+        raise AgenticContractError(f"Generated payload contains unresolved Airflow Jinja: {match.group(0)}")
+
+
 def _pipeline_list(payload: Any) -> list[dict[str, Any]]:
     if not isinstance(payload, dict):
         raise AgenticContractError("Translation report must be a JSON object")
diff --git a/src/flowx/sources/airflow/audit.py b/src/flowx/sources/airflow/audit.py
index f7f5abd..de05bb6 100644
--- a/src/flowx/sources/airflow/audit.py
+++ b/src/flowx/sources/airflow/audit.py
@@ -41,6 +41,7 @@ def finding(
     severity: str,
     candidate: AuditCandidate | None = None,
     details: dict[str, Any] | None = None,
+    identity_discriminator: str | None = None,
 ) -> dict[str, Any]:
     """Builds a stable, serializable reconciliation finding."""
     line = candidate.line if candidate else 0
@@ -48,6 +49,8 @@ def finding(
     end_line = candidate.end_line if candidate else 0
     end_column = candidate.end_column if candidate else 0
     identity = f"{source_file}:{line}:{column}:{end_line}:{end_column}:{code}"
+    if identity_discriminator:
+        identity = f"{identity}:{identity_discriminator}"
     return {
         "fingerprint": hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16],
         "code": code,
diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py
index dfbd5ef..702bb9f 100644
--- a/src/flowx/sources/airflow/loader.py
+++ b/src/flowx/sources/airflow/loader.py
@@ -2202,6 +2202,14 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
         return dbt_key_remap.get(key, key)
 
     tasks: list[Activity] = []
+    placeholder_capture_ids: dict[int, str] = {}
+    helper_expansion_ids = {str(item["capture_id"]) for item in visitor.helper_expansions}
+
+    def append_task(activity: Activity, capture_id: str) -> None:
+        for placeholder in _iter_placeholders([activity]):
+            placeholder_capture_ids[id(placeholder)] = capture_id
+        tasks.append(activity)
+
     semantic_findings: list[dict[str, Any]] = []
     argument_proofs = [
         {
@@ -2233,8 +2241,9 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
         depends_on = [Dependency(task_key=k, outcome=outcome) for k in sorted(dep_keys)] or None
 
         if operator in ops.COSMOS_CONSTRUCTS:
-            tasks.append(
-                _build_dbt_factory(task_id, task_key, [kwargs], depends_on, dbt_mode, operator_types=[operator])
+            append_task(
+                _build_dbt_factory(task_id, task_key, [kwargs], depends_on, dbt_mode, operator_types=[operator]),
+                var,
             )
             continue
         if operator in ops.DBT_CLI_OPERATORS:
@@ -2246,7 +2255,7 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
             # any that are downstream of the factory itself (a sandwiched task, which would cycle).
             factory_depends_on = [Dependency(task_key=k, outcome=outcome) for k in sorted(factory_dep_keys)] or None
             dbt_kwargs = [visitor.operators[v][2] for v in dbt_vars]
-            tasks.append(
+            append_task(
                 _build_dbt_factory(
                     task_id,
                     task_key,
@@ -2254,12 +2263,18 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
                     factory_depends_on,
                     dbt_mode,
                     operator_types=[visitor.operators[dbt_var][1] for dbt_var in dbt_vars],
-                )
+                ),
+                var,
             )
             continue
 
         call_node = visitor.calls.get(var)
-        call_source = ast.get_source_segment(source, call_node) or "" if call_node is not None else ""
+        if call_node is None:
+            call_source = ""
+        elif var in helper_expansion_ids:
+            call_source = ast.unparse(call_node)
+        else:
+            call_source = ast.get_source_segment(source, call_node) or ""
         ctx = ops.OperatorContext(
             task_id=task_id,
             task_key=task_key,
@@ -2286,6 +2301,7 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
                     code="unsupported_trigger_rule",
                     message=(f"Task {task_id!r} uses trigger_rule {trigger_mapping.rule!r}; {trigger_mapping.message}"),
                     task_key=task_key,
+                    capture_id=var,
                 )
             )
         elif trigger_mapping.status == "approximate":
@@ -2299,6 +2315,7 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
                         f"{trigger_mapping.outcome}. {trigger_mapping.message}"
                     ),
                     task_key=task_key,
+                    capture_id=var,
                 )
             )
         unconsumed = ops.unconsumed_kwargs(operator, kwargs)
@@ -2317,14 +2334,30 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
                     code="unconsumed_operator_arguments",
                     message=f"Task {task_id!r} has unconsumed operator argument(s): {names}.",
                     task_key=task_key,
+                    capture_id=var,
                     arguments=sorted(unconsumed),
                 )
             )
-        # Stamp DAG/task retry + timeout policy (per-task kwargs override default_args).
-        policy = templating.retry_policy(visitor.default_args, kwargs)
-        activity.max_retries = policy.get("max_retries")
-        activity.timeout_seconds = policy.get("timeout_seconds")
-        activity.min_retry_interval_millis = policy.get("min_retry_interval_millis")
+        unrepresented_policy = templating.unrepresented_retry_policy_arguments(visitor.default_args, kwargs)
+        if unrepresented_policy:
+            names = ", ".join(unrepresented_policy)
+            activity = ops.build_placeholder_with_comment(
+                ctx,
+                f"Airflow task policy argument(s) {names} cannot be represented statically; "
+                "resolve the policy before migration.",
+            )
+            activity.depends_on = depends_on
+            semantic_findings.append(
+                _semantic_finding(
+                    source_file or dag_path.name,
+                    visitor.calls.get(var),
+                    code="unrepresented_task_policy",
+                    message=f"Task {task_id!r} has unrepresented retry/timeout policy argument(s): {names}.",
+                    task_key=task_key,
+                    capture_id=var,
+                    arguments=unrepresented_policy,
+                )
+            )
         # Convert Airflow Jinja in the activity's parameter fields to DAB refs; collect params.
         referenced_params |= _convert_activity_templates(activity)
         unresolved_templates = _unresolved_activity_templates(activity)
@@ -2343,11 +2376,14 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
                     code="unresolved_airflow_template",
                     message=f"Task {task_id!r} contains unresolved Airflow template expression(s): {expressions}.",
                     task_key=task_key,
+                    capture_id=var,
                     expressions=sorted(unresolved_templates),
                 )
             )
 
-        if var in visitor.mapped:
+        is_mapped = var in visitor.mapped
+        mapped_names: list[str] = []
+        if is_mapped:
             mapped_names = visitor.expand_kwargs.get(var) or []
             partial_note = (
                 " The mapping also contains .partial() fixed arguments." if var in visitor.partial_mapped else ""
@@ -2368,13 +2404,26 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
                         "but the generated inner task cannot bind them safely."
                     ),
                     task_key=task_key,
+                    capture_id=var,
                     arguments=mapped_names,
                     has_partial=var in visitor.partial_mapped,
                 )
             )
-            tasks.append(_wrap_in_for_each(activity, task_id, task_key, depends_on, kwargs, mapped_names))
+
+        # Stamp policy only after every semantic guard has selected the final leaf activity.
+        policy = templating.retry_policy(visitor.default_args, kwargs)
+        activity.max_retries = policy.get("max_retries")
+        activity.timeout_seconds = policy.get("timeout_seconds")
+        activity.min_retry_interval_millis = policy.get("min_retry_interval_millis")
+        if isinstance(activity, PlaceholderActivity) and call_node is not None:
+            raw_definition = dict(activity.raw_definition or {})
+            raw_definition["bound_source"] = ast.unparse(call_node)
+            activity.raw_definition = raw_definition
+
+        if is_mapped:
+            append_task(_wrap_in_for_each(activity, task_id, task_key, depends_on, kwargs, mapped_names), var)
         else:
-            tasks.append(activity)
+            append_task(activity, var)
 
     # TaskFlow @task instances: emit each as a notebook that reads upstream return values via
     # dbutils.jobs.taskValues, calls the decorated function, and sets its own return value.
@@ -2410,17 +2459,21 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
                 },
             )
             placeholder.depends_on = depends_on
-            tasks.append(placeholder)
+            append_task(placeholder, var)
             continue
         activity = _build_taskflow_task(tf, var_to_task_key, functions, source, task_key)
         activity.depends_on = depends_on
+        if isinstance(activity, PlaceholderActivity):
+            raw_definition = dict(activity.raw_definition or {})
+            raw_definition["invocation"] = ast.get_source_segment(source, visitor.capture_source_nodes[var]) or ""
+            activity.raw_definition = raw_definition
         referenced_params |= _convert_activity_templates(activity)
         if var in visitor.mapped and isinstance(activity, NotebookActivity):
             # .expand(param=[literal list]) -> a for_each_task iterating the callable notebook; the
             # inner notebook reads the mapped parameter from the per-iteration `item` widget.
-            tasks.append(_wrap_taskflow_in_for_each(activity, tf, task_key, depends_on))
+            append_task(_wrap_taskflow_in_for_each(activity, tf, task_key, depends_on), var)
         else:
-            tasks.append(activity)
+            append_task(activity, var)
 
     # @task_group invocations: a group is a sub-pipeline of tasks with no single-task lowering, so
     # emit a placeholder + gap (never silently drop the whole group) for the agentic round to expand.
@@ -2444,10 +2497,11 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
             raw_definition={
                 "operator": "@task_group",
                 "source": ast.get_source_segment(source, group_func) if group_func is not None else "",
+                "invocation": ast.get_source_segment(source, visitor.capture_source_nodes[var]) or "",
             },
         )
         placeholder.depends_on = depends_on
-        tasks.append(placeholder)
+        append_task(placeholder, var)
 
     # Declare every job parameter -- those referenced in templates plus any from the DAG's
     # params={...} -- each with a default (Databricks requires one): the params={...} default when
@@ -2483,6 +2537,7 @@ def _dep(upstream_var: str, outcome: str | None) -> str:
         sensor_lift_proof=sensor_lift_proof,
         argument_proofs=argument_proofs,
         expected_ir_edges=expected_ir_edges,
+        placeholder_capture_ids=placeholder_capture_ids,
     )
 
 
@@ -2510,6 +2565,7 @@ def _semantic_finding(
     code: str,
     message: str,
     task_key: str,
+    capture_id: str,
     **details: Any,
 ) -> dict[str, Any]:
     """Builds a stable gap finding for a captured task-level semantic limitation."""
@@ -2521,7 +2577,7 @@ def _semantic_finding(
         occurrence=1,
         end_line=getattr(node, "end_lineno", 0),
         end_column=getattr(node, "end_col_offset", 0),
-        details={"task_key": task_key, **details},
+        details={"task_key": task_key, "capture_id": capture_id, **details},
     )
     return source_audit.finding(
         source_file=source_file,
@@ -2532,17 +2588,26 @@ def _semantic_finding(
     )
 
 
-def _iter_placeholders(tasks: list[Activity]) -> list[PlaceholderActivity]:
-    """Returns placeholders in top-level and Airflow-generated for_each tasks."""
-    placeholders: list[PlaceholderActivity] = []
-    for task in tasks:
+def _iter_placeholders_with_paths(
+    tasks: list[Activity],
+    path: tuple[str | int, ...] = ("tasks",),
+) -> list[tuple[tuple[str | int, ...], PlaceholderActivity]]:
+    """Returns placeholders with their stable serialized Pipeline IR paths."""
+    placeholders: list[tuple[tuple[str | int, ...], PlaceholderActivity]] = []
+    for index, task in enumerate(tasks):
+        task_path = (*path, index)
         if isinstance(task, PlaceholderActivity):
-            placeholders.append(task)
+            placeholders.append((task_path, task))
         if isinstance(task, ForEachActivity):
-            placeholders.extend(_iter_placeholders(task.inner_activities))
+            placeholders.extend(_iter_placeholders_with_paths(task.inner_activities, (*task_path, "inner_activities")))
     return placeholders
 
 
+def _iter_placeholders(tasks: list[Activity]) -> list[PlaceholderActivity]:
+    """Returns placeholders in top-level and Airflow-generated for_each tasks."""
+    return [placeholder for _, placeholder in _iter_placeholders_with_paths(tasks)]
+
+
 def _reconcile_pipeline(
     pipeline: Pipeline,
     *,
@@ -2556,6 +2621,7 @@ def _reconcile_pipeline(
     sensor_lift_proof: dict[str, Any] | None,
     argument_proofs: list[dict[str, Any]],
     expected_ir_edges: set[tuple[str, str]],
+    placeholder_capture_ids: dict[int, str],
 ) -> Pipeline:
     """Reconciles an independent source audit with captured graph and emitted IR."""
     findings: list[dict[str, Any]] = list(semantic_findings)
@@ -2938,19 +3004,18 @@ def source_task_id(capture_id: str) -> str:
             }
         )
 
-    placeholders = [
-        placeholder
-        for placeholder in _iter_placeholders(pipeline.tasks)
+    blocking_gaps = [*unsupported_settings, *unresolved]
+    placeholder_entries = [
+        (
+            ("tasks", int(task_path[1]) + 1, *task_path[2:]) if blocking_gaps else task_path,
+            placeholder,
+        )
+        for task_path, placeholder in _iter_placeholders_with_paths(pipeline.tasks)
         if not placeholder.task_key.startswith("__flowx_")
     ]
-    capture_by_placeholder_key: dict[str, str] = {}
-    for capture_id, task_key in expected_key_by_capture.items():
-        capture_by_placeholder_key[task_key] = capture_id
-        if capture_id in visitor.mapped:
-            capture_by_placeholder_key[f"{task_key}_iteration"] = capture_id
-
-    for placeholder in placeholders:
-        placeholder_capture_id = capture_by_placeholder_key.get(placeholder.task_key)
+    placeholders = [placeholder for _, placeholder in placeholder_entries]
+    for task_path, placeholder in placeholder_entries:
+        placeholder_capture_id = placeholder_capture_ids.get(id(placeholder))
         if placeholder_capture_id is None:
             findings.append(
                 source_audit.finding(
@@ -2958,7 +3023,11 @@ def source_task_id(capture_id: str) -> str:
                     code="operator_placeholder_capture_mismatch",
                     severity="failed",
                     message=(f"Placeholder task {placeholder.task_key!r} has no captured Airflow task identity."),
-                    details={"task_key": placeholder.task_key, "operator": placeholder.original_type},
+                    details={
+                        "task_key": placeholder.task_key,
+                        "operator": placeholder.original_type,
+                        "task_path": list(task_path),
+                    },
                 )
             )
             continue
@@ -2993,11 +3062,15 @@ def source_task_id(capture_id: str) -> str:
                     "operator": placeholder.original_type,
                     "capture_id": placeholder_capture_id,
                     "source_task_id": source_task_id(placeholder_capture_id),
+                    "task_path": list(task_path),
                 },
+                identity_discriminator=json.dumps(
+                    [pipeline.name, source_task_id(placeholder_capture_id)],
+                    separators=(",", ":"),
+                ),
             )
         )
 
-    blocking_gaps = [*unsupported_settings, *unresolved]
     if blocking_gaps:
         placeholder_key = "__flowx_source_gaps"
         gap_task = PlaceholderActivity(
diff --git a/src/flowx/sources/airflow/templating.py b/src/flowx/sources/airflow/templating.py
index 49776cd..a2bc606 100644
--- a/src/flowx/sources/airflow/templating.py
+++ b/src/flowx/sources/airflow/templating.py
@@ -286,6 +286,34 @@ def pick(key: str) -> ast.expr | None:
     return result
 
 
+def unrepresented_retry_policy_arguments(
+    dag_default_args: dict[str, ast.expr],
+    task_kwargs: dict[str, ast.expr],
+) -> list[str]:
+    """Returns supplied retry/timeout settings that cannot be lowered exactly."""
+
+    def supplied(key: str) -> ast.expr | None:
+        return task_kwargs.get(key, dag_default_args.get(key))
+
+    unresolved: list[str] = []
+    retries = supplied("retries")
+    if retries is not None and not (
+        isinstance(retries, ast.Constant)
+        and (
+            retries.value is None
+            or (isinstance(retries.value, int) and not isinstance(retries.value, bool) and retries.value >= 0)
+        )
+    ):
+        unresolved.append("retries")
+    for name in ("retry_delay", "execution_timeout"):
+        value = supplied(name)
+        if value is None or (isinstance(value, ast.Constant) and value.value is None):
+            continue
+        if _timedelta_seconds(value) is None:
+            unresolved.append(name)
+    return unresolved
+
+
 def email_on_failure(dag_default_args: dict[str, ast.expr], task_kwargs: dict[str, ast.expr]) -> list[str]:
     """Returns email recipients when email_on_failure is set (for a job-level notification note).
 
@@ -343,7 +371,19 @@ def _trigger_rule_name(task_kwargs: dict[str, ast.expr]) -> str | None:
 
 def trigger_rule_mapping(task_kwargs: dict[str, ast.expr]) -> TriggerRuleMapping:
     """Classifies an Airflow trigger rule as exact, approximate, or unsupported."""
-    rule = _trigger_rule_name(task_kwargs) or "all_success"
+    trigger_rule = task_kwargs.get("trigger_rule")
+    if trigger_rule is None:
+        rule = "all_success"
+    else:
+        resolved_rule = _trigger_rule_name(task_kwargs)
+        if resolved_rule is None:
+            return TriggerRuleMapping(
+                rule=ast.unparse(trigger_rule),
+                outcome=None,
+                status="unsupported",
+                message="The trigger rule cannot be resolved statically.",
+            )
+        rule = resolved_rule
     if rule == "none_failed_min_one_success":
         return TriggerRuleMapping(
             rule=rule,
diff --git a/tests/unit/test_airflow_agentic_resolution.py b/tests/unit/test_airflow_agentic_resolution.py
index 6b251f7..4c04767 100644
--- a/tests/unit/test_airflow_agentic_resolution.py
+++ b/tests/unit/test_airflow_agentic_resolution.py
@@ -8,6 +8,7 @@
 
 import pytest
 
+import flowx.agentic as agentic_contract
 from flowx.adapter.__main__ import main as adapter_main
 from flowx.agentic import AgenticContractError, _validate_candidate, summarize_persisted_agentic_resolutions
 from flowx.bundler.dab_writer import main as package_main
@@ -62,7 +63,32 @@ def _prepare(tmp_path: Path, *, two_tasks: bool = False) -> tuple[Path, Path, di
     return source, output, gaps
 
 
+def _prepare_source(source: Path, output: Path) -> list[dict]:
+    assert airflow_convert(["--source-dir", str(source), "--output-dir", str(output)]) == 0
+    report = output / ".work" / "translation_report.json"
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "prepare",
+                "--source",
+                "airflow",
+                "--source-path",
+                str(source),
+                "--report",
+                str(report),
+                "--output-dir",
+                str(output),
+            ]
+        )
+        == 0
+    )
+    return json.loads((output / ".work" / "agentic" / "gaps.json").read_text(encoding="utf-8"))
+
+
 def _candidate(gap: dict, *, source: str = "print('Migrated from Airflow')\n", status: str = "resolved") -> dict:
+    if not source.startswith("# Databricks notebook source\n"):
+        source = "# Databricks notebook source\n" + source
     generated_file = {
         "path": "task.py",
         "language": "python",
@@ -245,6 +271,180 @@ def test_placeholder_findings_bind_capture_identity_to_source(
     assert all(item["details"]["capture_id"] for item in findings)
 
 
+def test_source_expanded_placeholders_have_unique_gap_fingerprints(tmp_path: Path) -> None:
+    source = tmp_path / "loop.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "with DAG(dag_id='loop') as dag:\n"
+        "    for index in range(2):\n"
+        "        KubernetesPodOperator(task_id=f'pod_{index}', image='python:3.12')\n",
+        encoding="utf-8",
+    )
+
+    pipeline = load_airflow_dag(source)
+    findings = [item for item in pipeline.not_translatable if item["code"] == "operator_placeholder"]
+    gaps = _prepare_source(source, tmp_path / "output")
+
+    assert [item["details"]["source_task_id"] for item in findings] == ["pod_0", "pod_1"]
+    assert len({item["fingerprint"] for item in findings}) == 2
+    assert len({gap["gap_id"] for gap in gaps}) == 2
+
+
+def test_gap_fingerprint_is_stable_when_an_unrelated_source_gap_changes_task_path(tmp_path: Path) -> None:
+    source = tmp_path / "stable.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "with DAG(dag_id='stable') as dag:\n"
+        "    KubernetesPodOperator(task_id='pod', image='python:3.12')\n",
+        encoding="utf-8",
+    )
+    original = next(
+        item for item in load_airflow_dag(source).not_translatable if item["code"] == "operator_placeholder"
+    )
+
+    source.write_text(
+        "from airflow import DAG\n"
+        "with DAG(dag_id='stable', max_active_runs=1) as dag:\n"
+        "    KubernetesPodOperator(task_id='pod', image='python:3.12')\n",
+        encoding="utf-8",
+    )
+    changed = next(item for item in load_airflow_dag(source).not_translatable if item["code"] == "operator_placeholder")
+
+    assert original["details"]["task_path"] == ["tasks", 0]
+    assert changed["details"]["task_path"] == ["tasks", 1]
+    assert changed["fingerprint"] == original["fingerprint"]
+
+
+def test_nested_and_top_level_task_key_collision_keeps_gap_identity_distinct(tmp_path: Path) -> None:
+    source = tmp_path / "nested_collision.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "with DAG(dag_id='nested_collision') as dag:\n"
+        "    top = KubernetesPodOperator(task_id='pod_iteration', image='python:3.12')\n"
+        "    mapped = KubernetesPodOperator.partial(task_id='pod', image='python:3.12').expand(env=['prod'])\n",
+        encoding="utf-8",
+    )
+
+    pipeline = load_airflow_dag(source)
+    findings = [item for item in pipeline.not_translatable if item["code"] == "operator_placeholder"]
+    gaps = _prepare_source(source, tmp_path / "output")
+
+    assert [item["details"]["source_task_id"] for item in findings] == ["pod_iteration", "pod"]
+    assert len({item["fingerprint"] for item in findings}) == 2
+    assert len({tuple(gap["task_path"]) for gap in gaps}) == 2
+    assert {gap["capture_identity"] for gap in gaps} == {"top", "mapped"}
+
+
+def test_gap_envelope_carries_bound_helper_arguments(tmp_path: Path) -> None:
+    source = tmp_path / "helper.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "def make(task_id, image):\n"
+        "    return KubernetesPodOperator(task_id=task_id, image=image)\n"
+        "with DAG(dag_id='helper') as dag:\n"
+        "    pod = make('helper_pod', 'python:3.12')\n",
+        encoding="utf-8",
+    )
+
+    gap = _prepare_source(source, tmp_path / "output")[0]
+    arguments = {item["name"]: item for item in gap["arguments"]}
+
+    assert arguments["task_id"]["source_expression"] == "'helper_pod'"
+    assert arguments["image"]["source_expression"] == "'python:3.12'"
+
+
+def test_gap_envelope_carries_statically_bound_operator_arguments(tmp_path: Path) -> None:
+    source = tmp_path / "constants.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "IMAGE = 'python:3.12'\n"
+        "with DAG(dag_id='constants') as dag:\n"
+        "    pod = KubernetesPodOperator(task_id='pod', image=IMAGE)\n",
+        encoding="utf-8",
+    )
+
+    gap = _prepare_source(source, tmp_path / "output")[0]
+    arguments = {item["name"]: item for item in gap["arguments"]}
+
+    assert arguments["image"]["source_expression"] == "'python:3.12'"
+    assert "image=IMAGE" in gap["raw_definition"]["source"]
+
+
+def test_taskflow_gap_arguments_come_from_invocation_not_callable_body(tmp_path: Path) -> None:
+    source = tmp_path / "taskflow.py"
+    source.write_text(
+        "from airflow.decorators import dag, task\n"
+        "@task.branch\n"
+        "def choose(value):\n"
+        "    print(value)\n"
+        "    return 'next'\n"
+        "@dag(dag_id='taskflow')\n"
+        "def workflow():\n"
+        "    choose('selected')\n"
+        "workflow()\n",
+        encoding="utf-8",
+    )
+
+    gap = _prepare_source(source, tmp_path / "output")[0]
+
+    assert gap["arguments"] == [{"name": "$arg0", "source_expression": "'selected'", "preserved_by_flowx": False}]
+
+
+def test_only_actually_preserved_policy_arguments_are_flowx_owned(tmp_path: Path) -> None:
+    source = tmp_path / "policy.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='policy') as dag:\n"
+        "    BashOperator(task_id='work', bash_command='echo hi', retries=2, pool='critical', trigger_rule='always')\n",
+        encoding="utf-8",
+    )
+
+    gap = _prepare_source(source, tmp_path / "output")[0]
+    arguments = {item["name"]: item["preserved_by_flowx"] for item in gap["arguments"]}
+
+    assert arguments == {
+        "task_id": True,
+        "bash_command": False,
+        "retries": True,
+        "pool": False,
+        "trigger_rule": False,
+    }
+
+
+def test_classic_mapped_placeholder_preserves_retry_policy_claimed_by_flowx(tmp_path: Path) -> None:
+    source = tmp_path / "mapped_policy.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "with DAG(dag_id='mapped_policy') as dag:\n"
+        "    pod = KubernetesPodOperator.partial(task_id='pod', retries=2).expand(image=['a', 'b'])\n",
+        encoding="utf-8",
+    )
+
+    pipeline = load_airflow_dag(source)
+    inner = pipeline.tasks[0].inner_activities[0]
+    gap = _prepare_source(source, tmp_path / "output")[0]
+    arguments = {item["name"]: item for item in gap["arguments"]}
+
+    assert inner.max_retries == 2
+    assert arguments["retries"]["preserved_by_flowx"] is True
+
+
+def test_unlowered_dynamic_retry_policy_is_not_claimed_as_preserved(tmp_path: Path) -> None:
+    source = tmp_path / "dynamic_policy.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "with DAG(dag_id='dynamic_policy') as dag:\n"
+        "    pod = KubernetesPodOperator(task_id='pod', image='python:3.12', retries=get_retries())\n",
+        encoding="utf-8",
+    )
+
+    gap = _prepare_source(source, tmp_path / "output")[0]
+    arguments = {item["name"]: item for item in gap["arguments"]}
+
+    assert arguments["retries"]["preserved_by_flowx"] is False
+
+
 def test_resolve_agentic_is_explicitly_airflow_only(tmp_path: Path, capsys):
     exit_code = adapter_main(
         [
@@ -322,6 +522,67 @@ def test_stage_rejects_unresolved_jinja_provider_drift_and_file_hash_mismatch(tm
     assert "sha256 does not match" in capsys.readouterr().err
 
 
+def test_stage_rejects_python_script_without_databricks_notebook_marker(tmp_path: Path, capsys) -> None:
+    _, output, gaps = _prepare(tmp_path)
+    candidate = _candidate(gaps[0], source="print('plain script')\n")
+    content = "print('plain script')\n"
+    candidate["generated_files"][0]["content"] = content
+    candidate["generated_files"][0]["sha256"] = hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+    assert _stage(output, candidate) == 1
+    assert "Databricks notebook source marker" in capsys.readouterr().err
+
+
+def test_stage_rejects_duplicate_candidates_for_one_gap(tmp_path: Path, capsys) -> None:
+    _, output, gaps = _prepare(tmp_path)
+    first = tmp_path / "first.json"
+    second = tmp_path / "second.json"
+    first.write_text(json.dumps(_candidate(gaps[0], source="print('first')\n")), encoding="utf-8")
+    second.write_text(json.dumps(_candidate(gaps[0], source="print('second')\n")), encoding="utf-8")
+
+    exit_code = adapter_main(
+        [
+            "resolve-agentic",
+            "stage",
+            "--source",
+            "airflow",
+            "--output-dir",
+            str(output),
+            "--candidate",
+            str(first),
+            "--candidate",
+            str(second),
+        ]
+    )
+
+    assert exit_code == 1
+    assert "duplicate candidate" in capsys.readouterr().err.lower()
+    assert not list((output / ".work" / "agentic" / "candidates").iterdir())
+
+
+def test_stage_requires_dynamic_references_in_task_parameters_not_source_files(tmp_path: Path, capsys) -> None:
+    _, output, gaps = _prepare(tmp_path)
+    source = "print('{{job.parameters.env}}')\n"
+    candidate = _candidate(gaps[0], source=source)
+    candidate["replacement"]["base_parameters"] = {"env": "{{job.parameters.env}}"}
+
+    assert _stage(output, candidate) == 1
+    assert "cannot contain Databricks dynamic references" in capsys.readouterr().err
+
+    valid = _candidate(gaps[0], source="print(dbutils.widgets.get('env'))\n")
+    valid["replacement"]["base_parameters"] = {"env": "{{job.parameters.env}}"}
+    assert _stage(output, valid) == 0
+
+
+def test_stage_does_not_misclassify_airflow_input_names_as_dynamic_references(tmp_path: Path, capsys) -> None:
+    _, output, gaps = _prepare(tmp_path)
+    candidate = _candidate(gaps[0])
+    candidate["replacement"]["base_parameters"] = {"path": "{{ input_file }}"}
+
+    assert _stage(output, candidate) == 1
+    assert "unresolved Airflow Jinja" in capsys.readouterr().err
+
+
 def test_pinned_v020_provider_fixtures_satisfy_the_flowx_contract() -> None:
     root = Path(__file__).parents[2] / "skills" / "flowx-resolve-airflow-gaps" / "references" / "airflow-to-dabs-v0.2.0"
     provider = json.loads((root / "provider.json").read_text(encoding="utf-8"))
@@ -373,6 +634,50 @@ def test_apply_rebuilds_from_baseline_and_preserves_graph_policy(tmp_path: Path)
     assert (output / "metadata" / "agentic" / "accepted_resolutions.json").exists()
 
 
+@pytest.mark.parametrize(
+    ("field", "value"),
+    [
+        ("task_key", "HIJACKED"),
+        ("depends_on", [{"task_key": "missing"}]),
+        ("max_retries", 99),
+    ],
+)
+def test_post_apply_proof_rejects_graph_or_policy_mutation(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    capsys: pytest.CaptureFixture[str],
+    field: str,
+    value: object,
+) -> None:
+    _, output, gaps = _prepare(tmp_path)
+    assert _stage(output, _candidate(gaps[0])) == 0
+    original = agentic_contract._build_replacement
+
+    def mutated_replacement(placeholder: dict, candidate: dict) -> dict:
+        task = original(placeholder, candidate)
+        task[field] = value
+        return task
+
+    monkeypatch.setattr(agentic_contract, "_build_replacement", mutated_replacement)
+
+    exit_code = adapter_main(
+        [
+            "resolve-agentic",
+            "apply",
+            "--source",
+            "airflow",
+            "--output-dir",
+            str(output),
+            "--accept-gap",
+            gaps[0]["gap_id"],
+        ]
+    )
+
+    assert exit_code == 1
+    assert "changed task identity, dependencies, or policy" in capsys.readouterr().err
+    assert not (output / ".work" / "translation_report.agentic.json").exists()
+
+
 def test_apply_supports_sql_leaf_payload(tmp_path: Path):
     _, output, gaps = _prepare(tmp_path)
     candidate = _candidate(gaps[0])
@@ -408,6 +713,23 @@ def test_apply_supports_sql_leaf_payload(tmp_path: Path):
     assert task["type"] == "SqlActivity"
     assert task["sql"] == sql
     assert task["warehouse_ref"] == "${var.warehouse_id}"
+    report = output / ".work" / "translation_report.agentic.json"
+    assert (
+        package_main(
+            [
+                "--report",
+                str(report),
+                "--output-dir",
+                str(output),
+                "--no-download-workspace-files",
+            ]
+        )
+        == 0
+    )
+    resource = (output / "resources" / "agentic.yml").read_text(encoding="utf-8")
+    assert "sql_task:" in resource
+    assert "../src/sql/pod.sql" in resource
+    assert (output / "src" / "sql" / "pod.sql").read_text(encoding="utf-8") == sql
 
 
 def test_nested_for_each_resolution_preserves_enclosing_control_flow(tmp_path: Path):
@@ -747,6 +1069,71 @@ def test_reviewed_resolution_evidence_drives_honest_runnable_coverage(tmp_path:
     assert row["reconciliation_status"] == "verified_with_reviewed_resolutions"
 
 
+def test_reporting_keeps_non_resolver_source_gaps_unreviewed(tmp_path: Path) -> None:
+    source = tmp_path / "mixed_gaps.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "with DAG(dag_id='mixed_gaps') as dag:\n"
+        "    pod = KubernetesPodOperator(task_id='pod', image='python:3.12')\n"
+        "    for item in runtime_values:\n"
+        "        KubernetesPodOperator(task_id=f'dynamic_{item}', image='python:3.12')\n",
+        encoding="utf-8",
+    )
+    output = tmp_path / "output"
+    gaps = _prepare_source(source, output)
+    assert len(gaps) == 1
+    assert _stage(output, _candidate(gaps[0])) == 0
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+    baseline = json.loads((output / ".work" / "agentic" / "baseline.json").read_text(encoding="utf-8"))
+    metadata = output / "metadata"
+    (metadata / "inventory.json").write_text(
+        json.dumps(
+            {
+                "source": "airflow",
+                "pipelines": [
+                    {
+                        "name": "mixed_gaps",
+                        "activities": [],
+                        "audited_activity_count": baseline["audit"]["audited_activity_count"],
+                        "deterministic_count": baseline["audit"]["deterministic_count"],
+                        "agentic_count": baseline["audit"]["agentic_count"],
+                        "failed_count": baseline["audit"]["failed_count"],
+                        "excluded_count": 0,
+                        "reconciliation_status": baseline["reconciliation_status"],
+                        "migration_status": "included",
+                        "findings": baseline["not_translatable"],
+                    }
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    row = build_coverage_rows(metadata)[0]
+
+    assert json.loads(row["agentic_resolution_outcomes"]) == {
+        "resolved": 1,
+        "needs_input": 0,
+        "deferred": 0,
+        "unreviewed": 1,
+    }
+    assert row["unresolved_agentic_activities"] == 1
+
+
 def test_reporting_rejects_duplicate_hash_valid_agentic_evidence(tmp_path: Path) -> None:
     _, output, gaps = _prepare(tmp_path)
     assert _stage(output, _candidate(gaps[0])) == 0
diff --git a/tests/unit/test_airflow_production_readiness.py b/tests/unit/test_airflow_production_readiness.py
index fa6be39..dbf14a1 100644
--- a/tests/unit/test_airflow_production_readiness.py
+++ b/tests/unit/test_airflow_production_readiness.py
@@ -356,3 +356,37 @@ def test_unconsumed_operator_arguments_become_placeholder() -> None:
     assert classified["task_id"]["rationale"] == "capture_identity"
     assert classified["bash_command"]["rationale"] == "operator_adapter"
     assert classified["pool"]["status"] == "unconsumed"
+
+
+def test_unlowerable_retry_policy_becomes_placeholder(tmp_path: Path) -> None:
+    source = tmp_path / "dynamic_retry.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='dynamic_retry') as dag:\n"
+        "    BashOperator(task_id='work', bash_command='echo hi', retries=get_retries())\n",
+        encoding="utf-8",
+    )
+
+    pipeline = load_airflow_dag(source)
+
+    assert isinstance(pipeline.tasks[0], PlaceholderActivity)
+    finding = next(item for item in pipeline.not_translatable if item["code"] == "unrepresented_task_policy")
+    assert finding["details"]["arguments"] == ["retries"]
+
+
+def test_dynamic_trigger_rule_becomes_placeholder(tmp_path: Path) -> None:
+    source = tmp_path / "dynamic_trigger.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='dynamic_trigger') as dag:\n"
+        "    BashOperator(task_id='work', bash_command='echo hi', trigger_rule=get_rule())\n",
+        encoding="utf-8",
+    )
+
+    pipeline = load_airflow_dag(source)
+
+    assert isinstance(pipeline.tasks[0], PlaceholderActivity)
+    finding = next(item for item in pipeline.not_translatable if item["code"] == "unsupported_trigger_rule")
+    assert finding["details"]["task_key"] == "work"

From 164fd43e104d29e7fc072771c38b369afc274351 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Sun, 9 Aug 2026 13:38:17 -0700
Subject: [PATCH 59/77] Complete Airflow resolution review workflow

---
 skills/flowx-resolve-airflow-gaps/SKILL.md    |  25 +-
 src/flowx/adapter/__main__.py                 |  30 +-
 src/flowx/agentic.py                          | 291 ++++++++++++++++--
 src/flowx/mcp/server.py                       |   9 +-
 src/flowx/reporting/coverage.py               |   4 +-
 tests/unit/test_airflow_agentic_resolution.py | 173 ++++++++++-
 tests/unit/test_mcp_source_routing.py         |  17 +
 tests/unit/test_reporting_coverage.py         |   1 +
 8 files changed, 502 insertions(+), 48 deletions(-)

diff --git a/skills/flowx-resolve-airflow-gaps/SKILL.md b/skills/flowx-resolve-airflow-gaps/SKILL.md
index f027fd5..df98dd5 100644
--- a/skills/flowx-resolve-airflow-gaps/SKILL.md
+++ b/skills/flowx-resolve-airflow-gaps/SKILL.md
@@ -71,7 +71,9 @@ SQL named parameters. Comments may mention Airflow for provenance.
 
 Stage validates fingerprints, source/report hashes, the pinned provider version, argument
 disposition, generated-file hashes, Python imports, templates, and the constrained replacement
-schema. Tampering after staging is a hard failure.
+schema. Tampering after staging is a hard failure. Identical content is idempotent; use `--replace`
+to replace different content for an already-staged gap. Stage returns an immutable, hash-addressed
+review-manifest path for the complete staged candidate set.
 
 MCP accepts candidate objects inline with `action="stage"` and `candidates=[...]`.
 
@@ -88,15 +90,30 @@ provider version, and model provenance. Apply only the fingerprints the user acc
 ```
 
 `--accept-all` is only for replaying candidates already staged in a prior step; never combine it
-with live candidate generation. Apply always rebuilds from the immutable deterministic baseline,
+with live candidate generation. It requires `--review-manifest ` and rejects the operation if
+the reviewed candidate IDs or hashes no longer exactly match the staged set. Apply always rebuilds from the immutable deterministic baseline,
 then proves task count, location, keys, dependencies, policy, and enclosing control flow are
 unchanged. It writes `.work/translation_report.agentic.json` and keeps accepted evidence under
 `metadata/agentic/` so package pruning does not destroy provenance.
 
+To decline every staged candidate after reviewing that exact set, use:
+
+```bash
+"$PY" -m flowx.adapter resolve-agentic apply \
+  --source airflow \
+  --output-dir  \
+  --review-complete \
+  --review-manifest 
+```
+
+This records the staged candidates as declined. Prepared gaps without a staged candidate remain
+unreviewed; the flag makes no claim about artifacts that did not exist.
+
 Use a reduced `--accept-gap` allowlist to reject selected candidates while retaining others. Use
 `--reset` to discard all accepted resolutions and start over from the deterministic baseline. A
-source edit after prepare is a hard failure: rerun convert and prepare instead of applying stale
-results.
+normal apply after a source edit is a hard failure: rerun convert and prepare instead of applying
+stale results. Reset is the recovery path and restores the durable baseline even after source drift
+or normal `.work/` pruning.
 
 Package the reviewed report explicitly:
 
diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py
index 25e826a..84cf90e 100644
--- a/src/flowx/adapter/__main__.py
+++ b/src/flowx/adapter/__main__.py
@@ -117,12 +117,18 @@ def _run_resolve_agentic(args: argparse.Namespace) -> int:
                 dbt_mode=args.dbt_mode,
             )
         elif args.action == "stage":
-            payload = stage_airflow_resolutions(output_dir=args.output_dir, candidate_paths=args.candidate)
+            payload = stage_airflow_resolutions(
+                output_dir=args.output_dir,
+                candidate_paths=args.candidate,
+                replace=args.replace,
+            )
         else:
             payload = apply_airflow_resolutions(
                 output_dir=args.output_dir,
                 accepted_gap_ids=args.accept_gap,
                 accept_all=args.accept_all,
+                review_complete=args.review_complete,
+                review_manifest_path=args.review_manifest,
                 reset=args.reset,
                 source_path=args.source_path,
             )
@@ -428,13 +434,33 @@ def _build_parser() -> argparse.ArgumentParser:
         default=[],
         help="Provider-authored AgenticResolution JSON to validate and stage. Repeatable.",
     )
+    resolve_agentic.add_argument(
+        "--replace",
+        action="store_true",
+        help="Replace a different candidate already staged for the same gap.",
+    )
     resolve_agentic.add_argument(
         "--accept-gap",
         action="append",
         default=[],
         help="Prepared gap fingerprint to accept. Repeatable; the full allowlist is replayed from baseline.",
     )
-    resolve_agentic.add_argument("--accept-all", action="store_true", help="Accept all already-staged candidates.")
+    resolve_agentic.add_argument(
+        "--accept-all",
+        action="store_true",
+        help="Accept all candidates in an exact prior --review-manifest.",
+    )
+    resolve_agentic.add_argument(
+        "--review-complete",
+        action="store_true",
+        help="Record every candidate in an exact prior review manifest as reviewed and declined.",
+    )
+    resolve_agentic.add_argument(
+        "--review-manifest",
+        type=Path,
+        default=None,
+        help="Hash-bound staged-candidate manifest returned by stage.",
+    )
     resolve_agentic.add_argument("--reset", action="store_true", help="Restore the immutable deterministic baseline.")
     resolve_agentic.add_argument(
         "--dbt-mode",
diff --git a/src/flowx/agentic.py b/src/flowx/agentic.py
index ffbc64a..98247af 100644
--- a/src/flowx/agentic.py
+++ b/src/flowx/agentic.py
@@ -126,6 +126,8 @@ class PersistedResolutionEvidence:
     provider_version: str
     gaps: list[dict[str, Any]]
     resolutions: list[StagedResolution]
+    reviewed_resolutions: list[StagedResolution]
+    decisions: list[dict[str, str]]
     expected_report: dict[str, Any]
 
 
@@ -217,7 +219,12 @@ def prepare_airflow_resolutions(
     }
 
 
-def stage_airflow_resolutions(*, output_dir: Path, candidate_paths: list[Path]) -> dict[str, Any]:
+def stage_airflow_resolutions(
+    *,
+    output_dir: Path,
+    candidate_paths: list[Path],
+    replace: bool = False,
+) -> dict[str, Any]:
     """Validates provider candidates and records their immutable hashes in the agentic workspace."""
     workspace = _workspace(output_dir)
     manifest, gaps = _load_workspace(workspace)
@@ -243,13 +250,26 @@ def stage_airflow_resolutions(*, output_dir: Path, candidate_paths: list[Path])
     index = _load_candidate_index(workspace, valid_gap_ids=set(gap_by_id))
     for gap_id, resolution in staged.items():
         destination = candidates_dir / f"{gap_id}.json"
-        destination.write_bytes(_json_bytes(resolution.candidate))
+        existing = index.get(gap_id)
+        if existing is not None and existing["sha256"] != resolution.sha256 and not replace:
+            raise AgenticContractError(
+                f"A different candidate is already staged for gap {gap_id}; use --replace to replace it"
+            )
+        if existing is not None and destination.exists() and _sha256_file(destination) != existing["sha256"]:
+            raise AgenticContractError(f"staged candidate was modified after validation: {gap_id}")
+        _write_json_atomic(destination, resolution.candidate)
         index[gap_id] = {
             "sha256": resolution.sha256,
             "status": resolution.candidate["status"],
         }
     _write_json(workspace / "candidate_index.json", index)
-    return {"status": "staged", "staged": sorted(staged), "candidate_count": len(index)}
+    review_manifest_path = _write_review_manifest(workspace, manifest=manifest, index=index)
+    return {
+        "status": "staged",
+        "staged": sorted(staged),
+        "candidate_count": len(index),
+        "review_manifest": str(review_manifest_path),
+    }
 
 
 def apply_airflow_resolutions(
@@ -257,13 +277,18 @@ def apply_airflow_resolutions(
     output_dir: Path,
     accepted_gap_ids: list[str] | None = None,
     accept_all: bool = False,
+    review_complete: bool = False,
+    review_manifest_path: Path | None = None,
     reset: bool = False,
     source_path: Path | None = None,
 ) -> dict[str, Any]:
     """Rebuilds an agentic report from the immutable baseline and the declarative acceptance set."""
-    if sum(bool(option) for option in (accepted_gap_ids, accept_all, reset)) != 1:
-        raise AgenticContractError("Choose exactly one of --accept-gap, --accept-all, or --reset")
+    if sum(bool(option) for option in (accepted_gap_ids, accept_all, review_complete, reset)) != 1:
+        raise AgenticContractError("Choose exactly one of --accept-gap, --accept-all, --review-complete, or --reset")
     output_dir = output_dir.resolve()
+    if reset:
+        return _reset_airflow_resolutions(output_dir)
+
     workspace = _workspace(output_dir)
     manifest, gaps = _load_workspace(workspace)
     baseline_path = workspace / "baseline.json"
@@ -282,41 +307,203 @@ def apply_airflow_resolutions(
 
     gap_by_id = {gap["gap_id"]: gap for gap in gaps}
     index = _load_candidate_index(workspace, valid_gap_ids=set(gap_by_id))
-    selected_ids = [] if reset else sorted(index) if accept_all else list(dict.fromkeys(accepted_gap_ids or []))
+    review_manifest: dict[str, Any] | None = None
+    review_manifest_bytes: bytes | None = None
+    if accept_all or review_complete:
+        if review_manifest_path is None:
+            raise AgenticContractError("--accept-all and --review-complete require --review-manifest")
+        review_manifest, review_manifest_bytes = _load_review_manifest(
+            review_manifest_path,
+            manifest=manifest,
+            index=index,
+        )
+        if not index:
+            raise AgenticContractError("A reviewed operation requires at least one staged candidate")
+    elif review_manifest_path is not None:
+        raise AgenticContractError("--review-manifest is only valid with --accept-all or --review-complete")
+
+    selected_ids = (
+        sorted(index) if accept_all else [] if review_complete else list(dict.fromkeys(accepted_gap_ids or []))
+    )
     missing = sorted(set(selected_ids) - set(index))
     if missing:
         raise AgenticContractError(f"No staged candidate exists for gap(s): {', '.join(missing)}")
 
-    selected: list[StagedResolution] = []
-    for gap_id in selected_ids:
+    reviewed_ids = sorted(index) if review_manifest is not None else selected_ids
+    reviewed: dict[str, StagedResolution] = {}
+    for gap_id in reviewed_ids:
         candidate_path = workspace / "candidates" / f"{gap_id}.json"
         candidate_bytes = candidate_path.read_bytes()
         if _sha256_bytes(candidate_bytes) != index[gap_id]["sha256"]:
             raise AgenticContractError(f"staged candidate was modified after validation: {gap_id}")
         candidate = json.loads(candidate_bytes)
-        selected.append(_validate_candidate(candidate, gap_by_id=gap_by_id, manifest=manifest))
+        reviewed[gap_id] = _validate_candidate(candidate, gap_by_id=gap_by_id, manifest=manifest)
+
+    selected = [reviewed[gap_id] for gap_id in selected_ids]
+    decisions = [
+        {
+            "gap_id": gap_id,
+            "candidate_sha256": reviewed[gap_id].sha256,
+            "decision": "accepted" if gap_id in selected_ids else "declined",
+        }
+        for gap_id in reviewed_ids
+    ]
 
     applied = _apply_to_baseline(baseline, selected)
     report_path = output_dir / ".work" / "translation_report.agentic.json"
+    _persist_agentic_evidence(
+        output_dir=output_dir,
+        material_dir=workspace,
+        baseline_bytes=baseline_bytes,
+        reviewed=list(reviewed.values()),
+        selected=selected,
+        decisions=decisions,
+        review_manifest_bytes=review_manifest_bytes,
+    )
     _write_json_atomic(report_path, applied)
-
-    evidence = output_dir / "metadata" / "agentic"
-    evidence.mkdir(parents=True, exist_ok=True)
-    (evidence / "baseline.json").write_bytes(baseline_bytes)
-    (evidence / "gaps.json").write_bytes((workspace / "gaps.json").read_bytes())
-    (evidence / "manifest.json").write_bytes((workspace / "manifest.json").read_bytes())
-    accepted_payload = {
-        "contract_version": CONTRACT_VERSION,
-        "candidates": [resolution.candidate for resolution in selected],
-    }
-    _write_json(evidence / "accepted_resolutions.json", accepted_payload)
     return {
-        "status": "reset" if reset else "applied",
+        "status": "review_complete" if review_complete else "applied",
         "accepted_gap_ids": selected_ids,
+        "declined_gap_ids": [decision["gap_id"] for decision in decisions if decision["decision"] == "declined"],
         "report_path": str(report_path),
     }
 
 
+def _reset_airflow_resolutions(output_dir: Path) -> dict[str, Any]:
+    """Restores the durable deterministic baseline without consulting mutable live source."""
+    workspace = _workspace(output_dir)
+    evidence = output_dir / "metadata" / "agentic"
+    if workspace.is_dir():
+        manifest, _ = _load_workspace(workspace)
+        material_dir = workspace
+        _verify_snapshot(workspace, manifest)
+    elif evidence.is_dir():
+        _load_persisted_agentic_evidence(evidence)
+        manifest = _read_json_object(evidence / "manifest.json")
+        material_dir = evidence
+    else:
+        raise AgenticContractError("No durable agentic baseline exists; run prepare first")
+
+    baseline_bytes = (material_dir / "baseline.json").read_bytes()
+    if _sha256_bytes(baseline_bytes) != manifest.get("baseline_report_sha256"):
+        raise AgenticContractError("The immutable deterministic baseline was modified after prepare")
+    baseline = json.loads(baseline_bytes)
+    _require_airflow_baseline(baseline)
+    _persist_agentic_evidence(
+        output_dir=output_dir,
+        material_dir=material_dir,
+        baseline_bytes=baseline_bytes,
+        reviewed=[],
+        selected=[],
+        decisions=[],
+        review_manifest_bytes=None,
+    )
+
+    if workspace.is_dir():
+        candidates = workspace / "candidates"
+        if candidates.exists():
+            shutil.rmtree(candidates)
+        candidates.mkdir()
+        _write_json(workspace / "candidate_index.json", {})
+        review_manifests = workspace / "review_manifests"
+        if review_manifests.exists():
+            shutil.rmtree(review_manifests)
+
+    report_path = output_dir / ".work" / "translation_report.agentic.json"
+    _write_json_atomic(report_path, baseline)
+    return {"status": "reset", "accepted_gap_ids": [], "declined_gap_ids": [], "report_path": str(report_path)}
+
+
+def _persist_agentic_evidence(
+    *,
+    output_dir: Path,
+    material_dir: Path,
+    baseline_bytes: bytes,
+    reviewed: list[StagedResolution],
+    selected: list[StagedResolution],
+    decisions: list[dict[str, str]],
+    review_manifest_bytes: bytes | None,
+) -> None:
+    """Keeps replayable source, baseline, candidates, and review decisions outside transient work state."""
+    metadata_dir = output_dir / "metadata"
+    metadata_dir.mkdir(parents=True, exist_ok=True)
+    target = metadata_dir / "agentic"
+    with tempfile.TemporaryDirectory(prefix=".agentic-evidence-", dir=metadata_dir) as temporary:
+        staging = Path(temporary) / "agentic"
+        staging.mkdir()
+        (staging / "baseline.json").write_bytes(baseline_bytes)
+        (staging / "gaps.json").write_bytes((material_dir / "gaps.json").read_bytes())
+        (staging / "manifest.json").write_bytes((material_dir / "manifest.json").read_bytes())
+        shutil.copytree(material_dir / "source", staging / "source")
+        _write_json(
+            staging / "reviewed_candidates.json",
+            {"contract_version": CONTRACT_VERSION, "candidates": [item.candidate for item in reviewed]},
+        )
+        _write_json(
+            staging / "accepted_resolutions.json",
+            {"contract_version": CONTRACT_VERSION, "candidates": [item.candidate for item in selected]},
+        )
+        _write_json(
+            staging / "review_decisions.json",
+            {"contract_version": CONTRACT_VERSION, "decisions": decisions},
+        )
+        if review_manifest_bytes is not None:
+            (staging / "review_manifest.json").write_bytes(review_manifest_bytes)
+        if target.exists():
+            shutil.rmtree(target)
+        shutil.move(str(staging), target)
+
+
+def _review_manifest_payload(manifest: dict[str, Any], index: dict[str, dict[str, str]]) -> dict[str, Any]:
+    return {
+        "contract_version": CONTRACT_VERSION,
+        "source": "airflow",
+        "baseline_report_sha256": manifest["baseline_report_sha256"],
+        "gaps_sha256": manifest["gaps_sha256"],
+        "provider": manifest["provider"],
+        "candidates": [
+            {"gap_id": gap_id, "sha256": index[gap_id]["sha256"], "status": index[gap_id]["status"]}
+            for gap_id in sorted(index)
+        ],
+    }
+
+
+def _write_review_manifest(
+    workspace: Path,
+    *,
+    manifest: dict[str, Any],
+    index: dict[str, dict[str, str]],
+) -> Path:
+    payload = _review_manifest_payload(manifest, index)
+    payload_bytes = _json_bytes(payload)
+    digest = _sha256_bytes(payload_bytes)
+    path = workspace / "review_manifests" / f"{digest}.json"
+    if path.exists() and path.read_bytes() != payload_bytes:
+        raise AgenticContractError("Hash-addressed review manifest contains different content")
+    path.parent.mkdir(exist_ok=True)
+    path.write_bytes(payload_bytes)
+    return path
+
+
+def _load_review_manifest(
+    path: Path,
+    *,
+    manifest: dict[str, Any],
+    index: dict[str, dict[str, str]],
+) -> tuple[dict[str, Any], bytes]:
+    try:
+        payload_bytes = path.read_bytes()
+        payload = json.loads(payload_bytes)
+    except OSError as error:
+        raise AgenticContractError(f"Could not read review manifest: {error}") from error
+    except json.JSONDecodeError as error:
+        raise AgenticContractError(f"Review manifest contains invalid JSON: {error}") from error
+    expected = _review_manifest_payload(manifest, index)
+    if payload != expected:
+        raise AgenticContractError("Review manifest does not exactly match the currently staged candidate set")
+    return expected, payload_bytes
+
+
 def validate_persisted_agentic_report(report: dict[str, Any], *, evidence_dir: Path) -> list[str]:
     """Replays accepted candidates from kept evidence and compares the exact expected report."""
     try:
@@ -351,6 +538,17 @@ def summarize_persisted_agentic_resolutions(evidence_dir: Path) -> dict[str, Any
             raise AgenticContractError(f"agentic resolution over-accounts pipeline {pipeline_name!r}")
         outcomes["unreviewed"] -= 1
         outcomes[str(resolution.candidate["status"])] += 1
+    accepted_ids = {resolution.gap["gap_id"] for resolution in evidence.resolutions}
+    for decision in evidence.decisions:
+        if decision["decision"] != "declined" or decision["gap_id"] in accepted_ids:
+            continue
+        resolution = next(item for item in evidence.reviewed_resolutions if item.gap["gap_id"] == decision["gap_id"])
+        pipeline_name = str(resolution.gap["pipeline_name"])
+        outcomes = pipeline_outcomes.setdefault(pipeline_name, _empty_resolution_outcomes())
+        if outcomes["unreviewed"] <= 0:
+            raise AgenticContractError(f"agentic review over-accounts pipeline {pipeline_name!r}")
+        outcomes["unreviewed"] -= 1
+        outcomes["declined"] += 1
     return {
         "provider_version": evidence.provider_version,
         "pipelines": pipeline_outcomes,
@@ -365,6 +563,8 @@ def _load_persisted_agentic_evidence(evidence_dir: Path) -> PersistedResolutionE
         gaps = json.loads(gaps_bytes)
         manifest = _read_json_object(evidence_dir / "manifest.json")
         accepted = _read_json_object(evidence_dir / "accepted_resolutions.json")
+        reviewed = _read_json_object(evidence_dir / "reviewed_candidates.json")
+        decision_manifest = _read_json_object(evidence_dir / "review_decisions.json")
     except (OSError, json.JSONDecodeError, AgenticContractError) as error:
         raise AgenticContractError(str(error)) from error
 
@@ -386,6 +586,10 @@ def _load_persisted_agentic_evidence(evidence_dir: Path) -> PersistedResolutionE
         raise AgenticContractError("agentic resolution manifest has an unsupported contract, source, or provider")
     if accepted.get("contract_version") != CONTRACT_VERSION:
         raise AgenticContractError("accepted_resolutions.json has an unsupported contract_version")
+    if reviewed.get("contract_version") != CONTRACT_VERSION:
+        raise AgenticContractError("reviewed_candidates.json has an unsupported contract_version")
+    if decision_manifest.get("contract_version") != CONTRACT_VERSION:
+        raise AgenticContractError("review_decisions.json has an unsupported contract_version")
     if not isinstance(gaps, list):
         raise AgenticContractError("gaps.json must contain a list")
 
@@ -409,6 +613,9 @@ def _load_persisted_agentic_evidence(evidence_dir: Path) -> PersistedResolutionE
         raise AgenticContractError("agentic resolution manifest has invalid source_files") from error
     if not source_hashes:
         raise AgenticContractError("agentic resolution manifest has no source_files")
+    durable_source_hashes = {relative: _sha256_file(evidence_dir / "source" / relative) for relative in source_hashes}
+    if durable_source_hashes != source_hashes:
+        raise AgenticContractError("durable Airflow source snapshot does not match its manifest")
     expected_gaps = _build_gap_envelopes(
         baseline,
         baseline_hash=str(manifest["baseline_report_sha256"]),
@@ -417,9 +624,22 @@ def _load_persisted_agentic_evidence(evidence_dir: Path) -> PersistedResolutionE
     if gaps != expected_gaps:
         raise AgenticContractError("persisted gap envelopes do not match the immutable baseline")
 
+    reviewed_candidates = reviewed.get("candidates")
     candidates = accepted.get("candidates")
+    if not isinstance(reviewed_candidates, list):
+        raise AgenticContractError("reviewed_candidates.json must contain a candidates list")
     if not isinstance(candidates, list):
         raise AgenticContractError("accepted_resolutions.json must contain a candidates list")
+    reviewed_resolutions: list[StagedResolution] = []
+    reviewed_by_id: dict[str, StagedResolution] = {}
+    for candidate in reviewed_candidates:
+        resolution = _validate_candidate(candidate, gap_by_id=gap_by_id, manifest=manifest)
+        gap_id = str(resolution.gap["gap_id"])
+        if gap_id in reviewed_by_id:
+            raise AgenticContractError(f"duplicate reviewed resolution for gap_id: {gap_id}")
+        reviewed_by_id[gap_id] = resolution
+        reviewed_resolutions.append(resolution)
+
     resolutions: list[StagedResolution] = []
     accepted_ids: set[str] = set()
     for candidate in candidates:
@@ -430,6 +650,31 @@ def _load_persisted_agentic_evidence(evidence_dir: Path) -> PersistedResolutionE
         accepted_ids.add(gap_id)
         resolutions.append(resolution)
 
+    raw_decisions = decision_manifest.get("decisions")
+    if not isinstance(raw_decisions, list) or not all(isinstance(item, dict) for item in raw_decisions):
+        raise AgenticContractError("review_decisions.json must contain a decisions list")
+    decisions: list[dict[str, str]] = []
+    decision_ids: set[str] = set()
+    for item in raw_decisions:
+        if set(item) != {"gap_id", "candidate_sha256", "decision"}:
+            raise AgenticContractError("review decision contains unsupported fields")
+        gap_id = item.get("gap_id")
+        digest = item.get("candidate_sha256")
+        decision = item.get("decision")
+        if not isinstance(gap_id, str) or gap_id not in reviewed_by_id or gap_id in decision_ids:
+            raise AgenticContractError(f"review decision has an invalid gap_id: {gap_id!r}")
+        if digest != reviewed_by_id[gap_id].sha256:
+            raise AgenticContractError(f"review decision hash does not match candidate: {gap_id}")
+        if decision not in {"accepted", "declined"}:
+            raise AgenticContractError(f"review decision is invalid for gap: {gap_id}")
+        decision_ids.add(gap_id)
+        decisions.append({"gap_id": gap_id, "candidate_sha256": str(digest), "decision": str(decision)})
+    accepted_decision_ids = {item["gap_id"] for item in decisions if item["decision"] == "accepted"}
+    if accepted_ids != accepted_decision_ids:
+        raise AgenticContractError("accepted resolutions do not match the durable review decisions")
+    if set(reviewed_by_id) != decision_ids:
+        raise AgenticContractError("review decisions do not cover every persisted reviewed candidate")
+
     try:
         expected_report = _apply_to_baseline(baseline, resolutions)
     except AgenticContractError as error:
@@ -438,12 +683,14 @@ def _load_persisted_agentic_evidence(evidence_dir: Path) -> PersistedResolutionE
         provider_version=PROVIDER_VERSION,
         gaps=gaps,
         resolutions=resolutions,
+        reviewed_resolutions=reviewed_resolutions,
+        decisions=decisions,
         expected_report=expected_report,
     )
 
 
 def _empty_resolution_outcomes() -> dict[str, int]:
-    return {"resolved": 0, "needs_input": 0, "deferred": 0, "unreviewed": 0}
+    return {"resolved": 0, "needs_input": 0, "deferred": 0, "declined": 0, "unreviewed": 0}
 
 
 def _require_airflow_baseline(payload: Any) -> None:
diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py
index 79e113f..564bee9 100644
--- a/src/flowx/mcp/server.py
+++ b/src/flowx/mcp/server.py
@@ -262,8 +262,14 @@ def _cmd_resolve_agentic(p: dict[str, Any]) -> dict[str, Any]:
         args += ["--accept-gap", gap_id]
     if p.get("accept_all"):
         args.append("--accept-all")
+    if p.get("review_complete"):
+        args.append("--review-complete")
+    if p.get("review_manifest"):
+        args += ["--review-manifest", p["review_manifest"]]
     if p.get("reset"):
         args.append("--reset")
+    if p.get("replace"):
+        args.append("--replace")
 
     raw_candidate_paths = p.get("candidate_paths") or []
     candidate_paths = [raw_candidate_paths] if isinstance(raw_candidate_paths, str) else list(raw_candidate_paths)
@@ -537,7 +543,8 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A
         - "merge_agentic": source(req: "adf"), report_path(req), agentic_results_dir(req), output_path —
           merge ADF agent results. Airflow's legacy name-based merge is disabled; use resolve_agentic.
         - "resolve_agentic": source(req: "airflow"), action(req: prepare | stage | apply), output_dir,
-          airflow_source_path, report_path, candidates, accept_gap | accept_gaps, accept_all, reset —
+          airflow_source_path, report_path, candidates, replace, accept_gap | accept_gaps, accept_all,
+          review_complete, review_manifest, reset —
           prepare, stage, and explicitly apply fingerprint-bound Airflow leaf-gap resolutions.
         - "inspect": report_path(req) — return the full translation-option schema (every option with
           a `show_when` condition) for the agent to walk locally. See "Collecting options" below.
diff --git a/src/flowx/reporting/coverage.py b/src/flowx/reporting/coverage.py
index 5b0dcf9..9fa0a19 100644
--- a/src/flowx/reporting/coverage.py
+++ b/src/flowx/reporting/coverage.py
@@ -140,7 +140,7 @@ def build_coverage_rows(metadata_dir: Path) -> list[dict[str, Any]]:
                 )
                 if not isinstance(outcomes, dict) or any(
                     not isinstance(outcomes.get(key), int)
-                    for key in ("resolved", "needs_input", "deferred", "unreviewed")
+                    for key in ("resolved", "needs_input", "deferred", "declined", "unreviewed")
                 ):
                     raise AgenticContractError(f"invalid agentic resolution outcomes for pipeline {name!r}")
                 if sum(outcomes.values()) != agentic:
@@ -149,7 +149,7 @@ def build_coverage_rows(metadata_dir: Path) -> list[dict[str, Any]]:
                         f"{agentic} agentic activities in pipeline {name!r}"
                     )
             else:
-                outcomes = {"resolved": 0, "needs_input": 0, "deferred": 0, "unreviewed": agentic}
+                outcomes = {"resolved": 0, "needs_input": 0, "deferred": 0, "declined": 0, "unreviewed": agentic}
             resolved_agentic = outcomes["resolved"]
             unresolved_agentic = agentic - resolved_agentic
             runnable_coverage = _runnable_coverage_pct(deterministic, resolved_agentic, total)
diff --git a/tests/unit/test_airflow_agentic_resolution.py b/tests/unit/test_airflow_agentic_resolution.py
index 4c04767..dd73978 100644
--- a/tests/unit/test_airflow_agentic_resolution.py
+++ b/tests/unit/test_airflow_agentic_resolution.py
@@ -4,6 +4,7 @@
 
 import hashlib
 import json
+import shutil
 from pathlib import Path
 
 import pytest
@@ -128,21 +129,36 @@ def _candidate(gap: dict, *, source: str = "print('Migrated from Airflow')\n", s
     return candidate
 
 
-def _stage(output: Path, candidate: dict, *, name: str = "candidate.json") -> int:
+def _stage(output: Path, candidate: dict, *, name: str = "candidate.json", replace: bool = False) -> int:
     candidate_path = output / name
     candidate_path.write_text(json.dumps(candidate, indent=2), encoding="utf-8")
-    return adapter_main(
-        [
-            "resolve-agentic",
-            "stage",
-            "--source",
-            "airflow",
-            "--output-dir",
-            str(output),
-            "--candidate",
-            str(candidate_path),
-        ]
-    )
+    args = [
+        "resolve-agentic",
+        "stage",
+        "--source",
+        "airflow",
+        "--output-dir",
+        str(output),
+        "--candidate",
+        str(candidate_path),
+    ]
+    if replace:
+        args.append("--replace")
+    return adapter_main(args)
+
+
+def _review_manifest(output: Path) -> Path:
+    index = json.loads((output / ".work" / "agentic" / "candidate_index.json").read_text(encoding="utf-8"))
+    expected = [
+        {"gap_id": gap_id, "sha256": entry["sha256"], "status": entry["status"]}
+        for gap_id, entry in sorted(index.items())
+    ]
+    matches = []
+    for path in (output / ".work" / "agentic" / "review_manifests").glob("*.json"):
+        if json.loads(path.read_text(encoding="utf-8"))["candidates"] == expected:
+            matches.append(path)
+    assert len(matches) == 1
+    return matches[0]
 
 
 def _load_tasks(report: Path) -> dict[str, dict]:
@@ -813,7 +829,7 @@ def test_apply_rejects_staged_candidate_tampering(tmp_path: Path, capsys):
 
 
 def test_apply_rejects_malformed_candidate_index(tmp_path: Path, capsys):
-    _, output, _ = _prepare(tmp_path)
+    _, output, gaps = _prepare(tmp_path)
     index = output / ".work" / "agentic" / "candidate_index.json"
     index.write_text(
         json.dumps({"../../outside": {"sha256": "0" * 64, "status": "resolved"}}),
@@ -821,7 +837,16 @@ def test_apply_rejects_malformed_candidate_index(tmp_path: Path, capsys):
     )
 
     exit_code = adapter_main(
-        ["resolve-agentic", "apply", "--source", "airflow", "--output-dir", str(output), "--reset"]
+        [
+            "resolve-agentic",
+            "apply",
+            "--source",
+            "airflow",
+            "--output-dir",
+            str(output),
+            "--accept-gap",
+            gaps[0]["gap_id"],
+        ]
     )
 
     assert exit_code == 1
@@ -857,7 +882,19 @@ def test_reduced_allowlist_restores_unaccepted_placeholder_and_reset_restores_al
         assert _stage(output, _candidate(gap), name=f"candidate-{index}.json") == 0
 
     assert (
-        adapter_main(["resolve-agentic", "apply", "--source", "airflow", "--output-dir", str(output), "--accept-all"])
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-all",
+                "--review-manifest",
+                str(_review_manifest(output)),
+            ]
+        )
         == 0
     )
     applied_report = output / ".work" / "translation_report.agentic.json"
@@ -888,6 +925,107 @@ def test_reduced_allowlist_restores_unaccepted_placeholder_and_reset_restores_al
     assert {task["type"] for task in _load_tasks(applied_report).values()} == {"PlaceholderActivity"}
 
 
+def test_accept_all_requires_an_exact_prior_review_manifest(tmp_path: Path, capsys) -> None:
+    _, output, gaps = _prepare(tmp_path)
+    assert _stage(output, _candidate(gaps[0])) == 0
+
+    assert (
+        adapter_main(["resolve-agentic", "apply", "--source", "airflow", "--output-dir", str(output), "--accept-all"])
+        == 1
+    )
+    assert "require --review-manifest" in capsys.readouterr().err
+
+    stale = _review_manifest(output)
+    changed = _candidate(gaps[0], source="print('replacement')")
+    assert _stage(output, changed) == 1
+    assert "use --replace" in capsys.readouterr().err
+    assert _stage(output, changed, replace=True) == 0
+
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-all",
+                "--review-manifest",
+                str(stale),
+            ]
+        )
+        == 1
+    )
+    assert "does not exactly match" in capsys.readouterr().err
+
+
+def test_review_complete_declines_exact_staged_set_and_leaves_unstaged_gaps_unreviewed(tmp_path: Path) -> None:
+    _, output, gaps = _prepare(tmp_path, two_tasks=True)
+    assert _stage(output, _candidate(gaps[0])) == 0
+
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--review-complete",
+                "--review-manifest",
+                str(_review_manifest(output)),
+            ]
+        )
+        == 0
+    )
+
+    report = output / ".work" / "translation_report.agentic.json"
+    assert {task["type"] for task in _load_tasks(report).values()} == {"PlaceholderActivity"}
+    evidence = output / "metadata" / "agentic"
+    decisions = json.loads((evidence / "review_decisions.json").read_text(encoding="utf-8"))["decisions"]
+    assert decisions == [
+        {
+            "gap_id": gaps[0]["gap_id"],
+            "candidate_sha256": decisions[0]["candidate_sha256"],
+            "decision": "declined",
+        }
+    ]
+    outcomes = summarize_persisted_agentic_resolutions(evidence)["pipelines"]["agentic"]
+    assert outcomes == {"resolved": 0, "needs_input": 0, "deferred": 0, "declined": 1, "unreviewed": 1}
+
+
+def test_reset_uses_durable_baseline_after_source_change_and_work_pruning(tmp_path: Path) -> None:
+    source, output, gaps = _prepare(tmp_path)
+    assert _stage(output, _candidate(gaps[0])) == 0
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+    source.write_text(source.read_text(encoding="utf-8") + "# changed\n", encoding="utf-8")
+    shutil.rmtree(output / ".work")
+
+    assert (
+        adapter_main(["resolve-agentic", "apply", "--source", "airflow", "--output-dir", str(output), "--reset"]) == 0
+    )
+
+    report = output / ".work" / "translation_report.agentic.json"
+    assert _load_tasks(report)["pod"]["type"] == "PlaceholderActivity"
+    assert (output / "metadata" / "agentic" / "source" / "dag.py").exists()
+
+
 @pytest.mark.parametrize("status", ["needs_input", "deferred"])
 def test_unresolved_outcome_is_terminal_and_keeps_the_linked_placeholder(tmp_path: Path, status: str):
     _, output, gaps = _prepare(tmp_path)
@@ -1059,7 +1197,7 @@ def test_reviewed_resolution_evidence_drives_honest_runnable_coverage(tmp_path:
 
     assert summary == {
         "provider_version": "0.2.0",
-        "pipelines": {"agentic": {"resolved": 1, "needs_input": 1, "deferred": 0, "unreviewed": 0}},
+        "pipelines": {"agentic": {"resolved": 1, "needs_input": 1, "deferred": 0, "declined": 0, "unreviewed": 0}},
     }
     assert row["coverage_pct"] == 100.0
     assert row["deterministic_coverage_pct"] == 0.0
@@ -1129,6 +1267,7 @@ def test_reporting_keeps_non_resolver_source_gaps_unreviewed(tmp_path: Path) ->
         "resolved": 1,
         "needs_input": 0,
         "deferred": 0,
+        "declined": 0,
         "unreviewed": 1,
     }
     assert row["unresolved_agentic_activities"] == 1
diff --git a/tests/unit/test_mcp_source_routing.py b/tests/unit/test_mcp_source_routing.py
index ee265eb..4cbd8dc 100644
--- a/tests/unit/test_mcp_source_routing.py
+++ b/tests/unit/test_mcp_source_routing.py
@@ -158,6 +158,23 @@ def test_resolve_agentic_stage_materializes_inline_candidate(captured):
     assert result["ok"] is True
 
 
+def test_resolve_agentic_forwards_review_contract_flags(captured):
+    result = server._cmd_resolve_agentic(
+        {
+            "source": "airflow",
+            "action": "apply",
+            "output_dir": "/tmp/out",
+            "review_complete": True,
+            "review_manifest": "/tmp/review.json",
+        }
+    )
+
+    argv = _argv(captured, "resolve-agentic")
+    assert "--review-complete" in argv
+    assert argv[argv.index("--review-manifest") + 1] == "/tmp/review.json"
+    assert result["ok"] is True
+
+
 def test_resolve_agentic_rejects_adf_without_invoking_adapter(captured):
     result = server._cmd_resolve_agentic({"source": "adf", "action": "prepare", "output_dir": "/tmp/out"})
 
diff --git a/tests/unit/test_reporting_coverage.py b/tests/unit/test_reporting_coverage.py
index 906c1d0..43264c6 100644
--- a/tests/unit/test_reporting_coverage.py
+++ b/tests/unit/test_reporting_coverage.py
@@ -140,6 +140,7 @@ def test_audited_counts_drive_translation_and_deterministic_coverage(tmp_path: P
         "resolved": 0,
         "needs_input": 0,
         "deferred": 0,
+        "declined": 0,
         "unreviewed": 1,
     }
     assert verified["finding_count"] == 1

From 901a1e8437aab57b76ac3e4a40b4ad8b56248804 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Sun, 9 Aug 2026 13:47:03 -0700
Subject: [PATCH 60/77] Complete Airflow agentic gap contract

---
 skills/flowx-resolve-airflow-gaps/SKILL.md    |   2 +-
 .../airflow-to-dabs-v0.2.0/PROFILE.md         |   4 +-
 .../fixtures/gap-deferred.json                |   4 +
 .../fixtures/gap-needs-input.json             |   4 +
 .../fixtures/gap-notebook.json                |   4 +
 .../fixtures/gap-sql.json                     |   4 +
 .../fixtures/resolution-deferred.json         |   4 +
 .../fixtures/resolution-needs-input.json      |   4 +
 .../fixtures/resolution-notebook.json         |   4 +
 .../fixtures/resolution-sql.json              |   4 +
 .../airflow-to-dabs-v0.2.0/provider.json      |   2 +-
 .../references/contract-v1.md                 |  13 +-
 src/flowx/adapter/__main__.py                 |   6 +
 src/flowx/agentic.py                          | 206 ++++++++++++++++--
 src/flowx/bundler/dab_writer.py               |   1 +
 src/flowx/ir_serde.py                         |   2 +
 src/flowx/mcp/server.py                       |   7 +-
 src/flowx/models/ir.py                        |   3 +
 .../activity_preparers/spark_python.py        |   8 +-
 tests/unit/test_airflow_agentic_resolution.py | 143 +++++++++++-
 tests/unit/test_mcp_source_routing.py         |   2 +
 21 files changed, 400 insertions(+), 31 deletions(-)

diff --git a/skills/flowx-resolve-airflow-gaps/SKILL.md b/skills/flowx-resolve-airflow-gaps/SKILL.md
index df98dd5..e332022 100644
--- a/skills/flowx-resolve-airflow-gaps/SKILL.md
+++ b/skills/flowx-resolve-airflow-gaps/SKILL.md
@@ -51,7 +51,7 @@ or compute assumptions cannot be preserved from the envelope alone. Do not prese
 successful example.
 
 Every source argument must appear exactly once in `argument_disposition` as `consumed`,
-`preserved_by_flowx`, or `ignored`. Every disposition needs a rationale; an ignored argument must
+`preserved_by_flowx`, `ignored`, or `needs_input`. Every disposition needs a rationale; an ignored argument must
 state the specific semantic loss. Never include task names, task keys, dependencies, retries,
 timeouts, clusters, schedules, or other graph/policy fields in the replacement.
 
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/PROFILE.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/PROFILE.md
index e321507..d11ae8a 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/PROFILE.md
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/PROFILE.md
@@ -33,7 +33,7 @@ resolution; they do not grant authority to emit jobs, triggers, clusters, pipeli
 
 1. Classify the operator's intent from `operator_fqn`, `raw_definition`, and `arguments`.
 2. Decide the terminal status:
-   - `resolved`: one self-contained Python notebook or SQL file preserves the represented behavior.
+   - `resolved`: one self-contained Python notebook, SQL file, or Spark Python script preserves the represented behavior.
    - `needs_input`: a concrete deployment fact, credential mapping, runtime dependency, or semantic
      choice is required before a safe leaf implementation can be written.
    - `deferred`: a faithful migration requires graph, control-flow, schedule, compute, resource, or
@@ -42,6 +42,7 @@ resolution; they do not grant authority to emit jobs, triggers, clusters, pipeli
    - `consumed`: the generated payload or resolution decision uses it;
    - `preserved_by_flowx`: Flowx retains it as task identity or policy;
    - `ignored`: the resolution intentionally omits it and states the exact behavioral loss.
+   - `needs_input`: the argument depends on a concrete fact the user must provide before resolution.
 4. Enumerate prerequisites, warnings, and semantic deltas. Never hide a dropped behavior in prose or
    omit an argument from the disposition list.
 5. Return one `AgenticResolution` JSON object and no bundle files or graph patches.
@@ -73,6 +74,7 @@ be possible after the user supplies missing information.
 Use only these top-level fields:
 
 - always: `contract_version`, `gap_id`, `status`, `baseline_report_sha256`, `source_sha256`,
+  `task_sha256`, `graph_sha256`, `provider_sha256`, `request_sha256`,
   `provider`, `model`, `argument_disposition`, `prerequisites`, `warnings`, `semantic_deltas`;
 - `resolved`: add `replacement` and `generated_files`;
 - `needs_input` or `deferred`: add `reason` and omit `replacement` and `generated_files`.
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-deferred.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-deferred.json
index 894737e..bd7b72c 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-deferred.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-deferred.json
@@ -11,6 +11,10 @@
   "source_file": "branching.py",
   "source_sha256": "9999999999999999999999999999999999999999999999999999999999999999",
   "baseline_report_sha256": "8888888888888888888888888888888888888888888888888888888888888888",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
   "source_span": {"line": 21, "column": 4, "end_line": 25, "end_column": 5},
   "raw_definition": {
     "operator": "BranchPythonOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-needs-input.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-needs-input.json
index 9c6e6aa..7120dc0 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-needs-input.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-needs-input.json
@@ -11,6 +11,10 @@
   "source_file": "container_workload.py",
   "source_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
   "baseline_report_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
   "source_span": {"line": 14, "column": 4, "end_line": 22, "end_column": 5},
   "raw_definition": {
     "operator": "KubernetesPodOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-notebook.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-notebook.json
index bdb2196..b8ea6fe 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-notebook.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-notebook.json
@@ -11,6 +11,10 @@
   "source_file": "orders.py",
   "source_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
   "baseline_report_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
   "source_span": {"line": 18, "column": 4, "end_line": 24, "end_column": 5},
   "raw_definition": {
     "operator": "SimpleHttpOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-sql.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-sql.json
index c810269..792664a 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-sql.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-sql.json
@@ -11,6 +11,10 @@
   "source_file": "retention.py",
   "source_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
   "baseline_report_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
   "source_span": {"line": 9, "column": 4, "end_line": 15, "end_column": 5},
   "raw_definition": {
     "operator": "SQLExecuteQueryOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-deferred.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-deferred.json
index 944d268..eed0488 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-deferred.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-deferred.json
@@ -4,6 +4,10 @@
   "status": "deferred",
   "baseline_report_sha256": "8888888888888888888888888888888888888888888888888888888888888888",
   "source_sha256": "9999999999999999999999999999999999999999999999999999999999999999",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
   "provider": {
     "name": "airflow-to-dabs",
     "version": "0.2.0",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-needs-input.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-needs-input.json
index c57751d..6c11053 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-needs-input.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-needs-input.json
@@ -4,6 +4,10 @@
   "status": "needs_input",
   "baseline_report_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
   "source_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
   "provider": {
     "name": "airflow-to-dabs",
     "version": "0.2.0",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-notebook.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-notebook.json
index ab673d8..832198f 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-notebook.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-notebook.json
@@ -4,6 +4,10 @@
   "status": "resolved",
   "baseline_report_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
   "source_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
   "provider": {
     "name": "airflow-to-dabs",
     "version": "0.2.0",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-sql.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-sql.json
index 146b209..2bbca8b 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-sql.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-sql.json
@@ -4,6 +4,10 @@
   "status": "resolved",
   "baseline_report_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
   "source_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
   "provider": {
     "name": "airflow-to-dabs",
     "version": "0.2.0",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/provider.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/provider.json
index 6c64a60..757bbc4 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/provider.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/provider.json
@@ -11,7 +11,7 @@
     "contract_versions": ["1"],
     "entrypoint": "PROFILE.md",
     "statuses": ["resolved", "needs_input", "deferred"],
-    "replacement_kinds": ["notebook", "sql"]
+    "replacement_kinds": ["notebook", "sql", "spark_python"]
   },
   "knowledge": [
     {
diff --git a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
index 7f6b7ad..cac122a 100644
--- a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
+++ b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
@@ -16,6 +16,10 @@ Databricks `task_key`. `task_path` identifies the exact placeholder location, in
   "status": "resolved",
   "baseline_report_sha256": "copied from the envelope",
   "source_sha256": "copied from the envelope",
+  "task_sha256": "copied from the envelope",
+  "graph_sha256": "copied from the envelope",
+  "provider_sha256": "copied from the envelope",
+  "request_sha256": "copied from the envelope",
   "provider": {
     "name": "airflow-to-dabs",
     "version": "0.2.0",
@@ -47,20 +51,23 @@ Databricks `task_key`. `task_path` identifies the exact placeholder location, in
 SQL uses `{"kind": "sql", "file": "task.sql", "parameters": {}}` and a single generated file
 whose language is `sql`.
 
+Spark Python uses `{"kind": "spark_python", "file": "task.py", "parameters": ["--arg", "value"]}`
+and a single generated Python file. It is emitted as a Databricks `spark_python_task`.
+
 `needs_input` and `deferred` omit `replacement` and `generated_files` and add a non-empty `reason`.
 They are terminal reviewed outcomes: the linked `NotImplementedError` placeholder remains and no
 automatic retry occurs.
 
 ## Hard boundaries
 
-- Only `notebook` and `sql` leaf replacements are allowed in v1.
+- Only `notebook`, `sql`, and `spark_python` leaf replacements are allowed in v1.
 - The replacement cannot express `name`, `task_key`, `depends_on`, retries, timeouts, compute,
   libraries, schedules, or control-flow fields.
 - Generated file paths are relative and cannot contain `..`.
 - Every generated file is inline and hash-bound; external workspace paths are not accepted.
 - Python payloads may mention Airflow in comments but may not contain `import airflow` or
-  `from airflow ...` statements. They must start with `# Databricks notebook source` so the bundle
-  imports them as notebooks rather than ordinary Python files.
+  `from airflow ...` statements. Notebook payloads must start with `# Databricks notebook source`;
+  Spark Python scripts are ordinary valid Python files.
 - Airflow Jinja is rejected. Databricks dynamic references such as `{{job.parameters.x}}`,
   `{{tasks.upstream.values.x}}`, and `{{input}}` are valid in replacement parameter values, not in
   uploaded notebook or SQL source files; source files must read widgets or SQL named parameters.
diff --git a/src/flowx/adapter/__main__.py b/src/flowx/adapter/__main__.py
index 84cf90e..38545be 100644
--- a/src/flowx/adapter/__main__.py
+++ b/src/flowx/adapter/__main__.py
@@ -115,6 +115,7 @@ def _run_resolve_agentic(args: argparse.Namespace) -> int:
                 report_path=args.report,
                 output_dir=args.output_dir,
                 dbt_mode=args.dbt_mode,
+                gap_id=args.gap_id,
             )
         elif args.action == "stage":
             payload = stage_airflow_resolutions(
@@ -427,6 +428,11 @@ def _build_parser() -> argparse.ArgumentParser:
     resolve_agentic.add_argument("--output-dir", type=Path, required=True, help="Shared migration output directory.")
     resolve_agentic.add_argument("--source-path", type=Path, default=None, help="Airflow DAG file or directory.")
     resolve_agentic.add_argument("--report", type=Path, default=None, help="Deterministic translation report.")
+    resolve_agentic.add_argument(
+        "--gap-id",
+        default=None,
+        help="Optional prepared gap fingerprint to return through the caller while retaining the full workspace.",
+    )
     resolve_agentic.add_argument(
         "--candidate",
         type=Path,
diff --git a/src/flowx/agentic.py b/src/flowx/agentic.py
index 98247af..271c328 100644
--- a/src/flowx/agentic.py
+++ b/src/flowx/agentic.py
@@ -27,9 +27,10 @@
 PROVIDER_VERSION = "0.2.0"
 PROVIDER_REPOSITORY = "https://github.com/park-peter/airflow-to-dabs"
 
-_ALLOWED_REPLACEMENT_KINDS = ("notebook", "sql")
+_ALLOWED_REPLACEMENT_KINDS = ("notebook", "sql", "spark_python")
 _RESOLUTION_STATUSES = {"resolved", "needs_input", "deferred"}
-_DISPOSITIONS = {"consumed", "preserved_by_flowx", "ignored"}
+_DISPOSITIONS = {"consumed", "preserved_by_flowx", "ignored", "needs_input"}
+_MAX_GENERATED_FILE_BYTES = 1024 * 1024
 _COMMON_TASK_FIELDS = (
     "name",
     "task_key",
@@ -52,6 +53,7 @@
 _NESTED_TASK_FIELDS = ("inner_activities", "if_true_activities", "if_false_activities", "default_activities")
 _AIRFLOW_TEMPLATE = re.compile(r"{{\s*([^{}]+?)\s*}}|{%\s*([^{}]+?)\s*%}")
 _DAB_TEMPLATE_PREFIXES = ("job.", "tasks.", "input.", "backfill.")
+_UNSET = object()
 
 
 class AgenticContractError(ValueError):
@@ -64,6 +66,7 @@ class GapEnvelope:
 
     gap_id: str
     pipeline_name: str
+    dag_capture_identity: str
     capture_identity: str
     task_key: str
     task_path: list[str | int]
@@ -71,6 +74,10 @@ class GapEnvelope:
     source_file: str
     source_sha256: str
     baseline_report_sha256: str
+    task_sha256: str
+    graph_sha256: str
+    provider_sha256: str
+    finding_fingerprints: list[str]
     source_span: dict[str, int]
     raw_definition: dict[str, Any]
     arguments: list[dict[str, Any]]
@@ -81,11 +88,17 @@ class GapEnvelope:
 
     def as_dict(self) -> dict[str, Any]:
         """Returns the public GapEnvelope v1 representation."""
-        return {
+        provider = {
+            "name": PROVIDER_NAME,
+            "version": PROVIDER_VERSION,
+            "repository": PROVIDER_REPOSITORY,
+        }
+        payload = {
             "contract_version": CONTRACT_VERSION,
             "gap_id": self.gap_id,
             "source": "airflow",
             "pipeline_name": self.pipeline_name,
+            "dag_capture_identity": self.dag_capture_identity,
             "capture_identity": self.capture_identity,
             "task_key": self.task_key,
             "task_path": self.task_path,
@@ -94,6 +107,10 @@ def as_dict(self) -> dict[str, Any]:
             "source_file": self.source_file,
             "source_sha256": self.source_sha256,
             "baseline_report_sha256": self.baseline_report_sha256,
+            "task_sha256": self.task_sha256,
+            "graph_sha256": self.graph_sha256,
+            "provider_sha256": self.provider_sha256,
+            "finding_fingerprints": self.finding_fingerprints,
             "source_span": self.source_span,
             "raw_definition": self.raw_definition,
             "arguments": self.arguments,
@@ -102,12 +119,10 @@ def as_dict(self) -> dict[str, Any]:
             "dag_settings": self.dag_settings,
             "reason": self.reason,
             "allowed_replacement_kinds": list(_ALLOWED_REPLACEMENT_KINDS),
-            "knowledge_provider": {
-                "name": PROVIDER_NAME,
-                "version": PROVIDER_VERSION,
-                "repository": PROVIDER_REPOSITORY,
-            },
+            "knowledge_provider": provider,
         }
+        payload["request_sha256"] = _sha256_bytes(_json_bytes(payload))
+        return payload
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)
@@ -137,6 +152,7 @@ def prepare_airflow_resolutions(
     report_path: Path,
     output_dir: Path,
     dbt_mode: str = "static",
+    gap_id: str | None = None,
 ) -> dict[str, Any]:
     """Snapshots source and an exactly reproducible deterministic report, then emits GapEnvelope v1."""
     source_path = source_path.resolve()
@@ -182,11 +198,14 @@ def prepare_airflow_resolutions(
         gaps = _build_gap_envelopes(baseline, baseline_hash=baseline_hash, source_hashes=source_hashes)
         if not gaps:
             raise AgenticContractError("The deterministic report contains no eligible Airflow leaf gaps")
+        if gap_id is not None and gap_id not in {gap["gap_id"] for gap in gaps}:
+            raise AgenticContractError(f"No eligible Airflow gap matches --gap-id {gap_id!r}")
 
         (staging / "baseline.json").write_bytes(baseline_bytes)
         gaps_bytes = _json_bytes(gaps)
         (staging / "gaps.json").write_bytes(gaps_bytes)
         (staging / "candidates").mkdir()
+        _copy_provider_context(staging / "provider")
         _write_json(staging / "candidate_index.json", {})
         manifest = {
             "contract_version": CONTRACT_VERSION,
@@ -204,6 +223,7 @@ def prepare_airflow_resolutions(
             "dbt_mode": dbt_mode,
             "baseline_report_sha256": baseline_hash,
             "gaps_sha256": _sha256_bytes(gaps_bytes),
+            "requested_gap_id": gap_id,
         }
         _write_json(staging / "manifest.json", manifest)
         if target.exists():
@@ -215,6 +235,7 @@ def prepare_airflow_resolutions(
         "contract_version": CONTRACT_VERSION,
         "provider_version": PROVIDER_VERSION,
         "gap_count": len(gaps),
+        "requested_gap_id": gap_id,
         "workspace": str(target),
     }
 
@@ -269,6 +290,23 @@ def stage_airflow_resolutions(
         "staged": sorted(staged),
         "candidate_count": len(index),
         "review_manifest": str(review_manifest_path),
+        "review": [
+            {
+                "gap_id": gap_id,
+                "status": resolution.candidate["status"],
+                "provider": resolution.candidate["provider"],
+                "model": resolution.candidate["model"],
+                "argument_disposition": resolution.candidate["argument_disposition"],
+                "prerequisites": resolution.candidate["prerequisites"],
+                "warnings": resolution.candidate["warnings"],
+                "semantic_deltas": resolution.candidate["semantic_deltas"],
+                "replacement": resolution.candidate.get("replacement"),
+                "generated_files": resolution.candidate.get("generated_files", []),
+                "reason": resolution.candidate.get("reason"),
+                "candidate_sha256": resolution.sha256,
+            }
+            for gap_id, resolution in sorted(staged.items())
+        ],
     }
 
 
@@ -613,7 +651,12 @@ def _load_persisted_agentic_evidence(evidence_dir: Path) -> PersistedResolutionE
         raise AgenticContractError("agentic resolution manifest has invalid source_files") from error
     if not source_hashes:
         raise AgenticContractError("agentic resolution manifest has no source_files")
-    durable_source_hashes = {relative: _sha256_file(evidence_dir / "source" / relative) for relative in source_hashes}
+    try:
+        durable_source_hashes = {
+            relative: _sha256_file(evidence_dir / "source" / relative) for relative in source_hashes
+        }
+    except OSError as error:
+        raise AgenticContractError(f"durable Airflow source snapshot is missing: {error}") from error
     if durable_source_hashes != source_hashes:
         raise AgenticContractError("durable Airflow source snapshot does not match its manifest")
     expected_gaps = _build_gap_envelopes(
@@ -698,8 +741,14 @@ def _require_airflow_baseline(payload: Any) -> None:
     for pipeline in pipelines:
         if not isinstance(pipeline, dict) or (pipeline.get("tags") or {}).get("source") != "airflow":
             raise AgenticContractError("resolve-agentic requires a canonical Airflow translation report")
-        if pipeline.get("reconciliation_status") == "failed":
-            raise AgenticContractError("Agentic resolution cannot repair a failed source-reconciliation report")
+        status = pipeline.get("reconciliation_status")
+        if status not in {"verified", "verified_with_gaps", "excluded"}:
+            raise AgenticContractError(
+                f"Agentic resolution requires a successfully reconciled deterministic report, got {status!r}"
+            )
+        audit = pipeline.get("audit")
+        if not isinstance(audit, dict) or "audited_activity_count" not in audit or "transformations" not in audit:
+            raise AgenticContractError("Airflow deterministic report is missing source-audit metadata")
 
 
 def _rebuild_airflow_report(source_path: Path, baseline: dict[str, Any], *, dbt_mode: str) -> dict[str, Any]:
@@ -768,9 +817,16 @@ def _build_gap_envelopes(
             ]
             if not any(item.get("code") == "unsupported_trigger_rule" for item in related_findings):
                 flowx_owned_arguments.add("trigger_rule")
+            sanitized_definition = _sanitize_raw_definition(raw_definition)
+            provider = {
+                "name": PROVIDER_NAME,
+                "version": PROVIDER_VERSION,
+                "repository": PROVIDER_REPOSITORY,
+            }
             envelope = GapEnvelope(
                 gap_id=str(matched_finding["fingerprint"]),
                 pipeline_name=str(pipeline["name"]),
+                dag_capture_identity=f"dag:{source_file}:{pipeline['name']}",
                 capture_identity=capture_identity,
                 task_key=str(task["task_key"]),
                 task_path=list(task_path),
@@ -778,12 +834,18 @@ def _build_gap_envelopes(
                 source_file=str(source_file),
                 source_sha256=source_hash,
                 baseline_report_sha256=baseline_hash,
+                task_sha256=_sha256_bytes(_json_bytes(task)),
+                graph_sha256=_graph_hash(pipeline),
+                provider_sha256=_sha256_bytes(_json_bytes(provider)),
+                finding_fingerprints=sorted(
+                    {str(item["fingerprint"]) for item in related_findings if isinstance(item.get("fingerprint"), str)}
+                ),
                 source_span={
                     key: int(matched_finding.get(key, 0)) for key in ("line", "column", "end_line", "end_column")
                 },
-                raw_definition=raw_definition,
+                raw_definition=sanitized_definition,
                 arguments=_extract_arguments(
-                    raw_definition,
+                    sanitized_definition,
                     operator=operator,
                     flowx_owned_arguments=flowx_owned_arguments,
                 ),
@@ -854,11 +916,20 @@ def _extract_arguments(
 
 
 def _argument(name: str, expression: str, flowx_owned_arguments: set[str]) -> dict[str, Any]:
-    return {
+    preserved = name in flowx_owned_arguments
+    argument = {
         "name": name,
         "source_expression": expression,
-        "preserved_by_flowx": name in flowx_owned_arguments,
+        "owner": "flowx" if preserved else "provider",
+        "preserved_by_flowx": preserved,
     }
+    try:
+        literal = ast.literal_eval(expression)
+    except (ValueError, SyntaxError):
+        literal = _UNSET
+    if literal is not _UNSET and _is_json_literal(literal):
+        argument["normalized_value"] = literal
+    return argument
 
 
 def _validate_candidate(
@@ -875,6 +946,10 @@ def _validate_candidate(
         "status",
         "baseline_report_sha256",
         "source_sha256",
+        "task_sha256",
+        "graph_sha256",
+        "provider_sha256",
+        "request_sha256",
         "provider",
         "model",
         "argument_disposition",
@@ -900,6 +975,9 @@ def _validate_candidate(
         raise AgenticContractError("Candidate baseline_report_sha256 does not match the prepared baseline")
     if candidate.get("source_sha256") != gap.get("source_sha256"):
         raise AgenticContractError("Candidate source_sha256 does not match its GapEnvelope")
+    for field in ("task_sha256", "graph_sha256", "provider_sha256", "request_sha256"):
+        if candidate.get(field) != gap.get(field):
+            raise AgenticContractError(f"Candidate {field} does not match its GapEnvelope")
     provider = candidate.get("provider")
     expected_provider = {
         "name": PROVIDER_NAME,
@@ -919,6 +997,8 @@ def _validate_candidate(
             raise AgenticContractError(f"Candidate {field} must be a list of strings")
     _validate_argument_disposition(candidate.get("argument_disposition"), gap)
     if status == "resolved":
+        if any(item.get("disposition") == "needs_input" for item in candidate["argument_disposition"]):
+            raise AgenticContractError("Resolved candidate cannot retain a needs_input argument disposition")
         _validate_replacement(candidate, gap)
     else:
         if not isinstance(candidate.get("reason"), str) or not candidate["reason"].strip():
@@ -965,8 +1045,11 @@ def _validate_replacement(candidate: dict[str, Any], gap: dict[str, Any]) -> Non
     if not isinstance(file_name, str) or not _safe_relative_path(file_name):
         raise AgenticContractError("Replacement file must be a safe relative path")
     parameters_field = "base_parameters" if kind == "notebook" else "parameters"
-    parameters = replacement.get(parameters_field, {})
-    if not isinstance(parameters, dict) or not all(
+    parameters = replacement.get(parameters_field, [] if kind == "spark_python" else {})
+    if kind == "spark_python":
+        if not isinstance(parameters, list) or not all(isinstance(value, str) for value in parameters):
+            raise AgenticContractError("Replacement parameters must be a list of strings for spark_python")
+    elif not isinstance(parameters, dict) or not all(
         isinstance(key, str) and isinstance(val, str) for key, val in parameters.items()
     ):
         raise AgenticContractError(f"Replacement {parameters_field} must be a string-to-string object")
@@ -979,20 +1062,22 @@ def _validate_replacement(candidate: dict[str, Any], gap: dict[str, Any]) -> Non
         raise AgenticContractError("Generated file contains unsupported fields")
     if generated.get("path") != file_name:
         raise AgenticContractError("Replacement file does not match generated_files.path")
-    expected_language = "python" if kind == "notebook" else "sql"
+    expected_language = "sql" if kind == "sql" else "python"
     if generated.get("language") != expected_language:
         raise AgenticContractError(f"Generated file language must be {expected_language!r}")
     content = generated.get("content")
     if not isinstance(content, str) or not content.strip():
         raise AgenticContractError("Generated file content must be non-empty")
+    if len(content.encode("utf-8")) > _MAX_GENERATED_FILE_BYTES:
+        raise AgenticContractError(f"Generated file exceeds the {_MAX_GENERATED_FILE_BYTES}-byte contract limit")
     if generated.get("sha256") != _sha256_bytes(content.encode("utf-8")):
         raise AgenticContractError("Generated file sha256 does not match its content")
     _reject_unresolved_templates(replacement)
     _reject_generated_file_templates(content)
-    if kind == "notebook":
+    if kind in {"notebook", "spark_python"}:
         lines = content.splitlines()
         first_line = lines[0] if lines else ""
-        if first_line != "# Databricks notebook source":
+        if kind == "notebook" and first_line != "# Databricks notebook source":
             raise AgenticContractError("Generated notebook requires the Databricks notebook source marker")
         try:
             module = ast.parse(content)
@@ -1077,10 +1162,20 @@ def _build_replacement(placeholder: dict[str, Any], candidate: dict[str, Any]) -
         )
         if replacement.get("base_parameters"):
             task["base_parameters"] = dict(replacement["base_parameters"])
-    else:
+    elif replacement["kind"] == "sql":
         task.update({"type": "SqlActivity", "sql": generated["content"], "warehouse_ref": "${var.warehouse_id}"})
         if replacement.get("parameters"):
             task["parameters"] = dict(replacement["parameters"])
+    else:
+        task.update(
+            {
+                "type": "SparkPythonActivity",
+                "python_file": f"scripts/{placeholder['task_key']}.py",
+                "generated_source": generated["content"],
+            }
+        )
+        if replacement.get("parameters"):
+            task["parameters"] = list(replacement["parameters"])
     return task
 
 
@@ -1302,11 +1397,80 @@ def _workspace(output_dir: Path) -> Path:
     return output_dir.resolve() / ".work" / "agentic"
 
 
+def _copy_provider_context(destination: Path) -> None:
+    source = (
+        Path(__file__).resolve().parents[2]
+        / "skills"
+        / "flowx-resolve-airflow-gaps"
+        / "references"
+        / f"airflow-to-dabs-v{PROVIDER_VERSION}"
+    )
+    if not source.is_dir():
+        raise AgenticContractError(
+            f"provider_unavailable: pinned {PROVIDER_NAME} v{PROVIDER_VERSION} context is missing"
+        )
+    shutil.copytree(source, destination)
+
+
 def _safe_relative_path(value: str) -> bool:
     path = Path(value)
     return bool(value) and not path.is_absolute() and ".." not in path.parts and value == path.as_posix()
 
 
+def _is_json_literal(value: Any) -> bool:
+    if value is None or isinstance(value, (str, int, float, bool)):
+        return True
+    if isinstance(value, list):
+        return all(_is_json_literal(item) for item in value)
+    if isinstance(value, tuple):
+        return all(_is_json_literal(item) for item in value)
+    if isinstance(value, dict):
+        return all(isinstance(key, str) and _is_json_literal(item) for key, item in value.items())
+    return False
+
+
+_SENSITIVE_ARGUMENT = re.compile(
+    r"(?:password|passwd|token|secret|credential|private[_-]?key|access[_-]?key)",
+    re.IGNORECASE,
+)
+
+
+class _SecretLiteralRedactor(ast.NodeTransformer):
+    def visit_Call(self, node: ast.Call) -> ast.AST:
+        self.generic_visit(node)
+        for keyword in node.keywords:
+            if keyword.arg is not None and _SENSITIVE_ARGUMENT.search(keyword.arg):
+                keyword.value = ast.Constant(value="")
+        return node
+
+
+def _sanitize_python_source(value: str) -> str:
+    try:
+        module = ast.parse(textwrap.dedent(value))
+    except SyntaxError:
+        return re.sub(
+            r"(?i)((?:password|passwd|token|secret|credential|private[_-]?key|access[_-]?key)\s*=\s*)"
+            r"(['\"]).*?\2",
+            r"\1''",
+            value,
+        )
+    redacted = _SecretLiteralRedactor().visit(module)
+    ast.fix_missing_locations(redacted)
+    return ast.unparse(redacted)
+
+
+def _sanitize_raw_definition(value: Any, *, key: str = "") -> Any:
+    if _SENSITIVE_ARGUMENT.search(key):
+        return ""
+    if isinstance(value, dict):
+        return {item_key: _sanitize_raw_definition(item, key=str(item_key)) for item_key, item in value.items()}
+    if isinstance(value, list):
+        return [_sanitize_raw_definition(item) for item in value]
+    if isinstance(value, str) and key in {"source", "bound_source", "invocation", "mapping"}:
+        return _sanitize_python_source(value)
+    return value
+
+
 def _call_name(node: ast.expr) -> str:
     if isinstance(node, ast.Name):
         return node.id
diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py
index cbab4d2..39a5f55 100644
--- a/src/flowx/bundler/dab_writer.py
+++ b/src/flowx/bundler/dab_writer.py
@@ -2061,6 +2061,7 @@ def _reconstruct_ir(task_ir: dict[str, Any]) -> Activity:
             **base,
             python_file=task_ir.get("python_file", ""),
             parameters=task_ir.get("parameters"),
+            generated_source=task_ir.get("generated_source"),
         )
     if task_type == "ExecutePipelineActivity":
         return ExecutePipelineActivity(
diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py
index b253ee8..006e70f 100644
--- a/src/flowx/ir_serde.py
+++ b/src/flowx/ir_serde.py
@@ -289,6 +289,8 @@ def activity_extra_fields(activity: Activity) -> dict[str, Any]:
             extra["python_file"] = activity.python_file
             if activity.parameters:
                 extra["parameters"] = activity.parameters
+            if activity.generated_source is not None:
+                extra["generated_source"] = activity.generated_source
         case WebActivity():
             extra["url"] = activity.url
             extra["method"] = activity.method
diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py
index 564bee9..16fd03c 100644
--- a/src/flowx/mcp/server.py
+++ b/src/flowx/mcp/server.py
@@ -255,6 +255,8 @@ def _cmd_resolve_agentic(p: dict[str, Any]) -> dict[str, Any]:
         args += ["--report", p["report_path"]]
     if p.get("dbt_mode"):
         args += ["--dbt-mode", p["dbt_mode"]]
+    if p.get("gap_id"):
+        args += ["--gap-id", p["gap_id"]]
     accepted_gaps = p.get("accept_gap") or p.get("accept_gaps") or []
     if isinstance(accepted_gaps, str):
         accepted_gaps = [accepted_gaps]
@@ -287,7 +289,10 @@ def _cmd_resolve_agentic(p: dict[str, Any]) -> dict[str, Any]:
     payload = runner.parse_stdout_json(result)
     extra: dict[str, Any] = {"result": payload}
     if action == "prepare":
-        extra["gaps"] = runner.read_json(output_dir / ".work" / "agentic" / "gaps.json")
+        gaps = runner.read_json(output_dir / ".work" / "agentic" / "gaps.json")
+        if p.get("gap_id") and isinstance(gaps, list):
+            gaps = [gap for gap in gaps if isinstance(gap, dict) and gap.get("gap_id") == p["gap_id"]]
+        extra["gaps"] = gaps
     return {"ok": result.ok, "process": result.as_dict(), **extra}
 
 
diff --git a/src/flowx/models/ir.py b/src/flowx/models/ir.py
index f770a40..12d77f7 100644
--- a/src/flowx/models/ir.py
+++ b/src/flowx/models/ir.py
@@ -388,10 +388,13 @@ class SparkPythonActivity(Activity):
     Attributes:
         python_file: Path to the Python file to execute.
         parameters: Arguments passed to the script.
+        generated_source: Full Python source generated by a source front-end. When set, the
+            preparer writes it into the bundle instead of downloading the configured source path.
     """
 
     python_file: str
     parameters: list[str] | None = None
+    generated_source: str | None = None
 
 
 @dataclass(slots=True, kw_only=True)
diff --git a/src/flowx/preparer/activity_preparers/spark_python.py b/src/flowx/preparer/activity_preparers/spark_python.py
index 9156967..a9d386c 100644
--- a/src/flowx/preparer/activity_preparers/spark_python.py
+++ b/src/flowx/preparer/activity_preparers/spark_python.py
@@ -52,9 +52,13 @@ def prepare(activity: SparkPythonActivity, *, scope: str = "") -> PreparedActivi
         filename = f"{activity.task_key}.py"
     script_rel_path = f"scripts/{filename}"
 
-    downloaded = download_dbfs_file(original_path)
+    downloaded = None if activity.generated_source is not None else download_dbfs_file(original_path)
     content = (
-        downloaded.decode("utf-8") if downloaded is not None else _python_placeholder(original_path, activity.name)
+        activity.generated_source
+        if activity.generated_source is not None
+        else downloaded.decode("utf-8")
+        if downloaded is not None
+        else _python_placeholder(original_path, activity.name)
     )
     notebooks = [
         DabNotebook(
diff --git a/tests/unit/test_airflow_agentic_resolution.py b/tests/unit/test_airflow_agentic_resolution.py
index dd73978..3318ac5 100644
--- a/tests/unit/test_airflow_agentic_resolution.py
+++ b/tests/unit/test_airflow_agentic_resolution.py
@@ -110,6 +110,10 @@ def _candidate(gap: dict, *, source: str = "print('Migrated from Airflow')\n", s
         "status": status,
         "baseline_report_sha256": gap["baseline_report_sha256"],
         "source_sha256": gap["source_sha256"],
+        "task_sha256": gap["task_sha256"],
+        "graph_sha256": gap["graph_sha256"],
+        "provider_sha256": gap["provider_sha256"],
+        "request_sha256": gap["request_sha256"],
         "provider": {
             "name": "airflow-to-dabs",
             "version": "0.2.0",
@@ -178,9 +182,58 @@ def test_prepare_writes_versioned_fingerprint_bound_gap_without_changing_report(
     assert gap["task_key"] == "pod"
     assert gap["operator"] == "KubernetesPodOperator"
     assert gap["source_sha256"] == hashlib.sha256(source.read_bytes()).hexdigest()
+    assert gap["dag_capture_identity"] == "dag:dag.py:agentic"
+    assert gap["finding_fingerprints"] == [gap["gap_id"]]
+    assert all(len(gap[field]) == 64 for field in ("task_sha256", "graph_sha256", "provider_sha256", "request_sha256"))
     assert {argument["name"] for argument in gap["arguments"]} == {"task_id", "image", "retries"}
+    assert {argument["owner"] for argument in gap["arguments"]} == {"flowx", "provider"}
+    image = next(argument for argument in gap["arguments"] if argument["name"] == "image")
+    assert image["normalized_value"] == "python:3.11"
     assert (output / ".work" / "agentic" / "baseline.json").exists()
     assert (output / ".work" / "agentic" / "source" / "dag.py").read_bytes() == source.read_bytes()
+    assert (output / ".work" / "agentic" / "provider" / "PROFILE.md").exists()
+
+
+def test_prepare_can_select_one_gap_for_the_caller_without_losing_workspace_gaps(tmp_path: Path) -> None:
+    source, output, gaps = _prepare(tmp_path, two_tasks=True)
+    report = output / ".work" / "translation_report.json"
+
+    assert adapter_main(
+        [
+            "resolve-agentic",
+            "prepare",
+            "--source",
+            "airflow",
+            "--source-path",
+            str(source),
+            "--report",
+            str(report),
+            "--output-dir",
+            str(output),
+            "--gap-id",
+            gaps[0]["gap_id"],
+        ]
+    ) == 0
+    prepared = json.loads((output / ".work" / "agentic" / "gaps.json").read_text(encoding="utf-8"))
+    manifest = json.loads((output / ".work" / "agentic" / "manifest.json").read_text(encoding="utf-8"))
+    assert {gap["gap_id"] for gap in prepared} == {gap["gap_id"] for gap in gaps}
+    assert manifest["requested_gap_id"] == gaps[0]["gap_id"]
+
+
+def test_prepare_redacts_sensitive_constructor_literals(tmp_path: Path) -> None:
+    source = tmp_path / "secret.py"
+    source.write_text(
+        "from airflow import DAG\n"
+        "with DAG(dag_id='secret') as dag:\n"
+        "    pod = KubernetesPodOperator(task_id='pod', image='python:3.12', api_token='do-not-copy')\n",
+        encoding="utf-8",
+    )
+
+    gap = _prepare_source(source, tmp_path / "output")[0]
+
+    serialized = json.dumps(gap)
+    assert "do-not-copy" not in serialized
+    assert "" in serialized
 
 
 def test_gap_fingerprints_use_each_placeholder_own_source_span(tmp_path: Path) -> None:
@@ -403,7 +456,15 @@ def test_taskflow_gap_arguments_come_from_invocation_not_callable_body(tmp_path:
 
     gap = _prepare_source(source, tmp_path / "output")[0]
 
-    assert gap["arguments"] == [{"name": "$arg0", "source_expression": "'selected'", "preserved_by_flowx": False}]
+    assert gap["arguments"] == [
+        {
+            "name": "$arg0",
+            "source_expression": "'selected'",
+            "normalized_value": "selected",
+            "owner": "provider",
+            "preserved_by_flowx": False,
+        }
+    ]
 
 
 def test_only_actually_preserved_policy_arguments_are_flowx_owned(tmp_path: Path) -> None:
@@ -538,6 +599,32 @@ def test_stage_rejects_unresolved_jinja_provider_drift_and_file_hash_mismatch(tm
     assert "sha256 does not match" in capsys.readouterr().err
 
 
+@pytest.mark.parametrize("field", ["task_sha256", "graph_sha256", "provider_sha256", "request_sha256"])
+def test_stage_rejects_stale_request_identity(tmp_path: Path, capsys, field: str) -> None:
+    _, output, gaps = _prepare(tmp_path)
+    candidate = _candidate(gaps[0])
+    candidate[field] = "0" * 64
+
+    assert _stage(output, candidate) == 1
+    assert f"Candidate {field} does not match" in capsys.readouterr().err
+
+
+def test_stage_limits_artifact_size_and_allows_needs_input_disposition(tmp_path: Path, capsys) -> None:
+    _, output, gaps = _prepare(tmp_path)
+    oversized = _candidate(gaps[0], source="x = '" + "a" * (1024 * 1024) + "'\n")
+
+    assert _stage(output, oversized) == 1
+    assert "exceeds the 1048576-byte contract limit" in capsys.readouterr().err
+
+    unresolved = _candidate(gaps[0], status="needs_input")
+    provider_argument = next(
+        item for item in unresolved["argument_disposition"] if item["disposition"] != "preserved_by_flowx"
+    )
+    provider_argument["disposition"] = "needs_input"
+    provider_argument["rationale"] = "The deployment-specific value must be supplied."
+    assert _stage(output, unresolved) == 0
+
+
 def test_stage_rejects_python_script_without_databricks_notebook_marker(tmp_path: Path, capsys) -> None:
     _, output, gaps = _prepare(tmp_path)
     candidate = _candidate(gaps[0], source="print('plain script')\n")
@@ -748,6 +835,60 @@ def test_apply_supports_sql_leaf_payload(tmp_path: Path):
     assert (output / "src" / "sql" / "pod.sql").read_text(encoding="utf-8") == sql
 
 
+def test_apply_supports_spark_python_leaf_payload(tmp_path: Path) -> None:
+    _, output, gaps = _prepare(tmp_path)
+    candidate = _candidate(gaps[0])
+    source = "print('spark python')\n"
+    candidate["replacement"] = {"kind": "spark_python", "file": "task.py", "parameters": ["--mode", "full"]}
+    candidate["generated_files"] = [
+        {
+            "path": "task.py",
+            "language": "python",
+            "content": source,
+            "sha256": hashlib.sha256(source.encode("utf-8")).hexdigest(),
+        }
+    ]
+    assert _stage(output, candidate) == 0
+    assert (
+        adapter_main(
+            [
+                "resolve-agentic",
+                "apply",
+                "--source",
+                "airflow",
+                "--output-dir",
+                str(output),
+                "--accept-gap",
+                gaps[0]["gap_id"],
+            ]
+        )
+        == 0
+    )
+
+    report = output / ".work" / "translation_report.agentic.json"
+    task = _load_tasks(report)["pod"]
+    assert task["type"] == "SparkPythonActivity"
+    assert task["generated_source"] == source
+    assert task["parameters"] == ["--mode", "full"]
+    assert (
+        package_main(
+            [
+                "--report",
+                str(report),
+                "--output-dir",
+                str(output),
+                "--no-download-workspace-files",
+                "--keep-intermediates",
+            ]
+        )
+        == 0
+    )
+    resource = (output / "resources" / "agentic.yml").read_text(encoding="utf-8")
+    assert "spark_python_task:" in resource
+    assert "../src/scripts/pod.py" in resource
+    assert (output / "src" / "scripts" / "pod.py").read_text(encoding="utf-8") == source
+
+
 def test_nested_for_each_resolution_preserves_enclosing_control_flow(tmp_path: Path):
     fixture = Path(__file__).resolve().parents[1] / "resources" / "airflow" / "review_repros" / "a8_classic_mapping.py"
     source = tmp_path / "a8_classic_mapping.py"
diff --git a/tests/unit/test_mcp_source_routing.py b/tests/unit/test_mcp_source_routing.py
index 4cbd8dc..c9c8578 100644
--- a/tests/unit/test_mcp_source_routing.py
+++ b/tests/unit/test_mcp_source_routing.py
@@ -133,6 +133,7 @@ def test_resolve_agentic_prepare_routes_airflow_contract(captured):
             "airflow_source_path": "/tmp/dags",
             "report_path": "/tmp/out/.work/translation_report.json",
             "output_dir": "/tmp/out",
+            "gap_id": "abc123",
         }
     )
 
@@ -140,6 +141,7 @@ def test_resolve_agentic_prepare_routes_airflow_contract(captured):
     assert argv[:4] == ["resolve-agentic", "prepare", "--source", "airflow"]
     assert argv[argv.index("--source-path") + 1] == "/tmp/dags"
     assert argv[argv.index("--report") + 1] == "/tmp/out/.work/translation_report.json"
+    assert argv[argv.index("--gap-id") + 1] == "abc123"
     assert result["ok"] is True
 
 

From a8bb0626fe55ad35c684ab852a3045cbd8a9a995 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Sun, 9 Aug 2026 14:10:27 -0700
Subject: [PATCH 61/77] Complete reviewed Airflow provider workflow

---
 README.md                                     |    2 +-
 docs/content/docs/options.mdx                 |    2 +
 scripts/sync_airflow_provider.py              |  214 ++
 .../flowx-convert/sources/airflow-coverage.md |    2 +-
 skills/flowx-resolve-airflow-gaps/SKILL.md    |    9 +-
 .../providers/flowx-gap-resolver}/PROFILE.md  |   10 +-
 .../fixtures/gap-deferred.json                |   87 +-
 .../fixtures/gap-needs-input.json             |  102 +-
 .../fixtures/gap-notebook.json                |  100 +-
 .../fixtures/gap-spark-python.json            |   77 +
 .../flowx-gap-resolver}/fixtures/gap-sql.json |   94 +-
 .../fixtures/resolution-deferred.json         |   46 +-
 .../fixtures/resolution-needs-input.json      |   48 +-
 .../fixtures/resolution-notebook.json         |   68 +-
 .../fixtures/resolution-spark-python.json     |   53 +
 .../fixtures/resolution-sql.json              |   64 +-
 .../flowx-gap-resolver}/provider.json         |   57 +-
 .../references/airflow3-migration.md          |  192 ++
 .../references/dab-schema-reference.md        |  718 +++++++
 .../references/hadoop-migration-guide.md      |  387 ++++
 .../references/lakeflow-connect.md            |   96 +
 .../references/operator-mapping.md            | 1805 +++++++++++++++++
 .../references/schedule-trigger-mapping.md    |  349 ++++
 .../references/contract-v1.md                 |    4 +-
 src/flowx/agentic.py                          |   98 +-
 src/flowx/bundler/dab_writer.py               |    8 +-
 src/flowx/mcp/server.py                       |    2 +-
 src/flowx/reporting/coverage.py               |   25 +-
 src/flowx/reporting/dashboard_template.json   |   47 +-
 src/flowx/reporting/results.py                |    7 +-
 tests/unit/test_airflow_agentic_resolution.py |  161 +-
 tests/unit/test_airflow_provider_sync.py      |   58 +
 tests/unit/test_reporting_coverage.py         |   14 +-
 tests/unit/test_reporting_dashboard.py        |    9 +-
 tests/unit/test_reporting_results.py          |   21 +-
 35 files changed, 4666 insertions(+), 370 deletions(-)
 create mode 100644 scripts/sync_airflow_provider.py
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.0 => airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver}/PROFILE.md (95%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.0 => airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver}/fixtures/gap-deferred.json (61%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.0 => airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver}/fixtures/gap-needs-input.json (58%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.0 => airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver}/fixtures/gap-notebook.json (60%)
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-spark-python.json
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.0 => airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver}/fixtures/gap-sql.json (63%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.0 => airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver}/fixtures/resolution-deferred.json (84%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.0 => airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver}/fixtures/resolution-needs-input.json (91%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.0 => airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver}/fixtures/resolution-notebook.json (76%)
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.0 => airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver}/fixtures/resolution-sql.json (75%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.0 => airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver}/provider.json (70%)
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/airflow3-migration.md
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/dab-schema-reference.md
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/hadoop-migration-guide.md
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/lakeflow-connect.md
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/operator-mapping.md
 create mode 100644 skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/schedule-trigger-mapping.md
 create mode 100644 tests/unit/test_airflow_provider_sync.py

diff --git a/README.md b/README.md
index 33205ee..3820152 100644
--- a/README.md
+++ b/README.md
@@ -167,7 +167,7 @@ execution) and maps ~35 operator/sensor families to the shared IR. Highlights:
 
 Operators without a deterministic mapping become a failing placeholder and are recorded in
 `gaps.json` for review. Eligible leaf gaps can use the fingerprint-bound resolver backed by the
-pinned [`airflow-to-dabs` v0.2.0](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.0)
+pinned [`airflow-to-dabs` v0.2.1](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.1)
 provider profile; flowx retains ownership of parsing, graph identity, policy, IR, and packaging. Full matrix:
 [`skills/flowx-convert/sources/airflow-coverage.md`](skills/flowx-convert/sources/airflow-coverage.md).
 
diff --git a/docs/content/docs/options.mdx b/docs/content/docs/options.mdx
index 51584a7..6cd6397 100644
--- a/docs/content/docs/options.mdx
+++ b/docs/content/docs/options.mdx
@@ -117,6 +117,8 @@ phase surfaces three optional inputs — `results_table`, `results_warehouse_id`
   audited/deterministic/agentic/failed/excluded coverage breakdown, reconciliation and migration
   status, finding fingerprints, translation-path coverage, deterministic coverage, unresolved
   agentic count, reviewed-resolution outcomes/provider version, and code-attached coverage.
+  The corresponding result columns are `resolved_agentic_count`, `unresolved_agentic_count`, and
+  `code_attached_coverage_pct`.
   Airflow's audited count remains the denominator even for failed or excluded candidates.
   Code-attached coverage counts deterministic tasks plus accepted `resolved` provider candidates;
   it means the generated code passed mechanical contract validation, not that its semantics were
diff --git a/scripts/sync_airflow_provider.py b/scripts/sync_airflow_provider.py
new file mode 100644
index 0000000..e38bae2
--- /dev/null
+++ b/scripts/sync_airflow_provider.py
@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+"""Vendor and verify a tagged airflow-to-dabs provider release."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import posixpath
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+REPOSITORY = "https://github.com/park-peter/airflow-to-dabs"
+DEFAULT_TAG = "v0.2.1"
+PROVIDER_PATH = PurePosixPath("providers/flowx-gap-resolver/provider.json")
+PIN_FIELD = "flowx_pin"
+
+
+class ProviderSyncError(ValueError):
+    """Raised when provider source or vendored content violates the pin contract."""
+
+
+def _json_object(data: bytes, *, label: str) -> dict[str, Any]:
+    try:
+        value = json.loads(data)
+    except json.JSONDecodeError as error:
+        raise ProviderSyncError(f"{label} contains invalid JSON: {error}") from error
+    if not isinstance(value, dict):
+        raise ProviderSyncError(f"{label} must contain a JSON object")
+    return value
+
+
+def _canonical_bytes(path: PurePosixPath, data: bytes, *, strip_pin: bool = False) -> bytes:
+    if path.suffix != ".json":
+        return data
+    value = _json_object(data, label=path.as_posix())
+    if strip_pin:
+        value.pop(PIN_FIELD, None)
+    return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode()
+
+
+def _resolve_path(base: PurePosixPath, relative: str) -> PurePosixPath:
+    if not relative or PurePosixPath(relative).is_absolute():
+        raise ProviderSyncError(f"Provider manifest contains an unsafe path: {relative!r}")
+    normalized = PurePosixPath(posixpath.normpath((base / relative).as_posix()))
+    if normalized.as_posix() == ".." or normalized.as_posix().startswith("../"):
+        raise ProviderSyncError(f"Provider manifest path escapes the repository: {relative!r}")
+    return normalized
+
+
+def _allowlisted_paths(provider: dict[str, Any]) -> list[PurePosixPath]:
+    base = PROVIDER_PATH.parent
+    interface = provider.get("interface")
+    if not isinstance(interface, dict) or interface.get("contract_versions") != ["1"]:
+        raise ProviderSyncError("Provider manifest must declare flowx contract version 1")
+    paths = {PROVIDER_PATH, _resolve_path(base, str(interface.get("entrypoint", "")))}
+    knowledge = provider.get("knowledge")
+    fixtures = provider.get("fixtures")
+    if not isinstance(knowledge, list) or not isinstance(fixtures, list):
+        raise ProviderSyncError("Provider manifest knowledge and fixtures must be lists")
+    for item in knowledge:
+        if not isinstance(item, dict) or not isinstance(item.get("path"), str):
+            raise ProviderSyncError("Every provider knowledge entry requires a path")
+        paths.add(_resolve_path(base, item["path"]))
+    for item in fixtures:
+        if not isinstance(item, str):
+            raise ProviderSyncError("Every provider fixture entry must be a path string")
+        paths.add(_resolve_path(base, item))
+    return sorted(paths, key=lambda item: item.as_posix())
+
+
+def _combined_digest(files: dict[PurePosixPath, bytes]) -> str:
+    digest = hashlib.sha256()
+    for path in sorted(files, key=lambda item: item.as_posix()):
+        content = files[path]
+        digest.update(path.as_posix().encode())
+        digest.update(b"\0")
+        digest.update(str(len(content)).encode())
+        digest.update(b"\0")
+        digest.update(content)
+    return digest.hexdigest()
+
+
+def _git_output(checkout: Path, *args: str) -> bytes:
+    try:
+        return subprocess.check_output(["git", "-C", str(checkout), *args], stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as error:
+        message = error.output.decode(errors="replace").strip()
+        raise ProviderSyncError(message or "git command failed") from error
+
+
+def sync_provider(*, checkout: Path, tag: str, destination: Path) -> dict[str, str]:
+    commit = _git_output(checkout, "rev-parse", f"{tag}^{{commit}}").decode().strip()
+    provider_data = _git_output(checkout, "show", f"{commit}:{PROVIDER_PATH.as_posix()}")
+    provider = _json_object(provider_data, label=PROVIDER_PATH.as_posix())
+    version = str((provider.get("provider") or {}).get("version", ""))
+    if version != tag.removeprefix("v"):
+        raise ProviderSyncError(f"Provider version {version!r} does not match tag {tag!r}")
+
+    source_files: dict[PurePosixPath, bytes] = {}
+    for path in _allowlisted_paths(provider):
+        data = _git_output(checkout, "show", f"{commit}:{path.as_posix()}")
+        source_files[path] = _canonical_bytes(path, data)
+    content_digest = _combined_digest(source_files)
+
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix=".provider-sync-", dir=destination.parent) as temporary:
+        staging = Path(temporary) / destination.name
+        for path, data in source_files.items():
+            target = staging / path.as_posix()
+            target.parent.mkdir(parents=True, exist_ok=True)
+            if path == PROVIDER_PATH:
+                pinned = _json_object(data, label=path.as_posix())
+                pinned[PIN_FIELD] = {
+                    "repository": REPOSITORY,
+                    "tag": tag,
+                    "commit": commit,
+                    "contract_version": "1",
+                    "content_sha256": content_digest,
+                }
+                data = _canonical_bytes(path, json.dumps(pinned).encode())
+            target.write_bytes(data)
+        if destination.exists():
+            shutil.rmtree(destination)
+        shutil.move(str(staging), destination)
+    return {"tag": tag, "commit": commit, "content_sha256": content_digest}
+
+
+def verify_provider(destination: Path) -> dict[str, str]:
+    provider_file = destination / PROVIDER_PATH.as_posix()
+    provider_bytes = provider_file.read_bytes()
+    provider = _json_object(provider_bytes, label=str(provider_file))
+    pin = provider.get(PIN_FIELD)
+    if not isinstance(pin, dict):
+        raise ProviderSyncError("Vendored provider.json is missing flowx_pin metadata")
+    if pin.get("repository") != REPOSITORY or pin.get("contract_version") != "1":
+        raise ProviderSyncError("Vendored provider pin has an unsupported repository or contract")
+    version = str((provider.get("provider") or {}).get("version", ""))
+    if pin.get("tag") != f"v{version}":
+        raise ProviderSyncError("Vendored provider version does not match its pinned tag")
+
+    allowlisted_paths = set(_allowlisted_paths(provider))
+    actual_paths: set[PurePosixPath] = set()
+    for local in destination.rglob("*"):
+        if local.is_symlink():
+            raise ProviderSyncError(f"Vendored provider cannot contain symlinks: {local.relative_to(destination)}")
+        if local.is_file():
+            actual_paths.add(PurePosixPath(local.relative_to(destination).as_posix()))
+    unexpected = sorted(actual_paths - allowlisted_paths, key=lambda path: path.as_posix())
+    if unexpected:
+        raise ProviderSyncError(
+            "Vendored provider contains files outside its manifest allowlist: "
+            + ", ".join(path.as_posix() for path in unexpected)
+        )
+
+    files: dict[PurePosixPath, bytes] = {}
+    for path in sorted(allowlisted_paths, key=lambda item: item.as_posix()):
+        local = destination / path.as_posix()
+        if not local.is_file():
+            raise ProviderSyncError(f"Vendored provider reference is missing: {path.as_posix()}")
+        local_bytes = local.read_bytes()
+        canonical_bytes = _canonical_bytes(path, local_bytes)
+        if local_bytes != canonical_bytes:
+            raise ProviderSyncError(f"Vendored provider JSON is not canonical: {path.as_posix()}")
+        files[path] = _canonical_bytes(path, local_bytes, strip_pin=path == PROVIDER_PATH)
+    content_digest = _combined_digest(files)
+    if pin.get("content_sha256") != content_digest:
+        raise ProviderSyncError("Vendored provider content digest does not match flowx_pin metadata")
+    commit = pin.get("commit")
+    if not isinstance(commit, str) or len(commit) != 40:
+        raise ProviderSyncError("Vendored provider pin has an invalid commit")
+    return {"tag": str(pin["tag"]), "commit": commit, "content_sha256": content_digest}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--source", type=Path, help="Exact local airflow-to-dabs checkout used for synchronization.")
+    parser.add_argument("--tag", default=DEFAULT_TAG)
+    parser.add_argument(
+        "--destination",
+        type=Path,
+        default=Path(__file__).resolve().parents[1]
+        / "skills"
+        / "flowx-resolve-airflow-gaps"
+        / "references"
+        / f"airflow-to-dabs-{DEFAULT_TAG}",
+    )
+    parser.add_argument(
+        "--check", action="store_true", help="Verify the committed provider pin without network access."
+    )
+    args = parser.parse_args()
+    if not args.check and args.source is None:
+        parser.error("--source is required unless --check is used")
+    try:
+        result = (
+            verify_provider(args.destination)
+            if args.check
+            else sync_provider(
+                checkout=args.source.resolve() if args.source else Path(),
+                tag=args.tag,
+                destination=args.destination,
+            )
+        )
+    except (OSError, ProviderSyncError) as error:
+        parser.error(str(error))
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md
index c07b64b..c5c0909 100644
--- a/skills/flowx-convert/sources/airflow-coverage.md
+++ b/skills/flowx-convert/sources/airflow-coverage.md
@@ -46,7 +46,7 @@ safe fallback is a flagged, failing task rather than a silent omission. Callable
 (`**context` / `ti`) or XCom, and runtime-branching decorators, take the same route rather than
 emitting code that fails at runtime.
 
-The resolver consumes the pinned `airflow-to-dabs` v0.2.0 Flowx provider profile. It receives one
+The resolver consumes the pinned `airflow-to-dabs` v0.2.1 Flowx provider profile. It receives one
 flowx-produced gap envelope and cannot express graph or task-policy changes. Accepted `resolved`
 candidates contribute to mechanically validated code-attached coverage, but remain agentic and do
 not increase deterministic coverage. `needs_input`, `deferred`, and unreviewed candidates remain
diff --git a/skills/flowx-resolve-airflow-gaps/SKILL.md b/skills/flowx-resolve-airflow-gaps/SKILL.md
index e332022..9955532 100644
--- a/skills/flowx-resolve-airflow-gaps/SKILL.md
+++ b/skills/flowx-resolve-airflow-gaps/SKILL.md
@@ -10,16 +10,11 @@ description: >
 Use this workflow only for Airflow reports whose deterministic conversion succeeded with gaps.
 Flowx owns source parsing, task identity, dependencies, task policy, IR, and packaging. This skill
 reasons about one prepared gap at a time using the migration knowledge from
-[`park-peter/airflow-to-dabs` v0.2.0](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.0).
+[`park-peter/airflow-to-dabs` v0.2.1](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.1).
 It must not parse the DAG independently or generate a second bundle.
 
 Read [`references/contract-v1.md`](references/contract-v1.md) and the pinned
-[`airflow-to-dabs-v0.2.0/PROFILE.md`](references/airflow-to-dabs-v0.2.0/PROFILE.md) before authoring a
-resolution. The profile's `../../references/*.md` knowledge paths are relative to the upstream
-v0.2.0 release. Resolve them against
-`https://github.com/park-peter/airflow-to-dabs/tree/v0.2.0/references` or an exact local checkout of
-that tag. If required knowledge is unavailable, return `needs_input` or `deferred`; never infer
-missing operator semantics.
+[`airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md`](references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md) before authoring a resolution. The profile and every referenced knowledge file are vendored from the exact upstream tag and commit under `references/airflow-to-dabs-v0.2.1/`. If required knowledge is unavailable, return `needs_input` or `deferred`; never infer missing operator semantics.
 
 ## 1. Prepare immutable gap envelopes
 
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/PROFILE.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md
similarity index 95%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/PROFILE.md
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md
index d11ae8a..0942dbf 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/PROFILE.md
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md
@@ -1,15 +1,15 @@
-# Flowx Airflow Gap Resolver Profile
+# flowx Airflow Gap Resolver Profile
 
-Resolve exactly one source-reconciled Airflow leaf gap supplied by Flowx. Flowx owns DAG parsing,
+Resolve exactly one source-reconciled Airflow leaf gap supplied by flowx. flowx owns DAG parsing,
 capture identity, task keys, dependencies, task policy, control flow, IR, and bundle packaging. Do
 not reopen or parse the original DAG, construct another task graph, or generate a bundle.
 
-This profile implements Flowx Airflow agentic gap contract `1` with the pinned provider identity:
+This profile implements flowx Airflow agentic gap contract `1` with the pinned provider identity:
 
 ```json
 {
   "name": "airflow-to-dabs",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "repository": "https://github.com/park-peter/airflow-to-dabs"
 }
 ```
@@ -40,7 +40,7 @@ resolution; they do not grant authority to emit jobs, triggers, clusters, pipeli
      other changes outside the leaf-only contract.
 3. Account for every envelope argument exactly once in `argument_disposition`:
    - `consumed`: the generated payload or resolution decision uses it;
-   - `preserved_by_flowx`: Flowx retains it as task identity or policy;
+   - `preserved_by_flowx`: flowx retains it as task identity or policy;
    - `ignored`: the resolution intentionally omits it and states the exact behavioral loss.
    - `needs_input`: the argument depends on a concrete fact the user must provide before resolution.
 4. Enumerate prerequisites, warnings, and semantic deltas. Never hide a dropped behavior in prose or
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-deferred.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-deferred.json
similarity index 61%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-deferred.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-deferred.json
index bd7b72c..bd4cc76 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-deferred.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-deferred.json
@@ -1,41 +1,76 @@
 {
+  "allowed_replacement_kinds": [
+    "notebook",
+    "sql",
+    "spark_python"
+  ],
+  "arguments": [
+    {
+      "name": "task_id",
+      "preserved_by_flowx": true,
+      "source_expression": "'choose_path'"
+    },
+    {
+      "name": "python_callable",
+      "preserved_by_flowx": false,
+      "source_expression": "choose_target"
+    },
+    {
+      "name": "trigger_rule",
+      "preserved_by_flowx": true,
+      "source_expression": "'none_failed'"
+    }
+  ],
+  "baseline_report_sha256": "8888888888888888888888888888888888888888888888888888888888888888",
+  "capture_identity": "choose_path",
   "contract_version": "1",
+  "dag_settings": {
+    "parameters": [],
+    "schedule": null,
+    "tags": {
+      "source": "airflow"
+    }
+  },
+  "downstream_task_keys": [
+    "full_load",
+    "incremental_load"
+  ],
   "gap_id": "4444444444444444",
-  "source": "airflow",
-  "pipeline_name": "branching",
-  "capture_identity": "choose_path",
-  "task_key": "choose_path",
-  "task_path": ["tasks", 1],
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "knowledge_provider": {
+    "name": "airflow-to-dabs",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "version": "0.2.1"
+  },
   "operator": "BranchPythonOperator",
   "operator_fqn": "airflow.operators.python.BranchPythonOperator",
-  "source_file": "branching.py",
-  "source_sha256": "9999999999999999999999999999999999999999999999999999999999999999",
-  "baseline_report_sha256": "8888888888888888888888888888888888888888888888888888888888888888",
-  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
-  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "pipeline_name": "branching",
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
-  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
-  "source_span": {"line": 21, "column": 4, "end_line": 25, "end_column": 5},
   "raw_definition": {
     "operator": "BranchPythonOperator",
     "source": "choose = BranchPythonOperator(task_id=\"choose_path\", python_callable=choose_target, trigger_rule=\"none_failed\")"
   },
-  "arguments": [
-    {"name": "task_id", "source_expression": "'choose_path'", "preserved_by_flowx": true},
-    {"name": "python_callable", "source_expression": "choose_target", "preserved_by_flowx": false},
-    {"name": "trigger_rule", "source_expression": "'none_failed'", "preserved_by_flowx": true}
-  ],
-  "upstream_task_keys": ["read_config"],
-  "downstream_task_keys": ["full_load", "incremental_load"],
-  "dag_settings": {"schedule": null, "parameters": [], "tags": {"source": "airflow"}},
   "reason": {
     "code": "operator_placeholder",
     "message": "BranchPythonOperator requires a graph-aware branch conversion"
   },
-  "allowed_replacement_kinds": ["notebook", "sql"],
-  "knowledge_provider": {
-    "name": "airflow-to-dabs",
-    "version": "0.2.0",
-    "repository": "https://github.com/park-peter/airflow-to-dabs"
-  }
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
+  "source": "airflow",
+  "source_file": "branching.py",
+  "source_sha256": "9999999999999999999999999999999999999999999999999999999999999999",
+  "source_span": {
+    "column": 4,
+    "end_column": 5,
+    "end_line": 25,
+    "line": 21
+  },
+  "task_key": "choose_path",
+  "task_path": [
+    "tasks",
+    1
+  ],
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "upstream_task_keys": [
+    "read_config"
+  ]
 }
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-needs-input.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-needs-input.json
similarity index 58%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-needs-input.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-needs-input.json
index 7120dc0..2b4e564 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-needs-input.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-needs-input.json
@@ -1,47 +1,85 @@
 {
+  "allowed_replacement_kinds": [
+    "notebook",
+    "sql",
+    "spark_python"
+  ],
+  "arguments": [
+    {
+      "name": "task_id",
+      "preserved_by_flowx": true,
+      "source_expression": "'run_container'"
+    },
+    {
+      "name": "image",
+      "preserved_by_flowx": false,
+      "source_expression": "'registry.example.com/orders:7'"
+    },
+    {
+      "name": "cmds",
+      "preserved_by_flowx": false,
+      "source_expression": "['python', '/app/run.py']"
+    },
+    {
+      "name": "namespace",
+      "preserved_by_flowx": false,
+      "source_expression": "'data'"
+    },
+    {
+      "name": "secrets",
+      "preserved_by_flowx": false,
+      "source_expression": "[orders_secret]"
+    }
+  ],
+  "baseline_report_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+  "capture_identity": "run_container",
   "contract_version": "1",
+  "dag_settings": {
+    "parameters": [],
+    "schedule": null,
+    "tags": {
+      "source": "airflow"
+    }
+  },
+  "downstream_task_keys": [
+    "publish_results"
+  ],
   "gap_id": "3333333333333333",
-  "source": "airflow",
-  "pipeline_name": "container_workload",
-  "capture_identity": "run_container",
-  "task_key": "run_container",
-  "task_path": ["tasks", 1],
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "knowledge_provider": {
+    "name": "airflow-to-dabs",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "version": "0.2.1"
+  },
   "operator": "KubernetesPodOperator",
   "operator_fqn": "airflow.providers.cncf.kubernetes.operators.pod.KubernetesPodOperator",
-  "source_file": "container_workload.py",
-  "source_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
-  "baseline_report_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
-  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
-  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "pipeline_name": "container_workload",
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
-  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
-  "source_span": {"line": 14, "column": 4, "end_line": 22, "end_column": 5},
   "raw_definition": {
     "operator": "KubernetesPodOperator",
     "source": "run = KubernetesPodOperator(task_id=\"run_container\", image=\"registry.example.com/orders:7\", cmds=[\"python\", \"/app/run.py\"], namespace=\"data\", secrets=[orders_secret])"
   },
-  "arguments": [
-    {"name": "task_id", "source_expression": "'run_container'", "preserved_by_flowx": true},
-    {
-      "name": "image",
-      "source_expression": "'registry.example.com/orders:7'",
-      "preserved_by_flowx": false
-    },
-    {"name": "cmds", "source_expression": "['python', '/app/run.py']", "preserved_by_flowx": false},
-    {"name": "namespace", "source_expression": "'data'", "preserved_by_flowx": false},
-    {"name": "secrets", "source_expression": "[orders_secret]", "preserved_by_flowx": false}
-  ],
-  "upstream_task_keys": ["build_inputs"],
-  "downstream_task_keys": ["publish_results"],
-  "dag_settings": {"schedule": null, "parameters": [], "tags": {"source": "airflow"}},
   "reason": {
     "code": "operator_placeholder",
     "message": "KubernetesPodOperator requires deployment-specific migration decisions"
   },
-  "allowed_replacement_kinds": ["notebook", "sql"],
-  "knowledge_provider": {
-    "name": "airflow-to-dabs",
-    "version": "0.2.0",
-    "repository": "https://github.com/park-peter/airflow-to-dabs"
-  }
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
+  "source": "airflow",
+  "source_file": "container_workload.py",
+  "source_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+  "source_span": {
+    "column": 4,
+    "end_column": 5,
+    "end_line": 22,
+    "line": 14
+  },
+  "task_key": "run_container",
+  "task_path": [
+    "tasks",
+    1
+  ],
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "upstream_task_keys": [
+    "build_inputs"
+  ]
 }
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-notebook.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-notebook.json
similarity index 60%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-notebook.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-notebook.json
index b8ea6fe..d29ade1 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-notebook.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-notebook.json
@@ -1,47 +1,83 @@
 {
+  "allowed_replacement_kinds": [
+    "notebook",
+    "sql",
+    "spark_python"
+  ],
+  "arguments": [
+    {
+      "name": "task_id",
+      "preserved_by_flowx": true,
+      "source_expression": "'notify_orders'"
+    },
+    {
+      "name": "endpoint",
+      "preserved_by_flowx": false,
+      "source_expression": "'https://example.com/hooks/orders'"
+    },
+    {
+      "name": "method",
+      "preserved_by_flowx": false,
+      "source_expression": "'POST'"
+    },
+    {
+      "name": "data",
+      "preserved_by_flowx": false,
+      "source_expression": "{'event': 'orders_ready'}"
+    },
+    {
+      "name": "retries",
+      "preserved_by_flowx": true,
+      "source_expression": "2"
+    }
+  ],
+  "baseline_report_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+  "capture_identity": "notify_orders",
   "contract_version": "1",
+  "dag_settings": {
+    "parameters": [],
+    "schedule": null,
+    "tags": {
+      "source": "airflow"
+    }
+  },
+  "downstream_task_keys": [],
   "gap_id": "1111111111111111",
-  "source": "airflow",
-  "pipeline_name": "orders",
-  "capture_identity": "notify_orders",
-  "task_key": "notify_orders",
-  "task_path": ["tasks", 2],
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "knowledge_provider": {
+    "name": "airflow-to-dabs",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "version": "0.2.1"
+  },
   "operator": "SimpleHttpOperator",
   "operator_fqn": "airflow.providers.http.operators.http.SimpleHttpOperator",
-  "source_file": "orders.py",
-  "source_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
-  "baseline_report_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
-  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
-  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "pipeline_name": "orders",
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
-  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
-  "source_span": {"line": 18, "column": 4, "end_line": 24, "end_column": 5},
   "raw_definition": {
     "operator": "SimpleHttpOperator",
     "source": "notify = SimpleHttpOperator(task_id=\"notify_orders\", endpoint=\"https://example.com/hooks/orders\", method=\"POST\", data={\"event\": \"orders_ready\"}, retries=2)"
   },
-  "arguments": [
-    {"name": "task_id", "source_expression": "'notify_orders'", "preserved_by_flowx": true},
-    {
-      "name": "endpoint",
-      "source_expression": "'https://example.com/hooks/orders'",
-      "preserved_by_flowx": false
-    },
-    {"name": "method", "source_expression": "'POST'", "preserved_by_flowx": false},
-    {"name": "data", "source_expression": "{'event': 'orders_ready'}", "preserved_by_flowx": false},
-    {"name": "retries", "source_expression": "2", "preserved_by_flowx": true}
-  ],
-  "upstream_task_keys": ["publish_orders"],
-  "downstream_task_keys": [],
-  "dag_settings": {"schedule": null, "parameters": [], "tags": {"source": "airflow"}},
   "reason": {
     "code": "operator_placeholder",
     "message": "SimpleHttpOperator requires a provider-authored leaf implementation"
   },
-  "allowed_replacement_kinds": ["notebook", "sql"],
-  "knowledge_provider": {
-    "name": "airflow-to-dabs",
-    "version": "0.2.0",
-    "repository": "https://github.com/park-peter/airflow-to-dabs"
-  }
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
+  "source": "airflow",
+  "source_file": "orders.py",
+  "source_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  "source_span": {
+    "column": 4,
+    "end_column": 5,
+    "end_line": 24,
+    "line": 18
+  },
+  "task_key": "notify_orders",
+  "task_path": [
+    "tasks",
+    2
+  ],
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "upstream_task_keys": [
+    "publish_orders"
+  ]
 }
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-spark-python.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-spark-python.json
new file mode 100644
index 0000000..92a1241
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-spark-python.json
@@ -0,0 +1,77 @@
+{
+  "allowed_replacement_kinds": [
+    "notebook",
+    "sql",
+    "spark_python"
+  ],
+  "arguments": [
+    {
+      "name": "task_id",
+      "owner": "flowx",
+      "preserved_by_flowx": true,
+      "source_expression": "'run_custom_python'"
+    },
+    {
+      "name": "mode",
+      "normalized_value": "full",
+      "owner": "provider",
+      "preserved_by_flowx": false,
+      "source_expression": "'full'"
+    }
+  ],
+  "baseline_report_sha256": "5555555555555555555555555555555555555555555555555555555555555555",
+  "capture_identity": "run_custom_python",
+  "contract_version": "1",
+  "dag_capture_identity": "dag:orders.py:orders",
+  "dag_settings": {
+    "parameters": [],
+    "schedule": null,
+    "tags": {
+      "source": "airflow"
+    }
+  },
+  "downstream_task_keys": [
+    "publish"
+  ],
+  "finding_fingerprints": [
+    "4444444444444444"
+  ],
+  "gap_id": "4444444444444444",
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "knowledge_provider": {
+    "name": "airflow-to-dabs",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "version": "0.2.1"
+  },
+  "operator": "CustomPythonOperator",
+  "operator_fqn": "company.airflow.operators.CustomPythonOperator",
+  "pipeline_name": "orders",
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "raw_definition": {
+    "operator": "CustomPythonOperator",
+    "source": "run = CustomPythonOperator(task_id='run_custom_python', mode='full')"
+  },
+  "reason": {
+    "code": "operator_placeholder",
+    "message": "CustomPythonOperator has no deterministic mapping"
+  },
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
+  "source": "airflow",
+  "source_file": "orders.py",
+  "source_sha256": "4444444444444444444444444444444444444444444444444444444444444444",
+  "source_span": {
+    "column": 4,
+    "end_column": 5,
+    "end_line": 35,
+    "line": 30
+  },
+  "task_key": "run_custom_python",
+  "task_path": [
+    "tasks",
+    3
+  ],
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "upstream_task_keys": [
+    "prepare"
+  ]
+}
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-sql.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-sql.json
similarity index 63%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-sql.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-sql.json
index 792664a..91ee4ff 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/gap-sql.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-sql.json
@@ -1,46 +1,78 @@
 {
+  "allowed_replacement_kinds": [
+    "notebook",
+    "sql",
+    "spark_python"
+  ],
+  "arguments": [
+    {
+      "name": "task_id",
+      "preserved_by_flowx": true,
+      "source_expression": "'cleanup_events'"
+    },
+    {
+      "name": "conn_id",
+      "preserved_by_flowx": false,
+      "source_expression": "'databricks_default'"
+    },
+    {
+      "name": "sql",
+      "preserved_by_flowx": false,
+      "source_expression": "'DELETE FROM main.ops.events WHERE processed_at < current_date() - INTERVAL 30 DAYS'"
+    },
+    {
+      "name": "autocommit",
+      "preserved_by_flowx": false,
+      "source_expression": "True"
+    }
+  ],
+  "baseline_report_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
+  "capture_identity": "cleanup_events",
   "contract_version": "1",
+  "dag_settings": {
+    "parameters": [],
+    "schedule": null,
+    "tags": {
+      "source": "airflow"
+    }
+  },
+  "downstream_task_keys": [
+    "vacuum_events"
+  ],
   "gap_id": "2222222222222222",
-  "source": "airflow",
-  "pipeline_name": "retention",
-  "capture_identity": "cleanup_events",
-  "task_key": "cleanup_events",
-  "task_path": ["tasks", 0],
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "knowledge_provider": {
+    "name": "airflow-to-dabs",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "version": "0.2.1"
+  },
   "operator": "SQLExecuteQueryOperator",
   "operator_fqn": "airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator",
-  "source_file": "retention.py",
-  "source_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
-  "baseline_report_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
-  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
-  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "pipeline_name": "retention",
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
-  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
-  "source_span": {"line": 9, "column": 4, "end_line": 15, "end_column": 5},
   "raw_definition": {
     "operator": "SQLExecuteQueryOperator",
     "source": "cleanup = SQLExecuteQueryOperator(task_id=\"cleanup_events\", conn_id=\"databricks_default\", sql=\"DELETE FROM main.ops.events WHERE processed_at < current_date() - INTERVAL 30 DAYS\", autocommit=True)"
   },
-  "arguments": [
-    {"name": "task_id", "source_expression": "'cleanup_events'", "preserved_by_flowx": true},
-    {"name": "conn_id", "source_expression": "'databricks_default'", "preserved_by_flowx": false},
-    {
-      "name": "sql",
-      "source_expression": "'DELETE FROM main.ops.events WHERE processed_at < current_date() - INTERVAL 30 DAYS'",
-      "preserved_by_flowx": false
-    },
-    {"name": "autocommit", "source_expression": "True", "preserved_by_flowx": false}
-  ],
-  "upstream_task_keys": [],
-  "downstream_task_keys": ["vacuum_events"],
-  "dag_settings": {"schedule": null, "parameters": [], "tags": {"source": "airflow"}},
   "reason": {
     "code": "operator_placeholder",
     "message": "SQLExecuteQueryOperator requires a provider-authored leaf implementation"
   },
-  "allowed_replacement_kinds": ["notebook", "sql"],
-  "knowledge_provider": {
-    "name": "airflow-to-dabs",
-    "version": "0.2.0",
-    "repository": "https://github.com/park-peter/airflow-to-dabs"
-  }
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
+  "source": "airflow",
+  "source_file": "retention.py",
+  "source_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
+  "source_span": {
+    "column": 4,
+    "end_column": 5,
+    "end_line": 15,
+    "line": 9
+  },
+  "task_key": "cleanup_events",
+  "task_path": [
+    "tasks",
+    0
+  ],
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "upstream_task_keys": []
 }
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-deferred.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-deferred.json
similarity index 84%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-deferred.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-deferred.json
index eed0488..ecf4913 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-deferred.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-deferred.json
@@ -1,38 +1,40 @@
 {
-  "contract_version": "1",
-  "gap_id": "4444444444444444",
-  "status": "deferred",
-  "baseline_report_sha256": "8888888888888888888888888888888888888888888888888888888888888888",
-  "source_sha256": "9999999999999999999999999999999999999999999999999999999999999999",
-  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
-  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
-  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
-  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
-  "provider": {
-    "name": "airflow-to-dabs",
-    "version": "0.2.0",
-    "repository": "https://github.com/park-peter/airflow-to-dabs"
-  },
-  "model": {"name": "fixture-model"},
   "argument_disposition": [
     {
-      "name": "task_id",
       "disposition": "preserved_by_flowx",
-      "rationale": "Flowx preserves the collision-safe task identity."
+      "name": "task_id",
+      "rationale": "flowx preserves the collision-safe task identity."
     },
     {
-      "name": "python_callable",
       "disposition": "consumed",
+      "name": "python_callable",
       "rationale": "The callable is recognized as selecting downstream task identities."
     },
     {
-      "name": "trigger_rule",
       "disposition": "preserved_by_flowx",
-      "rationale": "Flowx preserves supported task-run policy independently of the provider."
+      "name": "trigger_rule",
+      "rationale": "flowx preserves supported task-run policy independently of the provider."
     }
   ],
+  "baseline_report_sha256": "8888888888888888888888888888888888888888888888888888888888888888",
+  "contract_version": "1",
+  "gap_id": "4444444444444444",
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "model": {
+    "name": "fixture-model"
+  },
   "prerequisites": [],
-  "warnings": [],
+  "provider": {
+    "name": "airflow-to-dabs",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "version": "0.2.1"
+  },
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "reason": "A faithful branch migration requires condition tasks and downstream dependency rewrites, which are outside the leaf-only provider contract.",
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
   "semantic_deltas": [],
-  "reason": "A faithful branch migration requires condition tasks and downstream dependency rewrites, which are outside the leaf-only provider contract."
+  "source_sha256": "9999999999999999999999999999999999999999999999999999999999999999",
+  "status": "deferred",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "warnings": []
 }
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-needs-input.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json
similarity index 91%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-needs-input.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json
index 6c11053..94a0f81 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-needs-input.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json
@@ -1,48 +1,50 @@
 {
-  "contract_version": "1",
-  "gap_id": "3333333333333333",
-  "status": "needs_input",
-  "baseline_report_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
-  "source_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
-  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
-  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
-  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
-  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
-  "provider": {
-    "name": "airflow-to-dabs",
-    "version": "0.2.0",
-    "repository": "https://github.com/park-peter/airflow-to-dabs"
-  },
-  "model": {"name": "fixture-model"},
   "argument_disposition": [
     {
-      "name": "task_id",
       "disposition": "preserved_by_flowx",
-      "rationale": "Flowx preserves the collision-safe task identity."
+      "name": "task_id",
+      "rationale": "flowx preserves the collision-safe task identity."
     },
     {
-      "name": "image",
       "disposition": "consumed",
+      "name": "image",
       "rationale": "The image identifies the runtime whose dependencies must be assessed."
     },
     {
-      "name": "cmds",
       "disposition": "consumed",
+      "name": "cmds",
       "rationale": "The command identifies the container entrypoint that must be migrated."
     },
     {
-      "name": "namespace",
       "disposition": "consumed",
+      "name": "namespace",
       "rationale": "The namespace is deployment context needed to locate Kubernetes dependencies."
     },
     {
-      "name": "secrets",
       "disposition": "consumed",
+      "name": "secrets",
       "rationale": "The secret reference must be mapped to Databricks secrets or Unity Catalog."
     }
   ],
+  "baseline_report_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+  "contract_version": "1",
+  "gap_id": "3333333333333333",
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "model": {
+    "name": "fixture-model"
+  },
   "prerequisites": [],
-  "warnings": [],
+  "provider": {
+    "name": "airflow-to-dabs",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "version": "0.2.1"
+  },
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "reason": "Provide the container source or packaged application, required Python/system dependencies, registry access requirements, and the Databricks secret or Unity Catalog mappings for orders_secret.",
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
   "semantic_deltas": [],
-  "reason": "Provide the container source or packaged application, required Python/system dependencies, registry access requirements, and the Databricks secret or Unity Catalog mappings for orders_secret."
+  "source_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+  "status": "needs_input",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "warnings": []
 }
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-notebook.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-notebook.json
similarity index 76%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-notebook.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-notebook.json
index 832198f..17032ed 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-notebook.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-notebook.json
@@ -1,56 +1,66 @@
 {
-  "contract_version": "1",
-  "gap_id": "1111111111111111",
-  "status": "resolved",
-  "baseline_report_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
-  "source_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
-  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
-  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
-  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
-  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
-  "provider": {
-    "name": "airflow-to-dabs",
-    "version": "0.2.0",
-    "repository": "https://github.com/park-peter/airflow-to-dabs"
-  },
-  "model": {"name": "fixture-model"},
   "argument_disposition": [
     {
-      "name": "task_id",
       "disposition": "preserved_by_flowx",
-      "rationale": "Flowx preserves the collision-safe task identity."
+      "name": "task_id",
+      "rationale": "flowx preserves the collision-safe task identity."
     },
     {
-      "name": "endpoint",
       "disposition": "consumed",
+      "name": "endpoint",
       "rationale": "The endpoint is embedded in the generated HTTP request."
     },
     {
-      "name": "method",
       "disposition": "consumed",
+      "name": "method",
       "rationale": "The generated notebook issues the captured POST request."
     },
     {
-      "name": "data",
       "disposition": "consumed",
+      "name": "data",
       "rationale": "The captured payload is passed as the JSON request body."
     },
     {
-      "name": "retries",
       "disposition": "preserved_by_flowx",
-      "rationale": "Flowx preserves retry policy on the enclosing job task."
+      "name": "retries",
+      "rationale": "flowx preserves retry policy on the enclosing job task."
     }
   ],
-  "prerequisites": ["The Databricks task must have outbound network access to example.com."],
-  "warnings": [],
-  "semantic_deltas": ["The HTTP request runs in a Databricks notebook instead of an Airflow worker."],
-  "replacement": {"kind": "notebook", "file": "notify_orders.py", "base_parameters": {}},
+  "baseline_report_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+  "contract_version": "1",
+  "gap_id": "1111111111111111",
   "generated_files": [
     {
-      "path": "notify_orders.py",
-      "language": "python",
       "content": "# Databricks notebook source\nimport requests\n\nresponse = requests.post(\n    \"https://example.com/hooks/orders\",\n    json={\"event\": \"orders_ready\"},\n    timeout=30,\n)\nresponse.raise_for_status()\n",
+      "language": "python",
+      "path": "notify_orders.py",
       "sha256": "07d382c58a610ecfabf55c0e3d2097988c4db1b1fd664d0e8cdc60dce9eaddec"
     }
-  ]
+  ],
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "model": {
+    "name": "fixture-model"
+  },
+  "prerequisites": [
+    "The Databricks task must have outbound network access to example.com."
+  ],
+  "provider": {
+    "name": "airflow-to-dabs",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "version": "0.2.1"
+  },
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "replacement": {
+    "base_parameters": {},
+    "file": "notify_orders.py",
+    "kind": "notebook"
+  },
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
+  "semantic_deltas": [
+    "The HTTP request runs in a Databricks notebook instead of an Airflow worker."
+  ],
+  "source_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  "status": "resolved",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "warnings": []
 }
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json
new file mode 100644
index 0000000..b8f7948
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json
@@ -0,0 +1,53 @@
+{
+  "argument_disposition": [
+    {
+      "disposition": "preserved_by_flowx",
+      "name": "task_id",
+      "rationale": "Flowx preserves task identity."
+    },
+    {
+      "disposition": "consumed",
+      "name": "mode",
+      "rationale": "The mode is passed as a script argument."
+    }
+  ],
+  "baseline_report_sha256": "5555555555555555555555555555555555555555555555555555555555555555",
+  "contract_version": "1",
+  "gap_id": "4444444444444444",
+  "generated_files": [
+    {
+      "content": "import argparse\n\nparser = argparse.ArgumentParser()\nparser.add_argument('--mode', required=True)\nargs = parser.parse_args()\nprint(args.mode)\n",
+      "language": "python",
+      "path": "run_custom_python.py",
+      "sha256": "6f05a8d42bf8a949dfa908c928713181e660e0b3a5e38534fbe01a33483f7a8c"
+    }
+  ],
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "model": {
+    "name": "fixture-model",
+    "runtime": "fixture"
+  },
+  "prerequisites": [],
+  "provider": {
+    "name": "airflow-to-dabs",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "version": "0.2.1"
+  },
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "replacement": {
+    "file": "run_custom_python.py",
+    "kind": "spark_python",
+    "parameters": [
+      "--mode",
+      "full"
+    ]
+  },
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
+  "semantic_deltas": [
+    "The custom operator runs as a Databricks Spark Python task."
+  ],
+  "source_sha256": "4444444444444444444444444444444444444444444444444444444444444444",
+  "status": "resolved",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "warnings": []
+}
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-sql.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-sql.json
similarity index 75%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-sql.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-sql.json
index 2bbca8b..53b33ba 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/fixtures/resolution-sql.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-sql.json
@@ -1,51 +1,63 @@
 {
-  "contract_version": "1",
-  "gap_id": "2222222222222222",
-  "status": "resolved",
-  "baseline_report_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
-  "source_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
-  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
-  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
-  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
-  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
-  "provider": {
-    "name": "airflow-to-dabs",
-    "version": "0.2.0",
-    "repository": "https://github.com/park-peter/airflow-to-dabs"
-  },
-  "model": {"name": "fixture-model"},
   "argument_disposition": [
     {
-      "name": "task_id",
       "disposition": "preserved_by_flowx",
-      "rationale": "Flowx preserves the collision-safe task identity."
+      "name": "task_id",
+      "rationale": "flowx preserves the collision-safe task identity."
     },
     {
-      "name": "conn_id",
       "disposition": "consumed",
+      "name": "conn_id",
       "rationale": "The connection identifies Databricks SQL as the execution target."
     },
     {
-      "name": "sql",
       "disposition": "consumed",
+      "name": "sql",
       "rationale": "The captured statement is emitted as the SQL file content."
     },
     {
-      "name": "autocommit",
       "disposition": "ignored",
+      "name": "autocommit",
       "rationale": "Databricks SQL task execution does not expose the Airflow autocommit toggle."
     }
   ],
-  "prerequisites": ["Configure the Flowx warehouse_id bundle variable for the target workspace."],
-  "warnings": ["Airflow autocommit behavior is not reproduced by the Databricks SQL task."],
-  "semantic_deltas": ["The statement runs on the configured Databricks SQL warehouse."],
-  "replacement": {"kind": "sql", "file": "cleanup_events.sql", "parameters": {}},
+  "baseline_report_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
+  "contract_version": "1",
+  "gap_id": "2222222222222222",
   "generated_files": [
     {
-      "path": "cleanup_events.sql",
-      "language": "sql",
       "content": "DELETE FROM main.ops.events\nWHERE processed_at < current_date() - INTERVAL 30 DAYS\n",
+      "language": "sql",
+      "path": "cleanup_events.sql",
       "sha256": "89a5ec5ce7c02bfdffa57473908407ef620918b53840d8b8ce848179790546a7"
     }
+  ],
+  "graph_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
+  "model": {
+    "name": "fixture-model"
+  },
+  "prerequisites": [
+    "Configure the flowx warehouse_id bundle variable for the target workspace."
+  ],
+  "provider": {
+    "name": "airflow-to-dabs",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "version": "0.2.1"
+  },
+  "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
+  "replacement": {
+    "file": "cleanup_events.sql",
+    "kind": "sql",
+    "parameters": {}
+  },
+  "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
+  "semantic_deltas": [
+    "The statement runs on the configured Databricks SQL warehouse."
+  ],
+  "source_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
+  "status": "resolved",
+  "task_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+  "warnings": [
+    "Airflow autocommit behavior is not reproduced by the Databricks SQL task."
   ]
 }
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/provider.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/provider.json
similarity index 70%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/provider.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/provider.json
index 757bbc4..993fb7f 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.0/provider.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/provider.json
@@ -1,17 +1,40 @@
 {
-  "profile_schema_version": "1",
-  "provider": {
-    "name": "airflow-to-dabs",
-    "version": "0.2.0",
-    "repository": "https://github.com/park-peter/airflow-to-dabs"
+  "fixtures": [
+    "fixtures/gap-notebook.json",
+    "fixtures/resolution-notebook.json",
+    "fixtures/gap-sql.json",
+    "fixtures/resolution-sql.json",
+    "fixtures/gap-spark-python.json",
+    "fixtures/resolution-spark-python.json",
+    "fixtures/gap-needs-input.json",
+    "fixtures/resolution-needs-input.json",
+    "fixtures/gap-deferred.json",
+    "fixtures/resolution-deferred.json"
+  ],
+  "flowx_pin": {
+    "commit": "75196aef85ebb2736b926f2d4db13ec7d5c2551c",
+    "content_sha256": "e1e7395204b3f2759722b9320cea08c6b58284a48fd8062686b36bf676b657fd",
+    "contract_version": "1",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "tag": "v0.2.1"
   },
   "interface": {
+    "contract_versions": [
+      "1"
+    ],
+    "entrypoint": "PROFILE.md",
     "name": "flowx-gap-resolver",
+    "replacement_kinds": [
+      "notebook",
+      "sql",
+      "spark_python"
+    ],
     "source": "airflow",
-    "contract_versions": ["1"],
-    "entrypoint": "PROFILE.md",
-    "statuses": ["resolved", "needs_input", "deferred"],
-    "replacement_kinds": ["notebook", "sql", "spark_python"]
+    "statuses": [
+      "resolved",
+      "needs_input",
+      "deferred"
+    ]
   },
   "knowledge": [
     {
@@ -39,14 +62,10 @@
       "purpose": "Spark-submit, HDFS, Hive, and Hadoop semantic guidance"
     }
   ],
-  "fixtures": [
-    "fixtures/gap-notebook.json",
-    "fixtures/resolution-notebook.json",
-    "fixtures/gap-sql.json",
-    "fixtures/resolution-sql.json",
-    "fixtures/gap-needs-input.json",
-    "fixtures/resolution-needs-input.json",
-    "fixtures/gap-deferred.json",
-    "fixtures/resolution-deferred.json"
-  ]
+  "profile_schema_version": "1",
+  "provider": {
+    "name": "airflow-to-dabs",
+    "repository": "https://github.com/park-peter/airflow-to-dabs",
+    "version": "0.2.1"
+  }
 }
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/airflow3-migration.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/airflow3-migration.md
new file mode 100644
index 0000000..4b4d771
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/airflow3-migration.md
@@ -0,0 +1,192 @@
+# Airflow 3 Recognition and Migration Guide
+
+Reference for converting DAGs authored against **Apache Airflow 3.x**. Airflow 3 keeps the same
+operator/sensor *semantics* as Airflow 2 — the DABs mappings in `references/operator-mapping.md`
+are unchanged — but the **import paths and scheduling APIs moved**. The risk in a naïve conversion
+is not a wrong mapping; it is a DAG whose tasks are **silently missed** because the parser only
+recognized Airflow 2 import paths. Recognize the Airflow 3 authoring surface, map the clean
+equivalents, and flag the rest.
+
+This skill's approach for Airflow 3 is **recognize → safe-map → flag**:
+- **Recognize** the `airflow.sdk` and `apache-airflow-providers-standard` import paths so no task
+  is dropped.
+- **Safe-map** the constructs with clean Lakeflow equivalents (operators via the existing tiers;
+  `Asset`-based scheduling per the resolution rule).
+- **Flag** constructs with no clean equivalent (`@asset` pipelines, `AssetWatcher`, asset aliases,
+  DAG versioning, deadline alerts) in `MIGRATION_NOTES.md` — do not invent a mapping.
+
+---
+
+## How to tell a DAG is Airflow 3
+
+Any of these signals Airflow 3 authoring; parse accordingly:
+
+- Imports from `airflow.sdk` (e.g. `from airflow.sdk import dag, task, task_group, Asset`).
+- Imports from `airflow.providers.standard.*` for common operators/sensors.
+- `Asset(...)` (the Airflow 3 name for `Dataset`) in `schedule=`.
+- `schedule=` used with a **list** of assets, a **boolean** asset expression (`|`, `&`), or an
+  `AssetOrTimeSchedule`.
+
+`schedule_interval=` is **removed** in Airflow 3 (use `schedule=`), and `SubDagOperator` is
+**removed** (see below). `@dag` / `@task` / `@task_group` behave the same as in Airflow 2 once
+their import path is recognized.
+
+---
+
+## Airflow 3 scheduling defaults and semantics
+
+Reading the DAG's schedule/backfill intent depends on these Airflow 3 defaults and behaviors:
+
+- **`schedule` defaults to `None`** — a DAG with no `schedule=` runs on manual trigger only. Emit no
+  DABs `schedule`/`trigger` for it (manual/`run_job_task`-driven).
+- **`catchup` defaults to `False`** — an unset `catchup` means the DAG does **not** backfill missed
+  intervals. Only treat backfill as intended when `catchup=True` is explicit; note the backfill
+  expectation (and that DABs jobs have no catchup) in `MIGRATION_NOTES.md`.
+- **A raw-cron `schedule` uses `CronTriggerTimetable`** — the run's `logical_date` is the fire time
+  (run-after), not the start of a data interval. When a cron/timetable DAG is date-sensitive (its
+  tasks read `logical_date`/`{{ ds }}` to pick the processing window), confirm the intended window and
+  record it before mapping `{{ ds }}` → `{{job.parameters.run_date}}`; flag any timetable that can't be
+  mapped deterministically.
+
+---
+
+## Airflow 3 execution-model additions: native async and resumable
+
+Two execution-model constructs are new in Airflow 3 and affect what you parse. Neither has a DABs
+"mode" switch; migrate the underlying operation. (**Deferrable operators are NOT Airflow-3-specific** —
+they date from Airflow 2.2 — so their migration rule lives with the operator mappings in
+`references/operator-mapping.md`, not here.)
+
+### Native async TaskFlow (`@task` on `async def`) — Airflow 3.2.0
+
+Airflow **3.2.0** added native async TaskFlow tasks: `@task` decorating an `async def`, using `await`,
+`asyncio.gather`, and async hooks (`HttpAsyncHook`, `SFTPHookAsync`). This is **distinct from
+deferrable** — async tasks do many concurrent I/O ops within **one** worker slot on a shared event
+loop; deferrable frees the slot during a wait. Migration:
+
+- Map to a `notebook_task` / wheel task; keep the concurrent I/O **inside one task** by default.
+- The coroutine is **not runnable as-is** — rewrite Airflow async hooks and Connections to native async
+  clients (e.g. `aiohttp`, `asyncssh`) with auth from `dbutils.secrets`; the notebook drives the event
+  loop itself.
+- Optionally split independent `asyncio.gather()` items into a `for_each_task` — flag the changed retry
+  and UI granularity. There is no DABs "async" setting.
+
+Reference: https://airflow.apache.org/docs/task-sdk/stable/deferred-vs-async-operators.html
+
+### Resumable external jobs (`ResumableJobMixin`) — Airflow 3.3.0
+
+Airflow **3.3.0** added `ResumableJobMixin`: an operator persists the external job id before polling and,
+on retry, **reattaches** to the running external job instead of resubmitting (implementers provide
+`submit_job`, `get_job_status`, `is_job_active`, `is_job_succeeded`, `poll_until_complete`,
+`get_job_result`). Migration:
+
+- If the operation becomes a **native Databricks task**, drop the resumption mechanics.
+- If the **external job is retained**, preserve the external job id / idempotency / reattachment or
+  **flag** for review — never silently turn a resumable submission into a notebook that resubmits the
+  external job on every retry.
+
+Reference: https://airflow.apache.org/docs/task-sdk/stable/resumable-job-mixin.html
+
+---
+
+## Task SDK import equivalence (`airflow.sdk`)
+
+Airflow 3 exposes the stable authoring interface under `airflow.sdk`. Map these to the same
+handling as their Airflow 2 equivalents:
+
+| Airflow 3 (`airflow.sdk`) | Airflow 2 equivalent | Handling |
+|---|---|---|
+| `from airflow.sdk import dag` | `from airflow.decorators import dag` | Same — DAG metadata source. |
+| `from airflow.sdk import task` | `from airflow.decorators import task` | Same — TaskFlow `@task` (see `operator-mapping.md`). |
+| `from airflow.sdk import task_group` | `from airflow.decorators import task_group` | Same — TaskGroup / mapped task group. |
+| `from airflow.sdk import Asset` | `from airflow.datasets import Dataset` | `Asset` == renamed `Dataset` — asset scheduling below. |
+| `from airflow.sdk import DAG` / `BaseOperator` | `from airflow import DAG` / `airflow.models.BaseOperator` | Same. |
+| `from airflow.sdk import Variable` / `Connection` | `airflow.models.Variable` / `Connection` | Same — Variables → job params/bundle vars; Connections → secrets/UC connections. |
+| `from airflow.sdk import chain` / `cross_downstream` | `airflow.models.baseoperator.chain` / `cross_downstream` | Same — dependency-graph helpers. |
+| `from airflow.sdk import Param` (or `airflow.sdk.definitions.param.Param`) | `airflow.models.param.Param` | Same — DAG/task `params` → job parameters. |
+
+---
+
+## Standard-provider import paths (`apache-airflow-providers-standard`)
+
+In Airflow 3, common operators and sensors moved out of `airflow-core` into the
+`apache-airflow-providers-standard` provider. The **classes and their DABs mappings are unchanged**
+— only the import path differs. Recognize both the new and legacy paths.
+
+| Class | Airflow 3 import path | DABs mapping (unchanged) |
+|---|---|---|
+| `PythonOperator` | `airflow.providers.standard.operators.python` | `notebook_task` (Tier 1) |
+| `BranchPythonOperator` | `airflow.providers.standard.operators.python` | `condition_task` (Tier 2) |
+| `ShortCircuitOperator` | `airflow.providers.standard.operators.python` | `condition_task` (Tier 2) |
+| `PythonVirtualenvOperator` | `airflow.providers.standard.operators.python` | `notebook_task` + env note (Tier 2) |
+| `ExternalPythonOperator` | `airflow.providers.standard.operators.python` | `notebook_task` + env note (Tier 2) |
+| `BashOperator` | `airflow.providers.standard.operators.bash` | `notebook_task` / `spark_python_task` (Tier 1) |
+| `TriggerDagRunOperator` | `airflow.providers.standard.operators.trigger_dagrun` | `run_job_task` (Tier 1) |
+| `LatestOnlyOperator` | `airflow.providers.standard.operators.latest_only` | Flag — no direct equivalent |
+| `ExternalTaskSensor` | `airflow.providers.standard.sensors.external_task` | `trigger.table_update` / `depends_on` (Tier 3) |
+| `FileSensor` | `airflow.providers.standard.sensors.filesystem` | `trigger.file_arrival` (Tier 3) |
+| `TimeSensor` | `airflow.providers.standard.sensors.time` | absorbed into `schedule` (Tier 3) |
+| `TimeDeltaSensor` | `airflow.providers.standard.sensors.time_delta` | absorbed into `schedule` (Tier 3) |
+| `DayOfWeekSensor` | `airflow.providers.standard.sensors.weekday` | map to `schedule` day-of-week (Tier 3) |
+| `EmptyOperator` | `airflow.providers.standard.operators.empty` | Remove + rewire `depends_on` (Tier 2) |
+
+> There is no `DateTimeSensor` in the standard provider; use `TimeSensor` / `TimeDeltaSensor` /
+> `DayOfWeekSensor`.
+
+**Legacy paths:** In Airflow 3.0–3.1 the old `airflow.operators.*` / `airflow.sensors.*` import
+paths still work with deprecation warnings and are slated for removal in a later release. Recognize
+**both** the legacy and standard-provider paths so a DAG on either side converts identically.
+
+---
+
+## Assets vs Datasets, and asset scheduling
+
+"Datasets" (Airflow 2) are renamed **Assets** (Airflow 3): `airflow.sdk.Asset` replaces
+`airflow.datasets.Dataset`. Asset-based **scheduling** maps to Lakeflow `trigger.table_update`;
+the boolean/list/time-combined forms and the **Asset → UC-table resolution rule** are documented
+in `references/schedule-trigger-mapping.md` (§ Timetable, Dataset, and Asset Scheduling). Summary:
+
+- `schedule=[asset]` → `trigger.table_update` on the resolved table (single).
+- `schedule=[a, b]` (list = ALL) → `condition: ALL_UPDATED`; `a | b` → `ANY_UPDATED`; `a & b` → `ALL_UPDATED`.
+- `AssetOrTimeSchedule(...)` (time **and** asset) → **flag**; a single Lakeflow job takes a schedule
+  **or** a trigger, not both as a clean 1:1.
+- An `Asset` URI is an arbitrary string, so map to a table **only** via explicit
+  `extra={"databricks_table": "catalog.schema.table"}`, a user-supplied mapping, or the skill-local
+  `x-databricks-table:` scheme — otherwise **flag**. Never infer a table from an arbitrary URI.
+
+### `@asset` and related — flag, do not auto-map
+
+These Airflow 3 asset features have no clean Lakeflow equivalent; **flag** them in
+`MIGRATION_NOTES.md` rather than inventing a mapping:
+
+- The **`@asset` decorator** (defining asset-producing workflows) — distinct from using `Asset`
+  objects in `schedule=`.
+- **`AssetWatcher`** and event-driven asset watchers.
+- **Asset aliases**.
+- **DAG versioning / DAG bundles** (a deployment concept, not a task-graph one).
+- **Deadline alerts** (the Airflow 3 successor to SLAs).
+
+---
+
+## Removed in Airflow 3
+
+| Removed | Replacement / handling |
+|---|---|
+| `schedule_interval=` | Use `schedule=`; the parser reads both. |
+| `SubDagOperator` | Use dynamic task mapping / `TaskGroup`. The SubDag flatten in `operator-mapping.md` applies to Airflow 2 DAGs only. |
+| `execution_date` context var | Use `logical_date` / `run_id`; the Jinja `{{ ds }}`/`{{ execution_date }}` mappings in `schedule-trigger-mapping.md` still apply for templated strings. |
+| `fail_stop` DAG arg | Renamed `fail_fast` (stop the DAG run on first task failure). Record the fail-fast intent in `MIGRATION_NOTES.md`; a Lakeflow job has no single equivalent switch. |
+
+---
+
+## Recognize → safe-map → flag checklist
+
+1. **Recognize imports.** Accept `airflow.sdk.*` and `airflow.providers.standard.{operators,sensors}.*`
+   in addition to the Airflow 2 `airflow.operators.*` / `airflow.sensors.*` paths. A task whose
+   import path is unrecognized must be surfaced, never dropped.
+2. **Map operators/sensors** through the existing Tier tables in `operator-mapping.md` — the mapping
+   is import-path-independent.
+3. **Map asset scheduling** per the resolution rule (above / `schedule-trigger-mapping.md`).
+4. **Flag** `@asset`, `AssetWatcher`, asset aliases, DAG versioning, deadline alerts,
+   `AssetOrTimeSchedule`, and any asset whose URI does not resolve to a UC table — in
+   `MIGRATION_NOTES.md`, with the reason.
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/dab-schema-reference.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/dab-schema-reference.md
new file mode 100644
index 0000000..16f5b2a
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/dab-schema-reference.md
@@ -0,0 +1,718 @@
+# Databricks Asset Bundles YAML Schema Reference
+
+Condensed reference for generating DABs configuration files. Covers all task types, triggers, clusters, and job-level configuration supported as of Jan 2026.
+
+---
+
+## Top-Level Structure: `databricks.yml`
+
+```yaml
+bundle:
+  name: 
+
+include:
+  - resources/*.yml
+
+variables:
+  spark_version:
+    description: Spark runtime version
+    default: ""
+  node_type_id:
+    description: Cluster node type
+    default: ""
+  warehouse_id:
+    description: SQL warehouse ID for SQL tasks
+    default: ""
+
+targets:
+  dev:
+    mode: development
+    workspace:
+      host: ${var.dev_workspace_url}
+  prod:
+    mode: production
+    workspace:
+      host: ${var.prod_workspace_url}
+    run_as:
+      service_principal_name: ${var.service_principal}
+```
+
+---
+
+## Python-Defined Resources (PyDABs)
+
+Resources can also be defined in Python instead of YAML via a top-level `python:` block in `databricks.yml`. Used by dbt factory mode (see `references/operator-mapping.md`) to generate one task per dbt object at deploy time.
+
+```yaml
+python:
+  venv_path: .venv                                # venv with databricks-bundles installed
+  resources:
+    - "resources.:load_resources"    # one module:function entry per generator
+```
+
+The referenced function is called by the Databricks CLI during both `bundle validate` and `bundle deploy`:
+
+```python
+from databricks.bundles.core import Bundle, Resources
+from databricks.bundles.jobs import Job
+
+def load_resources(bundle: Bundle) -> Resources:
+    resources = Resources()
+    resources.add_job("", Job.from_dict({...}))   # dict uses Jobs API fields
+    return resources
+```
+
+Rules:
+
+- `python:` coexists with `include: - resources/*.yml`. YAML jobs and Python-registered jobs share one resources namespace, so YAML can reference a Python-registered job (e.g. `job_id: ${resources.jobs..id}` in a `run_job_task`).
+- `load_resources` runs on every `bundle validate` too — any deploy-time file writers inside it must be idempotent.
+- Relative paths (e.g. `notebook_path`) in Python-defined jobs resolve against the bundle root.
+- Requires the venv at `venv_path` to exist with `databricks-bundles` installed before running `validate`/`deploy` (`uv sync --dev` with the generated `pyproject.toml`).
+
+---
+
+## Job Resource Definition
+
+Defined in `resources/*.yml` files, included by `databricks.yml`.
+
+```yaml
+resources:
+  jobs:
+    :
+      name: 
+      description: 
+      tags:
+        team: data-engineering
+        source: airflow-migration
+      max_concurrent_runs: 1
+      timeout_seconds: 3600
+
+      # Schedule (see Schedule section below)
+      schedule:
+        quartz_cron_expression: "0 0 8 * * ?"
+        timezone_id: "America/New_York"
+        pause_status: UNPAUSED
+
+      # OR Trigger (see Trigger section below)
+      trigger:
+        file_arrival:
+          url: 
+
+      # Email notifications (job-level)
+      email_notifications:
+        on_start:
+          - "team@example.com"
+        on_success:
+          - "team@example.com"
+        on_failure:
+          - "oncall@example.com"
+
+      # Job parameters (accessible by all tasks)
+      parameters:
+        # For an Airflow {{ ds }} that is a logical/partition date on a SCHEDULED job, default to the
+        # scheduled trigger time (correct on normal runs); a native Databricks backfill overrides it
+        # with {{backfill.iso_date}}. Use {{job.start_time.iso_date}} for wall-clock "today" semantics
+        # or an event-triggered job (trigger.time is unreliable there — see schedule-trigger-mapping.md).
+        - name: run_date
+          default: "{{job.trigger.time.iso_date}}"
+        - name: env
+          default: "dev"
+
+      # Shared cluster definitions
+      job_clusters:
+        - job_cluster_key: shared-cluster
+          new_cluster:
+            spark_version: ${var.spark_version}
+            node_type_id: ${var.node_type_id}
+            num_workers: 2
+            spark_conf:
+              spark.sql.shuffle.partitions: "200"
+            spark_env_vars:
+              ENV: "{{job.parameters.env}}"
+
+      # Task list
+      tasks:
+        - task_key: 
+          # ... task definition (see Task Types below)
+```
+
+---
+
+## Task Types
+
+Each task must have exactly one task type field (e.g., `notebook_task`, `sql_task`). All tasks share these common fields:
+
+### Common Task Fields
+
+```yaml
+- task_key:             # Required. 1-100 chars, [a-zA-Z0-9_-]
+  description: 
+  depends_on:                              # Optional dependency list
+    - task_key: 
+      outcome: "true"                      # Only for condition_task dependencies
+  timeout_seconds: 3600                    # 0 = no timeout
+  run_if: ALL_SUCCESS                      # ALL_SUCCESS | ALL_DONE | NONE_FAILED | AT_LEAST_ONE_SUCCESS | ALL_FAILED | AT_LEAST_ONE_FAILED
+  # Cluster (one of):
+  job_cluster_key: shared-cluster          # Reference to job_clusters entry
+  existing_cluster_id: "1234-567890-abc"   # Use existing cluster
+  new_cluster:                             # Create new cluster for this task
+    spark_version: ${var.spark_version}
+    node_type_id: ${var.node_type_id}
+    num_workers: 2
+  # Notifications (task-level)
+  email_notifications:
+    on_start: []
+    on_success: []
+    on_failure: []
+```
+
+---
+
+### notebook_task
+
+Runs a Databricks notebook (.py, .ipynb, .sql, .r, .scala).
+
+```yaml
+- task_key: my_notebook
+  notebook_task:
+    notebook_path: ../src/my_notebook.py        # Required. Relative to config file.
+    source: WORKSPACE                           # WORKSPACE (default) or GIT
+    base_parameters:                            # Optional key-value params
+      param1: "value1"
+      param2: "{{job.parameters.env}}"
+    warehouse_id: ${var.warehouse_id}           # Optional. For SQL-only notebooks.
+```
+
+---
+
+### spark_python_task
+
+Runs a Python file on a Spark cluster.
+
+```yaml
+- task_key: my_python_script
+  spark_python_task:
+    python_file: ../src/my_script.py            # Required. Path to .py file.
+    source: WORKSPACE
+    parameters:                                 # Optional positional args
+      - "--date"
+      - "{{job.parameters.run_date}}"
+```
+
+---
+
+### python_wheel_task
+
+Runs an entry point from a Python wheel package.
+
+```yaml
+- task_key: my_wheel_task
+  python_wheel_task:
+    entry_point: run                            # Required. Function or class name.
+    package_name: my_package                    # Required. Package name.
+    named_parameters:                           # Optional keyword args (OR parameters, not both)
+      env: "prod"
+      date: "{{job.parameters.run_date}}"
+  libraries:
+    - whl: ../dist/my_package-*.whl
+```
+
+---
+
+### spark_jar_task
+
+Runs a main class from a JAR file.
+
+```yaml
+- task_key: my_jar_task
+  spark_jar_task:
+    main_class_name: com.example.Main           # Required. Fully-qualified class name.
+    parameters:                                 # Optional positional args
+      - "--input"
+      - "/data/input"
+  libraries:
+    - jar: /Volumes/main/default/jars/app.jar
+```
+
+---
+
+### sql_task
+
+Runs a SQL query, SQL file, or refreshes a SQL alert/dashboard.
+
+```yaml
+# SQL file
+- task_key: my_sql_file
+  sql_task:
+    warehouse_id: ${var.warehouse_id}           # Required.
+    file:
+      path: ../src/query.sql                    # Path to .sql file
+      source: WORKSPACE
+    parameters:
+      run_date: "{{job.parameters.run_date}}"
+
+# SQL query (by ID)
+- task_key: my_sql_query
+  sql_task:
+    warehouse_id: ${var.warehouse_id}
+    query:
+      query_id: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
+
+# SQL alert
+- task_key: my_sql_alert
+  sql_task:
+    warehouse_id: ${var.warehouse_id}
+    alert:
+      alert_id: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
+```
+
+---
+
+### pipeline_task
+
+Triggers a Lakeflow Declarative Pipeline update (a DLT/declarative pipeline, or a Lakeflow Connect
+managed-ingestion pipeline — see below).
+
+```yaml
+- task_key: my_pipeline
+  pipeline_task:
+    pipeline_id: ${resources.pipelines.my_pipeline.id}   # Required. Bundle ref or pipeline ID.
+    full_refresh: false                                    # Optional. Default false.
+```
+
+---
+
+### Managed-ingestion pipelines (Lakeflow Connect)
+
+A Lakeflow Connect ingestion pipeline is a `resources.pipelines.` entry carrying an
+`ingestion_definition`. See `references/lakeflow-connect.md` for when to choose Connect over a Jobs
+task. The schema's `ingestion_definition` description warns it should not be mixed with a normal DLT
+pipeline's `libraries` settings; note, however, that current query-based-ingestion examples do set
+`catalog`/`target` alongside `ingestion_definition` — follow the field combination in the current docs /
+`databricks bundle schema` for your CLI version rather than assuming a blanket incompatibility.
+
+**Combined ingestion (primary/canonical)** — one pipeline, `connection_name` on the ingestion
+definition (SaaS, files, query-based DB, and CDC via `connection_name`; add `connector_type` when the
+source supports both query-based and CDC):
+
+```yaml
+resources:
+  pipelines:
+    salesforce_ingest:
+      name: salesforce_ingest
+      ingestion_definition:
+        connection_name: ${var.salesforce_connection}   # UC connection (created out-of-band)
+        objects:
+          - table:
+              source_schema: salesforce
+              source_table: opportunity
+              destination_catalog: ${var.catalog}
+              destination_schema: ${var.schema}
+```
+
+**Foreign-catalog ingestion (query-based, for federated sources — Snowflake/BigQuery/Redshift/Synapse)**
+— set `ingest_from_uc_foreign_catalog: true` and reference the source by `source_catalog/schema/table`
+(no `connection_name` / gateway on the ingestion definition):
+
+```yaml
+resources:
+  pipelines:
+    snowflake_ingest:
+      name: snowflake_ingest
+      ingestion_definition:
+        ingest_from_uc_foreign_catalog: true
+        objects:
+          - table:
+              source_catalog: ${var.snowflake_foreign_catalog}   # a UC foreign catalog (below)
+              source_schema: public
+              source_table: orders
+              destination_catalog: ${var.catalog}
+              destination_schema: ${var.schema}
+```
+
+The **foreign catalog** is a bundle resource (`resources.catalogs`), created from a UC connection. It
+requires `bundle.engine: direct` — "defining catalogs is only supported if you are using the direct
+deployment engine." A foreign catalog needs `connection_name` **plus source-specific `options`** (e.g.
+`options: { database: '' }` for Snowflake/PostgreSQL/Redshift per CREATE FOREIGN CATALOG); a
+`connection_name`-only catalog can pass schema validation but fail at deploy. **Reference an existing
+foreign catalog by default; only create one when the bundle should own it.**
+
+```yaml
+bundle:
+  name: snowflake-ingest
+  engine: direct                     # required to define catalogs in a bundle
+
+resources:
+  catalogs:
+    snowflake_fc:
+      name: ${var.snowflake_foreign_catalog}
+      connection_name: ${var.snowflake_connection}
+      options:
+        database: ${var.snowflake_database}   # source-specific; confirm required options per source
+```
+
+> **UC connections are NOT bundle resources.** Create the connection out-of-band (`CREATE CONNECTION`
+> / UI) and reference it by name. Record it as a prerequisite in `MIGRATION_NOTES.md` (name, auth,
+> networking).
+
+**Gateway CDC (Private Preview — requires enrollment).** Log-based CDC for a database source uses a
+**separate** `gateway_definition` pipeline plus an ingestion pipeline joined by `ingestion_gateway_id`
+(`gateway_definition` and `ingestion_definition` are never on the same pipeline). The bundle schema
+marks `gateway_definition` `[Private Preview]` / `doNotSuggest` — generate this path **only** with
+connector-specific verification and confirmed workspace Private-Preview enrollment; it is not the
+default. Prefer combined CDC (`connection_name` + `connector_type`) where the connector supports it.
+
+**Orchestration.** A **triggered** ingestion pipeline is driven by a `pipeline_task` at the original
+dependency position. A **continuous** pipeline (streaming connectors like Kafka/RabbitMQ, or any
+connector documented continuous-only) is not `pipeline_task`-driven — run it standalone and have the
+downstream job depend on a job-level `trigger.table_update` on its destination table. Run mode is
+per-connector; confirm it, don't assume.
+
+---
+
+### dbt_task
+
+Runs dbt commands.
+
+```yaml
+- task_key: my_dbt_task
+  dbt_task:
+    commands:                                   # Required. Up to 10 commands.
+      - "dbt deps"
+      - "dbt seed"
+      - "dbt run"
+      - "dbt test"
+    project_directory: ../dbt/my_project        # Optional. Defaults to repo root.
+    warehouse_id: ${var.warehouse_id}           # Optional. Omit profiles_directory when set.
+    # profiles_directory: ../dbt/profiles       # Optional. Use only when warehouse_id is omitted.
+    catalog: main                               # Optional. Requires warehouse_id.
+    schema: transforms                          # Optional.
+  libraries:
+    - pypi:
+        package: "dbt-databricks>=1.0.0,<2.0.0"
+```
+
+A single `dbt_task` runs the whole invocation as one opaque task. For one task per dbt model/seed/snapshot/test (per-model observability and retries), use dbt factory mode instead — see the dbt conversion decision point in `references/operator-mapping.md`.
+
+---
+
+### run_job_task
+
+Triggers another Databricks job.
+
+```yaml
+- task_key: trigger_downstream
+  run_job_task:
+    job_id: ${resources.jobs.downstream-job.id}  # Required. Job ID or substitution.
+    job_parameters:                               # Optional.
+      env: "prod"
+```
+
+**Nesting limit:** Run Job tasks may nest at most **3 levels deep** (a job runs a job runs a
+job); Databricks rejects deeper nesting and circular dependencies. A `for_each_task` whose body
+is a `run_job_task` (the mapped-task-group pattern) consumes one of those levels — budget the
+remaining depth accordingly.
+
+**Concurrency of the target job:** The target job's own `max_concurrent_runs` (default **1**)
+gates how many of its runs proceed at once. When a job is triggered repeatedly — e.g. a
+`for_each_task` with `concurrency > 1` whose body is a `run_job_task` — raise the target job's
+`max_concurrent_runs` to at least that concurrency (and account for **overlapping parent runs**),
+otherwise excess triggers serialize. Also set `queue: { enabled: true }` explicitly on the target
+job: a bundle/API-defined job does **not** inherit the UI's default-on queueing, so without it
+excess concurrent triggers are **skipped** rather than queued (queued runs wait up to 48 h).
+
+```yaml
+resources:
+  jobs:
+    region_pipeline_job:
+      name: region_pipeline
+      max_concurrent_runs: 8          # ≥ the driving for_each concurrency (+ overlapping parents)
+      queue:
+        enabled: true                 # bundle jobs don't inherit UI default-on queueing
+      # ... tasks ...
+```
+
+---
+
+### condition_task
+
+If/else conditional logic. Does not require a cluster.
+
+```yaml
+- task_key: check_condition
+  condition_task:
+    left: "{{job.parameters.env}}"              # Required. String, dynamic ref, or task value.
+    op: EQUAL_TO                                # Required. See operators below.
+    right: "prod"                               # Required.
+
+# Operators: EQUAL_TO, NOT_EQUAL, GREATER_THAN, GREATER_THAN_OR_EQUAL, LESS_THAN, LESS_THAN_OR_EQUAL
+
+# Downstream tasks use outcome:
+- task_key: prod_task
+  depends_on:
+    - task_key: check_condition
+      outcome: "true"
+  notebook_task:
+    notebook_path: ../src/prod.py
+
+- task_key: dev_task
+  depends_on:
+    - task_key: check_condition
+      outcome: "false"
+  notebook_task:
+    notebook_path: ../src/dev.py
+```
+
+---
+
+### for_each_task
+
+Iterates a **single** nested task over an array of inputs.
+
+```yaml
+- task_key: process_all
+  for_each_task:
+    inputs: "{{tasks.generate_list.values.items}}"  # Required. JSON array or ref (see forms below).
+    concurrency: 5                                   # Optional. Max parallel iterations. Default 1.
+    task:                                            # Required. Exactly ONE nested task definition.
+      task_key: process_item
+      notebook_task:
+        notebook_path: ../src/process_item.py
+        base_parameters:
+          item: "{{input}}"                          # Whole element. Use {{input.field}} for a field.
+```
+
+**Nested task — exactly one.** `for_each_task.task` holds a single task, not a subgraph, and it
+**cannot** be another `for_each_task`. It may be any standard task type, including a
+`run_job_task` — to fan a *multi-step subgraph* out over a collection, make the nested task a
+`run_job_task` pointing at a child job that contains the subgraph (see `run_job_task` above for
+the concurrency/nesting rules that pattern requires).
+
+**Iteration reference.** Inside the nested task, `{{input}}` is the current element and
+`{{input.}}` is a field of an object element. Use them in the nested task's parameter values
+(`notebook_task.base_parameters`, `run_job_task.job_parameters`, task `parameters`).
+
+**`inputs` forms and size limits** (all must be JSON-serializable — choose the transport by size):
+
+| Form | Max size |
+|---|---|
+| JSON-array literal, e.g. `'["a","b"]'` or `'[{"t":"x"}]'` | 5,000 characters |
+| Task-value ref `{{tasks..values.}}` (array produced upstream) | 48 KiB |
+| Job-parameter ref `{{job.parameters.}}` | 10,000 characters |
+
+**`concurrency`** defaults to **1** (sequential). Set it to restore parallel fan-out; when the
+body is a `run_job_task`, also raise the child job's `max_concurrent_runs` and enable its queue
+(see `run_job_task`).
+
+**No cross-iteration outputs.** A task outside the `for_each_task` can depend on the for-each task
+as a whole, but cannot read the individual iterations' task values. To consume per-iteration
+results downstream, have each iteration persist its result (e.g. write to a table/volume) and add
+a separate aggregation task that reads those **persisted** results — not the original input array.
+
+---
+
+### dashboard_task
+
+Refreshes a Lakeview dashboard.
+
+```yaml
+- task_key: refresh_dashboard
+  dashboard_task:
+    dashboard_id: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"  # Required.
+    warehouse_id: ${var.warehouse_id}                       # Optional.
+```
+
+---
+
+### clean_rooms_notebook_task
+
+Runs a notebook inside a Databricks Clean Room.
+
+```yaml
+- task_key: clean_room_analysis
+  clean_rooms_notebook_task:
+    clean_room_name: "partner-clean-room"       # Required.
+    notebook_name: "shared_analysis"             # Required.
+```
+
+---
+
+## Schedule Configuration
+
+Time-based scheduling using Quartz cron expressions (6-7 fields).
+
+```yaml
+schedule:
+  quartz_cron_expression: "0 0 8 * * ?"     # Required. Seconds Minutes Hours DayOfMonth Month DayOfWeek [Year]
+  timezone_id: "America/New_York"             # Required.
+  pause_status: UNPAUSED                      # PAUSED or UNPAUSED
+```
+
+**Quartz cron field order:** `Seconds Minutes Hours DayOfMonth Month DayOfWeek [Year]`
+
+Use `?` for DayOfMonth or DayOfWeek when the other is specified. This differs from standard 5-field Unix cron.
+
+---
+
+## Trigger Configuration
+
+Event-driven triggers (mutually exclusive with `schedule`).
+
+### File Arrival
+
+```yaml
+trigger:
+  file_arrival:
+    url: "s3://bucket/path/"                             # Required. UC external location or volume URL.
+    min_time_between_triggers_seconds: 60                # Optional.
+    wait_after_last_change_seconds: 60                   # Optional. Minimum allowed is 60.
+```
+
+### Table Update
+
+```yaml
+trigger:
+  table_update:
+    condition: ANY_UPDATED                               # ANY_UPDATED or ALL_UPDATED
+    table_names:                                         # Required. List of UC table names.
+      - "main.silver.transactions"
+      - "main.silver.customers"
+    min_time_between_triggers_seconds: 300               # Optional.
+    wait_after_last_change_seconds: 60                   # Optional.
+```
+
+### Continuous
+
+Use continuous mode for always-on execution semantics (`@continuous` in Airflow).
+
+```yaml
+continuous:
+  pause_status: UNPAUSED
+```
+
+For periodic event triggers, use:
+
+```yaml
+trigger:
+  periodic:
+    interval: 1
+    unit: HOURS                                          # HOURS, DAYS, WEEKS
+```
+
+---
+
+## Cluster Configuration
+
+Three ways to assign compute to a task:
+
+### New Cluster (per-task)
+
+```yaml
+new_cluster:
+  spark_version: ${var.spark_version}
+  node_type_id: ${var.node_type_id}
+  num_workers: 2                                # Fixed size
+  # OR autoscale:
+  autoscale:
+    min_workers: 1
+    max_workers: 8
+  spark_conf:
+    spark.sql.shuffle.partitions: "200"
+  spark_env_vars:
+    ENV: "prod"
+  data_security_mode: SINGLE_USER               # For Unity Catalog
+```
+
+### Job Cluster (shared across tasks in same job)
+
+```yaml
+# Defined at job level:
+job_clusters:
+  - job_cluster_key: shared-cluster
+    new_cluster:
+      spark_version: ${var.spark_version}
+      node_type_id: ${var.node_type_id}
+      num_workers: 2
+
+# Referenced in task:
+- task_key: my_task
+  job_cluster_key: shared-cluster
+```
+
+### Existing Cluster
+
+```yaml
+- task_key: my_task
+  existing_cluster_id: "1234-567890-abcdef12"
+```
+
+---
+
+### Serverless Environments
+
+Serverless notebook tasks omit all cluster fields (`job_cluster_key`, `new_cluster`, `existing_cluster_id`). Referencing a job-level environment via `environment_key` is OPTIONAL — use it to pin dependencies; without it the task runs on the default serverless environment.
+
+```yaml
+resources:
+  jobs:
+    :
+      environments:
+        - environment_key: Default
+          spec:
+            # Either a pre-built base-environment file synced with the bundle
+            # (built once; tasks skip per-run pip installs):
+            base_environment: ${workspace.file_path}/dbt_serverless_env.yaml
+            # OR inline dependencies (mutually exclusive with base_environment):
+            # environment_version: "5"
+            # dependencies:
+            #   - dbt-databricks==1.12.2
+            #   - dbt-core==1.11.12    # pin dbt-core too (see note below)
+      tasks:
+        - task_key: my_task
+          environment_key: Default
+          notebook_task:
+            notebook_path: ../src/my_task.py
+```
+
+The base-environment file itself contains the same spec fields:
+
+```yaml
+environment_version: "5"        # serverless environment version
+dependencies:
+  - dbt-databricks==1.12.2
+  - dbt-core==1.11.12           # pin dbt-core too, not just the adapter
+```
+
+For dbt factory mode, pin **both** `dbt-databricks` and `dbt-core` to the exact versions in the bundle venv. `dbt-databricks` alone allows a `dbt-core` range, but the factory glue imports the local `dbt-core` for its selector-exactness check — the runtime environment must resolve the identical `dbt-core` for that guarantee to hold.
+
+---
+
+## Variable Substitutions
+
+DABs supports dynamic substitutions using `${}` syntax:
+
+| Pattern | Description |
+|---|---|
+| `${var.}` | Bundle variable |
+| `${resources.jobs..id}` | Job ID from another resource in the bundle |
+| `${resources.pipelines..id}` | Pipeline ID from the bundle |
+| `${workspace.root_path}` | Workspace root path for the bundle |
+| `${bundle.name}` | Bundle name |
+
+---
+
+## Dynamic Value References (in task parameters)
+
+Used within task parameter values using `{{}}` syntax:
+
+| Pattern | Description |
+|---|---|
+| `{{job.parameters.}}` | Job-level parameter |
+| `{{job.run_id}}` | Current run ID |
+| `{{job.start_time.iso_date}}` | Actual execution start date, UTC (YYYY-MM-DD). Wall-clock — drifts with queue delay/retries. |
+| `{{job.trigger.time.iso_date}}` | Scheduled trigger date, UTC (rounded to the minute for cron). The right default for a logical/partition `{{ ds }}` on a scheduled job — correct on normal runs, where `start_time` would drift. Other parts: `iso_datetime`, `year`, `month`, `day`, `timestamp_ms`. |
+| `{{backfill.iso_date}}` | Start of the time range for a native [backfill](https://docs.databricks.com/aws/en/jobs/backfill-jobs) run — the logical date being replayed. Set by the backfill UI as a per-run override of a date/time job parameter. Also `iso_datetime`, `timestamp_ms`, `year`, `month`, `day`. |
+| `{{tasks..values.}}` | Task value set by upstream task via `dbutils.jobs.taskValues.set()` |
+| `{{input}}` | Current element inside a `for_each_task` nested task |
+| `{{input.}}` | A field of the current element (when iterating objects) |
+| `{{job.repair_count}}` | Number of repair attempts |
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/hadoop-migration-guide.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/hadoop-migration-guide.md
new file mode 100644
index 0000000..ebab686
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/hadoop-migration-guide.md
@@ -0,0 +1,387 @@
+# Hadoop/HDFS to Databricks Migration Guide
+
+Reference for converting on-prem Airflow DAGs that orchestrate Spark jobs on Hadoop/YARN clusters to Databricks Asset Bundles. Covers HDFS path conversion, YARN Spark config cleanup, Hive metastore migration, data ingestion alternatives, and detection of `spark-submit` commands embedded in BashOperator/SSHOperator tasks.
+
+---
+
+## HDFS Path Conversion
+
+All `hdfs://` paths in Spark job code, operator parameters, and configs must be converted to Databricks-compatible storage paths.
+
+### Path Mapping Table
+
+| On-Prem Pattern | Databricks Equivalent | Notes |
+|---|---|---|
+| `hdfs://namenode:8020/data/...` | `s3://bucket/data/...` or `abfss://container@account.dfs.core.windows.net/data/...` | Cloud storage mounted or accessed directly |
+| `hdfs:///user/hive/warehouse/db.db/table` | Unity Catalog managed table: `catalog.schema.table` | No path needed -- use `spark.read.table()` |
+| `/user/data/landing/` (implicit HDFS) | `/Volumes/catalog/schema/volume/landing/` | Unity Catalog volumes for file-based access |
+| `hdfs://namenode/tmp/staging/` | `/tmp/` or a UC volume for staging | Ephemeral staging paths |
+| `dbfs:/mnt/...` (legacy DBFS mount) | `/Volumes/catalog/schema/volume/...` | Migrate mounts to UC volumes |
+
+### Conversion Rules
+
+1. **Identify all HDFS paths** in Spark job source files (`.py`, `.jar` configs, `.sql`). Search for:
+   - `hdfs://` prefixed paths
+   - Bare absolute paths used with `spark.read`/`spark.write` (often implicit HDFS)
+   - Paths in `--files`, `--jars`, `--py-files` arguments to `spark-submit`
+
+2. **Map to cloud storage or Unity Catalog:**
+   - **Tables** (Hive warehouse paths): convert to UC table references (`catalog.schema.table`)
+   - **Landing/raw files**: convert to UC external locations or volumes
+   - **Intermediate/staging**: convert to UC volumes or temp paths
+   - **JARs/wheels/dependencies**: upload to UC volumes (`/Volumes/catalog/schema/libs/`)
+
+3. **In generated notebooks**, replace HDFS reads/writes:
+
+   ```python
+   # Before (HDFS)
+   df = spark.read.parquet("hdfs://namenode:8020/data/raw/events/")
+   df.write.parquet("hdfs://namenode:8020/data/silver/events/")
+
+   # After (Unity Catalog / cloud storage)
+   df = spark.read.parquet("s3://datalake-bucket/data/raw/events/")
+   df.write.format("delta").saveAsTable("catalog.silver.events")
+   ```
+
+4. **Flag in MIGRATION_NOTES.md**: list every HDFS path found with its proposed Databricks equivalent. This requires input from the customer to confirm cloud storage bucket names, Unity Catalog catalog/schema structure, and volume locations.
+
+---
+
+## YARN/Hadoop Spark Config Translation
+
+SparkSubmitOperator `conf` and `spark-submit` `--conf` flags include YARN/Hadoop-specific settings that must be cleaned up for Databricks.
+
+### Configs to Remove (not applicable on Databricks)
+
+| Spark Config | Reason |
+|---|---|
+| `spark.yarn.queue` | No YARN queues. Databricks uses cluster policies for governance. |
+| `spark.yarn.executor.memoryOverhead` | Use `spark.executor.memoryOverhead` instead (same effect, YARN prefix removed). |
+| `spark.yarn.driver.memoryOverhead` | Use `spark.driver.memoryOverhead` instead. |
+| `spark.yarn.am.memory` | Not applicable. |
+| `spark.yarn.am.cores` | Not applicable. |
+| `spark.yarn.submit.waitAppCompletion` | Not applicable. |
+| `spark.yarn.maxAppAttempts` | Use DABs `max_retries` on the task instead. |
+| `spark.hadoop.fs.defaultFS` | Not needed -- Databricks configures storage access via UC or instance profiles. |
+| `spark.hadoop.dfs.*` | HDFS namenode configs not needed. |
+| `spark.hadoop.mapreduce.*` | MapReduce configs not applicable. |
+| `spark.hadoop.yarn.*` | All YARN-specific Hadoop configs. |
+| `spark.eventLog.dir` (HDFS path) | Databricks manages Spark event logs automatically. |
+| `spark.history.fs.logDirectory` | Managed by Databricks. |
+
+### Configs to Translate
+
+| On-Prem Config | Databricks Equivalent | Notes |
+|---|---|---|
+| `spark.executor.instances` | `num_workers` on `new_cluster` | Fixed cluster size. Or use `autoscale.min_workers`/`max_workers`. |
+| `spark.executor.memory` | `spark.executor.memory` in `spark_conf` | Still valid, but Databricks auto-tunes. Often removable. |
+| `spark.executor.cores` | `spark.executor.cores` in `spark_conf` | Still valid. Databricks optimizes by default. |
+| `spark.driver.memory` | `spark.driver.memory` in `spark_conf` | Still valid. |
+| `spark.sql.shuffle.partitions` | `spark.sql.shuffle.partitions` in `spark_conf` | Still valid. Databricks AQE auto-tunes this. Consider removing. |
+| `spark.dynamicAllocation.enabled` | Databricks autoscaling | Use `autoscale` on `new_cluster` instead. Remove the Spark config. |
+| `spark.dynamicAllocation.minExecutors` | `autoscale.min_workers` | Map directly. |
+| `spark.dynamicAllocation.maxExecutors` | `autoscale.max_workers` | Map directly. |
+| `spark.sql.warehouse.dir` | Not needed | UC manages warehouse location. |
+| `spark.hive.metastore.uris` | Not needed if using UC | UC is the metastore. For external HMS, use `spark.hadoop.hive.metastore.uris`. |
+| `--master yarn` | Remove | Databricks manages the Spark master. |
+| `--deploy-mode cluster\|client` | Remove | Databricks always runs in cluster mode. |
+| `--keytab` / `--principal` | Remove | Kerberos not needed. Use UC/instance profiles for auth. |
+
+### DABs Cluster Config Example (translated from YARN)
+
+**Before (spark-submit on YARN):**
+
+```bash
+spark-submit \
+  --master yarn \
+  --deploy-mode cluster \
+  --queue etl_queue \
+  --num-executors 10 \
+  --executor-memory 8g \
+  --executor-cores 4 \
+  --driver-memory 4g \
+  --conf spark.dynamicAllocation.enabled=true \
+  --conf spark.dynamicAllocation.minExecutors=5 \
+  --conf spark.dynamicAllocation.maxExecutors=20 \
+  --conf spark.yarn.executor.memoryOverhead=2g \
+  --conf spark.sql.shuffle.partitions=400 \
+  --conf spark.hadoop.fs.defaultFS=hdfs://namenode:8020 \
+  /opt/spark/jobs/etl_pipeline.py --date 2024-01-15
+```
+
+**After (DABs job cluster):**
+
+```yaml
+job_clusters:
+  - job_cluster_key: etl-cluster
+    new_cluster:
+      spark_version: "15.4.x-scala2.12"
+      node_type_id: ${var.node_type_id}
+      autoscale:
+        min_workers: 5
+        max_workers: 20
+      spark_conf:
+        spark.executor.memory: "8g"
+        spark.executor.cores: "4"
+        spark.driver.memory: "4g"
+        spark.executor.memoryOverhead: "2g"
+        # spark.sql.shuffle.partitions removed -- AQE handles this
+      data_security_mode: SINGLE_USER
+```
+
+---
+
+## Hive Metastore to Unity Catalog
+
+On-prem Hadoop clusters use a Hive metastore. Tables referenced as `database.table` need to become `catalog.schema.table` in Unity Catalog.
+
+### Table Reference Conversion
+
+| Hive Pattern | Unity Catalog Equivalent |
+|---|---|
+| `database_name.table_name` | `catalog.schema.table_name` |
+| `default.table_name` | `catalog.default.table_name` |
+| `spark.sql("SELECT * FROM db.table")` | `spark.sql("SELECT * FROM catalog.schema.table")` |
+| `spark.read.table("db.table")` | `spark.read.table("catalog.schema.table")` |
+| `spark.write.saveAsTable("db.table")` | `spark.write.saveAsTable("catalog.schema.table")` |
+| `CREATE TABLE db.table ...` | `CREATE TABLE catalog.schema.table ...` |
+| `INSERT INTO db.table ...` | `INSERT INTO catalog.schema.table ...` |
+
+### Conversion Rules
+
+1. **Define a catalog/schema mapping** as a DABs variable:
+
+   ```yaml
+   variables:
+     catalog:
+       description: Unity Catalog name
+       default: "main"
+     schema_prefix:
+       description: Schema prefix mapping from Hive databases
+       default: ""
+   ```
+
+2. **In generated notebooks**, add a `USE CATALOG` / `USE SCHEMA` at the top:
+
+   ```python
+   # Databricks notebook source
+   spark.sql(f"USE CATALOG {dbutils.widgets.get('catalog')}")
+   spark.sql(f"USE SCHEMA {dbutils.widgets.get('schema')}")
+   ```
+
+3. **Flag in MIGRATION_NOTES.md**: list all Hive databases referenced and their proposed UC catalog.schema mapping. This requires customer input on UC structure.
+
+---
+
+## BashOperator spark-submit Detection
+
+On-prem Airflow setups frequently wrap `spark-submit` in a `BashOperator` or `SSHOperator` instead of using `SparkSubmitOperator`. The skill should detect this pattern and convert it to a proper DABs task type.
+
+### Detection Pattern
+
+Look for these patterns in `bash_command` or `command` parameters:
+
+```python
+# Pattern 1: Direct spark-submit
+BashOperator(
+    task_id="run_etl",
+    bash_command="spark-submit --master yarn --class com.example.ETL /opt/jars/etl.jar --date {{ ds }}"
+)
+
+# Pattern 2: spark-submit via script
+BashOperator(
+    task_id="run_etl",
+    bash_command="/opt/scripts/run_etl.sh {{ ds }}"
+)
+# Where run_etl.sh contains: spark-submit ...
+
+# Pattern 3: SSHOperator to edge node
+SSHOperator(
+    task_id="run_etl",
+    ssh_conn_id="hadoop_edge_node",
+    command="spark-submit --master yarn /opt/spark/jobs/etl.py"
+)
+```
+
+### Conversion Rules
+
+1. **If `bash_command` contains `spark-submit`**:
+   - Parse out the application path (`.py` or `.jar`), `--class`, `--conf` flags, and application arguments
+   - Convert to `spark_python_task` (if `.py`) or `spark_jar_task` (if `.jar`)
+   - Apply YARN config cleanup (see above)
+   - Extract the application file to `src/` and update the path
+
+2. **If `bash_command` calls a shell script** that wraps `spark-submit`:
+   - Flag in MIGRATION_NOTES.md: "Shell script `run_etl.sh` wraps spark-submit. Extract the Spark job and convert to a direct task."
+   - If the script is available, parse the `spark-submit` command from it
+
+3. **If SSHOperator runs `spark-submit` on a remote host**:
+   - Same as pattern 1 -- extract the spark-submit command and convert to a DABs task
+   - The SSH hop is no longer needed since Databricks runs the job directly
+
+### Example Conversion
+
+**Airflow:**
+
+```python
+run_etl = BashOperator(
+    task_id="run_daily_etl",
+    bash_command="""
+        spark-submit \
+            --master yarn \
+            --deploy-mode cluster \
+            --num-executors 10 \
+            --executor-memory 8g \
+            --conf spark.yarn.queue=etl \
+            --conf spark.sql.shuffle.partitions=200 \
+            --class com.example.DailyETL \
+            /opt/jars/analytics-1.0.jar \
+            --date {{ ds }} \
+            --input hdfs:///data/raw/ \
+            --output hdfs:///data/silver/
+    """,
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: run_daily_etl
+  job_cluster_key: etl-cluster
+  spark_jar_task:
+    main_class_name: com.example.DailyETL
+    parameters:
+      - "--date"
+      - "{{job.parameters.run_date}}"
+      - "--input"
+      - "s3://datalake-bucket/data/raw/"
+      - "--output"
+      - "catalog.silver.daily_output"
+  libraries:
+    - jar: /Volumes/main/default/libs/analytics-1.0.jar
+```
+
+**MIGRATION_NOTES.md entry:**
+
+```
+- Task `run_daily_etl`: BashOperator wrapping spark-submit detected and converted to spark_jar_task.
+  - HDFS paths `hdfs:///data/raw/` and `hdfs:///data/silver/` need cloud storage mapping.
+  - JAR `/opt/jars/analytics-1.0.jar` must be uploaded to a UC volume.
+  - YARN configs removed: --master yarn, --deploy-mode cluster, spark.yarn.queue.
+```
+
+---
+
+## Data Ingestion Alternatives (Sqoop Replacement)
+
+On-prem Hadoop pipelines commonly use Apache Sqoop to move data between RDBMS and HDFS. Sqoop has no
+direct equivalent in Databricks. **Import (RDBMS→lakehouse) and export (lakehouse→RDBMS) are different
+problems and map differently** — do not route both to Lakeflow Connect. See
+`references/lakeflow-connect.md` for the ingestion-style distinctions.
+
+### Sqoop Operator Mapping
+
+| Sqoop operation | Databricks migration | Notes |
+|---|---|---|
+| **RDBMS→HDFS import** | Lakeflow Connect (ingestion pipeline), JDBC ingestion notebook, or federation | Connect only for a supported source into a Delta table it owns; else a JDBC read notebook, or federation for read-only query. |
+| **Incremental import** (`--incremental append`/`lastmodified`, a cursor column) | **query-based** Lakeflow Connect (cursor), **not** CDC | Sqoop's cursor `--incremental` maps to query-based ingestion — it is NOT log-based change capture. |
+| **Log-based change capture** (a true CDC source) | **CDC** Lakeflow Connect where the connector supports it | Only when the source emits a change log (MySQL/PostgreSQL/SQL Server). |
+| **HDFS/Hive→RDBMS export** | JDBC/connector write in a notebook, or a reverse-ETL tool | **NOT Lakeflow Connect** — Connect only ingests *into* the lakehouse. |
+| Custom file-based ingestion | Auto Loader + `cloudFiles` | For files landing in cloud storage — not a managed connector. |
+
+### Conversion Approach
+
+1. **For Sqoop import tasks**: choose the ingestion style before converting.
+   - If the source has a **supported Lakeflow Connect connector** and Connect can own a new destination
+     table, emit a `resources.pipelines` ingestion pipeline (query-based for a cursor `--incremental`;
+     CDC only for a true log-based source). Document the source connection, target table, cursor/primary
+     keys, and networking in MIGRATION_NOTES.md; the UC connection is a manual prerequisite.
+   - Otherwise use a **JDBC read notebook** (`spark.read.format("jdbc")`) into Delta, or **federation**
+     for read-only query. See `references/lakeflow-connect.md`.
+
+2. **For Sqoop export tasks** (lakehouse→RDBMS): convert to a `notebook_task` using JDBC write:
+
+   ```python
+   # Databricks notebook source
+   df = spark.read.table("catalog.schema.aggregated_data")
+   df.write \
+       .format("jdbc") \
+       .option("url", dbutils.secrets.get("scope", "jdbc_url")) \
+       .option("dbtable", "target_schema.target_table") \
+       .option("user", dbutils.secrets.get("scope", "jdbc_user")) \
+       .option("password", dbutils.secrets.get("scope", "jdbc_password")) \
+       .mode("overwrite") \
+       .save()
+   ```
+
+---
+
+## Bulk Conversion Guidance (Hundreds of Tasks)
+
+For DAGs with hundreds of Spark tasks on Hadoop, follow these additional practices:
+
+### 1. Group by pattern
+
+Before converting task-by-task, categorize all tasks:
+
+| Pattern | Expected Count | Conversion |
+|---|---|---|
+| `SparkSubmitOperator` with `.py` | N tasks | Bulk -> `spark_python_task` |
+| `SparkSubmitOperator` with `.jar` | N tasks | Bulk -> `spark_jar_task` |
+| `BashOperator` wrapping `spark-submit` | N tasks | Parse and convert (see above) |
+| `HiveOperator` / SQL tasks | N tasks | Bulk -> `sql_task` |
+| Sensors (HDFS, External) | N tasks | Convert to triggers or remove |
+| Other | N tasks | Case-by-case |
+
+Present this summary to the user before proceeding with individual task conversion.
+
+### 2. Shared cluster strategy
+
+With hundreds of tasks, avoid creating per-task clusters. Define a small set of shared `job_clusters`:
+
+```yaml
+job_clusters:
+  - job_cluster_key: small-cluster     # For lightweight tasks
+    new_cluster:
+      spark_version: ${var.spark_version}
+      node_type_id: ${var.node_type_id}
+      num_workers: 2
+
+  - job_cluster_key: medium-cluster    # For standard ETL
+    new_cluster:
+      spark_version: ${var.spark_version}
+      node_type_id: ${var.node_type_id}
+      autoscale:
+        min_workers: 2
+        max_workers: 8
+
+  - job_cluster_key: large-cluster     # For heavy processing
+    new_cluster:
+      spark_version: ${var.spark_version}
+      node_type_id: ${var.node_type_id}
+      autoscale:
+        min_workers: 4
+        max_workers: 20
+```
+
+Assign tasks to clusters based on their original YARN resource requests (executor count, memory).
+
+### 3. Split large DAGs
+
+If a single Airflow DAG has 200+ tasks, consider splitting into multiple DABs jobs connected via `run_job_task`. Group by:
+- Logical pipeline stage (ingest -> transform -> aggregate -> publish)
+- Independent branches that can run as separate jobs
+- Tasks with different SLAs or ownership
+
+### 4. Dependency file upload
+
+Collect all JARs, Python files, and config files referenced by the Spark jobs. Create an inventory:
+
+```
+DEPENDENCY_INVENTORY.md
+- /opt/jars/analytics-1.0.jar        -> /Volumes/main/default/libs/analytics-1.0.jar
+- /opt/spark/jobs/etl_pipeline.py     -> src/etl_pipeline.py (bundled)
+- /opt/spark/jobs/common_utils.py     -> src/common_utils.py (bundled)
+- /etc/spark/conf/hive-site.xml       -> Remove (UC replaces Hive metastore)
+- /opt/jars/hadoop-aws-3.3.4.jar      -> Remove (built into Databricks runtime)
+```
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/lakeflow-connect.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/lakeflow-connect.md
new file mode 100644
index 0000000..eff4681
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/lakeflow-connect.md
@@ -0,0 +1,96 @@
+# Lakeflow Connect (managed ingestion) as a migration target
+
+Reference for converting Airflow **ingestion** tasks to Databricks **Lakeflow Connect** managed
+ingestion pipelines, emitted as DABs `resources.pipelines` entries. Lakeflow Connect is the right
+target for **recurring ingestion/replication from an external source into Delta** — not a generic
+fallback for any operator a Lakeflow Jobs task type doesn't cover. Use the Source-aware classification
+step in `references/operator-mapping.md` first; this file covers what to do once a task is a Connect
+candidate.
+
+## When Lakeflow Connect (vs a regular Jobs task)
+
+A task is a Connect candidate only when **all** hold — otherwise map it to a Jobs task (notebook/SDK,
+federation, Auto Loader) or flag it:
+
+- The operation is **recurring ingestion / replication** (not a one-shot backfill, not a transform).
+- A **connector exists** for the source. The list below is **illustrative, not exhaustive** — the SaaS
+  connector set grows, and foreign-catalog ingestion covers **all Lakehouse Federation sources**.
+  Classify from the **current Databricks docs / connector metadata**, not this list alone.
+- **Connect can create and own the destination streaming table.** Ingestion into a table that already
+  exists is not supported — an existing production target needs a new landing table + a downstream
+  merge/cutover step, or a different strategy.
+- Source **objects / columns / cursor / primary keys / deletion handling** are representable.
+- No **intermediate file that is itself an external contract** (e.g. "land a CSV another team consumes").
+- The required **UC connection + networking** are known.
+- The connector's **release state is acceptable**; for a **Private Preview** connector, the workspace
+  has **confirmed enrollment/entitlement** (not merely user acceptance).
+
+Not ingestion → regular Jobs task. Files from cloud storage → **Auto Loader** (not Connect).
+Unsupported source → notebook/SDK using the driver, and flag.
+
+## Ingestion styles
+
+1. **CDC / log-based** — database connectors reading the change log: **MySQL, PostgreSQL, SQL Server**.
+2. **Query-based direct** — cursor/incremental over a direct connection (not log CDC): **Oracle,
+   Teradata, SQL Server, MySQL, MariaDB, PostgreSQL**. (SQL Server / MySQL / PostgreSQL support both
+   CDC and query-based; pick per source capability and customer preference.)
+3. **Query-based from a UC foreign catalog** — ingest from a **Lakehouse Federation** source through a
+   foreign catalog, no dedicated connector: **Snowflake, Redshift, Synapse, BigQuery**. This is how
+   **recurring Snowflake→Delta** is done — there is no dedicated Snowflake managed connector.
+
+## Connectors (verify current release state — status changes)
+
+Name connectors by capability; **do not hardcode GA/Preview status or dates** — always tell the user to
+verify the connector's current release state in the Databricks docs before relying on it.
+
+- **SaaS**: Salesforce, Workday, ServiceNow, Google Analytics, HubSpot.
+- **Files**: Google Drive, SharePoint.
+- **Streaming**: Kafka (Lakeflow Connect managed Kafka connector), RabbitMQ — **continuous-only** (see
+  Orchestration).
+- **Databases**: per the ingestion styles above.
+
+## DABs generation contract
+
+Pick the architecture by source and emit the matching resources (schema/examples in
+`references/dab-schema-reference.md`):
+
+- **Combined ingestion** (SaaS, files, query-based DB, CDC via `connection_name`): ONE
+  `resources/_ingestion.pipeline.yml` with `ingestion_definition` (+ `connector_type` when the
+  source supports both query-based and CDC). No gateway. The bundle schema's `ingestion_definition`
+  description historically warned it "cannot be used with the `libraries`/`schema`/`target`/`catalog`
+  settings," but current query-based-ingestion DAB examples do pair it with `catalog`/`target` — follow
+  the field combination in the current Databricks docs / `databricks bundle schema` for your CLI version
+  rather than treating it as a blanket ban.
+- **Gateway CDC** (log-based DB via a gateway — **Private Preview / `doNotSuggest`**): a **separate**
+  gateway pipeline (`gateway_definition`) + an ingestion pipeline joined by `ingestion_gateway_id`.
+  Only with connector-specific verification + confirmed Private-Preview enrollment; never the default.
+- **Foreign-catalog ingestion** (Snowflake/BigQuery/Redshift/Synapse): `ingest_from_uc_foreign_catalog:
+  true` + `source_catalog/schema/table`. The foreign catalog is a `resources.catalogs` entry
+  (`connection_name` **plus** source-specific `options`, e.g. `options.database`) and requires
+  `bundle.engine: direct`. **Reference an existing foreign catalog by default; create only when the
+  bundle should own it.**
+- **UC connection**: a documented **manual prerequisite**, not a bundle resource — reference by name.
+
+## Orchestration (mode-driven, per connector)
+
+- **Triggered** ingestion pipeline → a `pipeline_task` at the original dependency position in the Jobs
+  graph.
+- **Continuous** pipeline (streaming connectors; any connector documented continuous-only) → run it
+  standalone and have the downstream job depend on a **job-level `trigger.table_update`** on the
+  destination table. `trigger.table_update` is job-level, not a task dependency — if continuous
+  ingestion sits mid-DAG, the graph splits into (upstream job) → (continuous pipeline) → (downstream
+  job); flag that upstream gating semantics change.
+- **Confirm the connector's run mode; do not assume it** from the architecture. If unknown, flag.
+
+## MIGRATION_NOTES.md checklist
+
+Record for every Connect conversion:
+
+- UC **connection name** + the out-of-band creation prerequisite, and the **auth method**.
+- **Foreign-catalog** configuration (catalog name, `options`) when federated; whether the bundle
+  creates or references it.
+- **Source and destination** objects; **cursor / primary keys**; **deletion / SCD** behavior.
+- **Networking** prerequisites (private link / firewall / gateway).
+- The connector's **release state** (and Private-Preview enrollment if applicable).
+- Whether the conversion is **exact** or a **rearchitecture** (e.g. new landing table + merge because
+  the original target already exists, or a continuous-mode job-graph split).
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/operator-mapping.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/operator-mapping.md
new file mode 100644
index 0000000..690e3da
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/operator-mapping.md
@@ -0,0 +1,1805 @@
+# Airflow Operator to DABs Task Type Mapping
+
+Authoritative reference for converting Apache Airflow operators to Databricks Asset Bundles (DABs) job task types. All task types confirmed supported in DABs YAML as of Jan 2026.
+
+---
+
+## Source-aware classification (do this before the Tier tables)
+
+Operator **class** alone does not determine the DABs mapping — the **connection** does. A
+connection-agnostic `SQLExecuteQueryOperator` against a Databricks SQL connection is a `sql_task`; the
+same operator against a remote Postgres connection is federation, a connector notebook, or Lakeflow
+Connect. Provider-specific operators like `PostgresOperator` bind to their own database hook, so the
+operator fixes the remote engine. Resolve each task in this order before applying a Tier mapping:
+
+`operator → connection type → operation intent → data direction → destination contract → strategy`
+
+**Routing table:**
+
+| Situation | Strategy |
+|---|---|
+| Databricks connection + Databricks SQL | `sql_task` (the Tier-1 default) |
+| Remote DB, **read-only SELECT**, source is a **federatable** engine | Lakehouse Federation: `sql_task` over a foreign catalog (or a connector notebook) |
+| Remote **DML/DDL/COPY/CALL** | Keep remote via a connector/API notebook, or migrate the target to Delta |
+| **Recurring** source→Delta ingestion/replication, **eligible** source | **Lakeflow Connect** (see `references/lakeflow-connect.md`) — a decision point, not an auto-swap |
+| Files in cloud storage | **Auto Loader** (existing file path — NOT Lakeflow Connect) |
+| Unsupported source | notebook/wheel using the driver/SDK — **flag** |
+
+**Federation source list (verified):** MySQL, PostgreSQL, SQL Server, Oracle, Teradata, Redshift,
+Snowflake, BigQuery, Synapse, Salesforce Data 360, Databricks. **Athena, Trino, Presto are NOT
+federatable** — route those through JDBC/SDK/connector notebooks.
+
+**Connection resolution is fail-closed.** A DAG usually contains only an arbitrary `conn_id` string,
+not the connection *type*. Automatic routing is allowed **only** from one of: (a) **operator/provider
+certainty** — the operator class fixes the engine (`PostgresOperator`→postgres,
+`SnowflakeSqlApiOperator`→snowflake); (b) the **actual sanitized Airflow `conn_type`** when the
+connection definition is available; or (c) an **explicit user-provided `conn_id → {type, target}`
+mapping**. A `conn_id` name or host string is a **hint only** — surface it as a suggestion in
+`MIGRATION_NOTES.md`, but do not let it drive automatic routing. Anything unresolved is **manual
+review**. **Never export or inline credentials** — auth becomes a UC connection (federation/Connect)
+or `dbutils.secrets` (connector notebook), created out-of-band.
+
+**Lakeflow Connect eligibility** (all must hold, else flag): recurring ingestion/replication; a
+connector exists for the source; **Connect can create and own the destination streaming table** (it
+fails if the destination already exists — an existing target needs a new landing table + downstream
+merge/cutover); source objects/columns/cursor/keys/deletion-handling representable; no intermediate
+file that is itself an external contract; required UC connection + networking known; the connector's
+release state is acceptable — and for a **Private Preview** connector, **confirmed workspace
+enrollment/entitlement**, not just user acceptance. Full rules in `references/lakeflow-connect.md`.
+
+---
+
+## Deferrable operators and sensors (any Airflow version)
+
+Applies across all tiers and to **any Airflow version** — deferrability has existed since Airflow 2.2
+(`deferrable=True`, `*DeferrableOperator` variants, `mode="reschedule"` sensors, the triggerer). It is a
+worker-efficiency mechanism (release a worker slot while waiting) and does **not** change what the task
+does, so **ignore the deferrability and map the underlying operation normally**:
+
+- Drop `deferrable=True` / the `*DeferrableOperator` suffix, triggerer configuration, and `poke_interval`.
+- Keep task **timeout** and **retry** settings where they apply.
+- Generate **no polling** — Lakeflow owns waiting/queueing/triggers natively (sensors → job triggers).
+- Preserve **wait-for-completion** behavior **only** when Databricks submits to an external system via a
+  notebook/wheel and the original operator waited; `wait_for_completion=False` → submit and return.
+- The `[operators] default_deferrable` config only affects operators that support switching modes — it
+  does not change the mapping.
+
+Reference: https://airflow.apache.org/docs/apache-airflow/stable/authoring-and-scheduling/deferring.html
+(Airflow 3's *native async* `@task` and *resumable* external jobs are a separate, v3-only concern — see
+`references/airflow3-migration.md`.)
+
+---
+
+## Tier 1: Direct 1:1 Mappings
+
+These operators have clear, deterministic equivalents in DABs.
+
+---
+
+### PythonOperator / @task (TaskFlow API)
+
+**DABs task type:** `notebook_task`
+
+Extract the `python_callable` function body into a standalone `.py` notebook file in `src/`. Map `op_kwargs` to `base_parameters`, retrieved via `dbutils.widgets.get()` in the notebook.
+
+**Airflow:**
+
+```python
+def extract_data(source_table, target_path):
+    df = spark.read.table(source_table)
+    df.write.format("delta").save(target_path)
+
+extract_task = PythonOperator(
+    task_id="extract_data",
+    python_callable=extract_data,
+    op_kwargs={"source_table": "raw.events", "target_path": "/mnt/silver/events"},
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: extract_data
+  notebook_task:
+    notebook_path: ../src/extract_data.py
+    base_parameters:
+      source_table: "raw.events"
+      target_path: "/mnt/silver/events"
+```
+
+**Generated `src/extract_data.py`:**
+
+```python
+# Databricks notebook source
+dbutils.widgets.text("source_table", "")
+dbutils.widgets.text("target_path", "")
+
+source_table = dbutils.widgets.get("source_table")
+target_path = dbutils.widgets.get("target_path")
+
+df = spark.read.table(source_table)
+df.write.format("delta").save(target_path)
+```
+
+#### TaskFlow dependency extraction
+
+TaskFlow (`@task`) DAGs express dependencies through **function-call wiring (XComArg)**, not only
+`>>`/`<<`. Extract the graph from the calls as well as the operators:
+
+- **Call wiring implies `depends_on`.** `b(a())` (or `x = a(); b(x)`) means `b` depends on `a`.
+  Each passed return value is an XComArg edge — add a `depends_on` entry for it. A task called with
+  no upstream XComArgs and no explicit `>>` is a root task.
+- **`.override(task_id="...")`** renames the task; use the overridden id as the task key. The same
+  decorated function called twice (with or without `.override`) is **two** tasks — key them by the
+  resolved `task_id`, not the function name.
+- **Return value → task value.** A `@task` return becomes `dbutils.jobs.taskValues.set(key="return_value", value=...)`
+  in the generated notebook; the consumer reads it via `{{tasks..values.return_value}}` (as a
+  parameter) or `dbutils.jobs.taskValues.get(taskKey="", key="return_value")` (in-notebook).
+  Values must be JSON-serializable and ≤48 KiB (see `references/dab-schema-reference.md`).
+- **`multiple_outputs=True`** (or a dict-returning `@task` with a dict return annotation) splits the
+  returned dict into one task value **per key**, each separately referenceable
+  (`{{tasks..values.}}`). Set each key with its own `taskValues.set` call.
+- **Mixed classic + TaskFlow.** A classic operator instance can be passed into or wired around
+  `@task` calls; resolve both the `>>`/`<<` edges and the call-wiring edges into one dependency
+  graph before emitting `depends_on`.
+
+**Worked return-value example** — `load(transform(extract()))`:
+
+```python
+@task
+def extract() -> dict:                       # multiple_outputs inferred from dict return
+    return {"rows": 1000, "path": "/mnt/bronze/events"}
+
+@task
+def transform(rows: int, path: str) -> str:
+    return f"{path}_silver"
+
+@task
+def load(silver_path: str) -> None:
+    print(f"publishing {silver_path}")
+
+extracted = extract()
+load(transform(rows=extracted["rows"], path=extracted["path"]))
+```
+
+```yaml
+- task_key: extract
+  notebook_task:
+    notebook_path: ../src/extract.py
+- task_key: transform
+  depends_on:
+    - task_key: extract
+  notebook_task:
+    notebook_path: ../src/transform.py
+    base_parameters:
+      rows: "{{tasks.extract.values.rows}}"
+      path: "{{tasks.extract.values.path}}"
+- task_key: load
+  depends_on:
+    - task_key: transform
+  notebook_task:
+    notebook_path: ../src/load.py
+    base_parameters:
+      silver_path: "{{tasks.transform.values.return_value}}"
+```
+
+Generated `src/extract.py` sets one task value per output key:
+
+```python
+# Databricks notebook source
+dbutils.jobs.taskValues.set(key="rows", value=1000)
+dbutils.jobs.taskValues.set(key="path", value="/mnt/bronze/events")
+```
+
+`src/transform.py` reads its inputs from widgets and sets its single return value:
+
+```python
+# Databricks notebook source
+dbutils.widgets.text("rows", "")
+dbutils.widgets.text("path", "")
+rows = int(dbutils.widgets.get("rows"))
+path = dbutils.widgets.get("path")
+
+silver_path = f"{path}_silver"
+dbutils.jobs.taskValues.set(key="return_value", value=silver_path)
+```
+
+#### TaskFlow decorator variants
+
+The direct mapping above covers **core `@task` dataflow**. Other `@task.*` / lifecycle decorators
+need their own handling — recognize each explicitly so a variant is never silently emitted as a
+plain `notebook_task`:
+
+| Decorator | Disposition |
+|---|---|
+| `@task` (core) | `notebook_task`; dataflow via `dbutils.jobs.taskValues` (above). |
+| `@task.bash` | `notebook_task` wrapping the returned command (follow BashOperator rules — parse for `spark-submit`). |
+| `@task.branch` | Same as `BranchPythonOperator` — `condition_task` (simple comparison) or notebook-sets-value + `condition_task` (complex). |
+| `@task.short_circuit` | `condition_task` gating downstream (skip on false). **Flag** when `ignore_downstream_trigger_rules` is non-default or the skip fan-out is complex — Lakeflow's skip propagation differs from Airflow's. |
+| `@task.virtualenv` / `@task.external_python` | `notebook_task`; **flag** the environment/deps — recreate via a serverless environment or `%pip install`, record in `MIGRATION_NOTES.md`. |
+| `@task.sensor` | Tier-3 job-level trigger **only** when it is a *root* sensor whose `PokeReturnValue` output is unused; otherwise **flag** or keep the polling logic in a notebook. |
+| `@task.run_if` / `@task.skip_if` | The predicate is arbitrary runtime context, but Lakeflow `run_if` evaluates only **upstream task states**. Map a status-equivalent predicate to `run_if`; map other predicates through a `condition_task`; **flag** anything not reducible to either. |
+| `@setup` / `@teardown` | **Flag** — no native Lakeflow setup/teardown lifecycle. Emit as ordinary first/last tasks and note the semantic loss. |
+| `@task.kubernetes` / `@task.docker` / other provider `@task.*` | **Flag** — route through the matching Tier-2/Tier-4 operator rule (`KubernetesPodOperator`, `DockerOperator`, …). |
+
+---
+
+### BashOperator
+
+**DABs task type:** `notebook_task` (general) or `spark_python_task` / `spark_jar_task` (if wrapping `spark-submit`)
+
+For general bash commands, wrap in a notebook using `subprocess.run()`. **If the `bash_command` contains a `spark-submit` invocation**, parse it and convert to a proper `spark_python_task` or `spark_jar_task` instead. See `references/hadoop-migration-guide.md` for spark-submit detection and YARN config cleanup.
+
+**Airflow:**
+
+```python
+cleanup = BashOperator(
+    task_id="cleanup_staging",
+    bash_command="rm -rf /tmp/staging/* && echo 'Staging cleaned'",
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: cleanup_staging
+  notebook_task:
+    notebook_path: ../src/cleanup_staging.py
+```
+
+**Generated `src/cleanup_staging.py`:**
+
+```python
+# Databricks notebook source
+import subprocess
+result = subprocess.run(
+    ["bash", "-c", "rm -rf /tmp/staging/* && echo 'Staging cleaned'"],
+    capture_output=True, text=True
+)
+print(result.stdout)
+if result.returncode != 0:
+    raise RuntimeError(f"Command failed: {result.stderr}")
+```
+
+---
+
+### SparkSubmitOperator
+
+**DABs task type:** `spark_python_task` (for .py files) or `spark_jar_task` (for .jar files)
+
+Map `application` to `python_file` or JAR `main_class_name`. Map Spark `conf` to cluster-level `spark_conf`.
+
+**Airflow (Python):**
+
+```python
+spark_etl = SparkSubmitOperator(
+    task_id="spark_etl",
+    application="/opt/spark/jobs/etl_pipeline.py",
+    conf={"spark.executor.memory": "4g", "spark.executor.cores": "2"},
+    application_args=["--date", "{{ ds }}"],
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: spark_etl
+  new_cluster:
+    spark_version: "15.4.x-scala2.12"
+    node_type_id: ${var.node_type_id}
+    num_workers: 2
+    spark_conf:
+      spark.executor.memory: "4g"
+      spark.executor.cores: "2"
+  spark_python_task:
+    python_file: ../src/etl_pipeline.py
+    parameters:
+      - "--date"
+      - "{{job.parameters.run_date}}"
+```
+
+**Airflow (JAR):**
+
+```python
+spark_jar = SparkSubmitOperator(
+    task_id="spark_jar_job",
+    application="/opt/spark/jars/analytics.jar",
+    java_class="com.example.Analytics",
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: spark_jar_job
+  spark_jar_task:
+    main_class_name: com.example.Analytics
+  libraries:
+    - jar: /Volumes/main/default/jars/analytics.jar
+```
+
+---
+
+### DatabricksSubmitRunOperator / DatabricksSubmitRunDeferrableOperator
+
+**DABs task type:** native DABs task (extract `json` payload directly)
+
+The operator's `json` parameter already describes a Databricks task. Translate the JSON structure directly into DABs YAML. The deferrable variant (`DatabricksSubmitRunDeferrableOperator`) maps identically -- the deferrable behavior is an Airflow scheduler optimization that has no DABs equivalent.
+
+**Airflow:**
+
+```python
+submit_run = DatabricksSubmitRunOperator(
+    task_id="run_notebook",
+    json={
+        "new_cluster": {
+            "spark_version": "15.4.x-scala2.12",
+            "node_type_id": "i3.xlarge",
+            "num_workers": 2,
+        },
+        "notebook_task": {
+            "notebook_path": "/Workspace/Users/user@example.com/etl",
+            "base_parameters": {"env": "prod"},
+        },
+    },
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: run_notebook
+  new_cluster:
+    spark_version: "15.4.x-scala2.12"
+    node_type_id: i3.xlarge
+    num_workers: 2
+  notebook_task:
+    notebook_path: ../src/etl.py
+    base_parameters:
+      env: "prod"
+```
+
+---
+
+### DatabricksRunNowOperator / DatabricksRunNowDeferrableOperator
+
+**DABs task type:** `run_job_task`
+
+Map `job_id` directly. Map `notebook_params`, `python_params`, or `jar_params` to `job_parameters`. The deferrable variant maps identically.
+
+**Airflow:**
+
+```python
+trigger_job = DatabricksRunNowOperator(
+    task_id="trigger_downstream",
+    job_id=12345,
+    notebook_params={"env": "prod", "date": "{{ ds }}"},
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: trigger_downstream
+  run_job_task:
+    job_id: 12345
+    job_parameters:
+      env: "prod"
+      date: "{{job.parameters.run_date}}"
+```
+
+---
+
+### DatabricksNotebookOperator
+
+**DABs task type:** `notebook_task`
+
+Direct 1:1 mapping. The operator already runs a Databricks notebook with parameters -- translate to `notebook_task` with `base_parameters`. Map `source` to the notebook path in the bundle (copy notebook into `src/` if the path is workspace-local).
+
+Compute mapping:
+- If the Airflow task uses `new_cluster`, emit `new_cluster` on the DABs task.
+- If it uses `job_cluster_key` or `existing_cluster_id`, preserve that field in DABs.
+
+**Airflow:**
+
+```python
+from airflow.providers.databricks.operators.databricks import DatabricksNotebookOperator
+
+notebook_run = DatabricksNotebookOperator(
+    task_id="run_etl_notebook",
+    databricks_conn_id="databricks_default",
+    notebook_path="/Workspace/Users/user@example.com/etl_pipeline",
+    notebook_params={"env": "prod", "date": "{{ ds }}"},
+    source="WORKSPACE",
+    new_cluster={
+        "spark_version": "15.4.x-scala2.12",
+        "node_type_id": "i3.xlarge",
+        "num_workers": 2,
+    },
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: run_etl_notebook
+  new_cluster:
+    spark_version: ${var.spark_version}
+    node_type_id: ${var.node_type_id}
+    num_workers: 2
+  notebook_task:
+    notebook_path: ../src/etl_pipeline.py
+    source: WORKSPACE
+    base_parameters:
+      env: "prod"
+      date: "{{job.parameters.run_date}}"
+```
+
+---
+
+### DatabricksSqlOperator / DatabricksSQLStatementsOperator
+
+**DABs task type:** `sql_task` (warehouse-backed) or `notebook_task`/`spark_python_task` (cluster-backed SQL)
+
+`DatabricksSQLStatementsOperator` uses the Statement Execution API and requires a warehouse context, so it maps directly to `sql_task`.
+
+`DatabricksSqlOperator` supports either a SQL warehouse or a Databricks cluster (`http_path`). Map by backend:
+- Warehouse-backed (`sql_endpoint_name` or warehouse `http_path`) -> `sql_task`
+- Cluster-backed (`http_path` for interactive cluster) -> `notebook_task`/`spark_python_task` that executes `spark.sql(...)` on cluster compute
+
+Extract inline SQL to a `.sql` file when using `sql_task`. If SQL references an existing query ID, use `sql_task.query.query_id`.
+
+**Airflow:**
+
+```python
+from airflow.providers.databricks.operators.databricks_sql import DatabricksSqlOperator
+
+sql_report = DatabricksSqlOperator(
+    task_id="daily_aggregation",
+    databricks_conn_id="databricks_default",
+    sql="""
+        CREATE OR REPLACE TABLE gold.daily_metrics AS
+        SELECT date, COUNT(*) as events, SUM(revenue) as total
+        FROM silver.transactions
+        WHERE date = '{{ ds }}'
+        GROUP BY date
+    """,
+    http_path="/sql/1.0/warehouses/abc123",
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: daily_aggregation
+  sql_task:
+    warehouse_id: ${var.warehouse_id}
+    file:
+      path: ../src/daily_aggregation.sql
+      source: WORKSPACE
+    parameters:
+      run_date: "{{job.parameters.run_date}}"
+```
+
+**Generated `src/daily_aggregation.sql`:**
+
+```sql
+CREATE OR REPLACE TABLE gold.daily_metrics AS
+SELECT date, COUNT(*) as events, SUM(revenue) as total
+FROM silver.transactions
+WHERE date = :run_date
+GROUP BY date
+```
+
+---
+
+### DatabricksCopyIntoOperator
+
+**DABs task type:** `sql_task` (warehouse-backed) or `notebook_task`/`spark_python_task` (cluster-backed SQL)
+
+The operator runs a `COPY INTO` SQL command to ingest files into a Delta table.
+
+Map by backend:
+- Warehouse-backed (`sql_endpoint_name` or warehouse `http_path`) -> `sql_task` with extracted `.sql`
+- Cluster-backed (`http_path` for interactive cluster) -> cluster compute task (`notebook_task`/`spark_python_task`) that runs `spark.sql("COPY INTO ...")`
+
+**Airflow:**
+
+```python
+from airflow.providers.databricks.operators.databricks_sql import DatabricksCopyIntoOperator
+
+ingest = DatabricksCopyIntoOperator(
+    task_id="ingest_csv_data",
+    databricks_conn_id="databricks_default",
+    table_name="bronze.raw_events",
+    file_location="s3://data-landing/events/",
+    file_format="CSV",
+    format_options={"header": "true", "inferSchema": "true"},
+    force_copy=True,
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: ingest_csv_data
+  sql_task:
+    warehouse_id: ${var.warehouse_id}
+    file:
+      path: ../src/ingest_csv_data.sql
+      source: WORKSPACE
+```
+
+**Generated `src/ingest_csv_data.sql`:**
+
+```sql
+COPY INTO bronze.raw_events
+FROM 's3://data-landing/events/'
+FILEFORMAT = CSV
+FORMAT_OPTIONS ('header' = 'true', 'inferSchema' = 'true')
+COPY_OPTIONS ('force' = 'true')
+```
+
+---
+
+### SQLExecuteQueryOperator
+
+`SQLExecuteQueryOperator` is connection-agnostic. Its **DABs task type is `sql_task` only when the
+resolved connection targets Databricks SQL**; otherwise apply the source-aware classification step
+above.
+
+- **Databricks SQL connection** → `sql_task` (the mapping shown below). If SQL is inline, extract it to a
+  `.sql` file and reference via `sql_task.file.path`; if it references an existing Databricks SQL query,
+  use `sql_task.query.query_id`. Requires a `warehouse_id`.
+- **Remote DB connection, read-only SELECT, federatable engine** → run the SQL as a `sql_task` over a
+  **Lakehouse Federation foreign catalog** (auth via a UC connection), or a connector notebook.
+- **Remote DB connection, DML/DDL** → keep it remote via a connector/API notebook, or migrate the target
+  to Delta and rewrite the SQL for Databricks.
+- **Recurring remote-DB→Delta load** → consider **Lakeflow Connect** (see `references/lakeflow-connect.md`).
+- **Connection unresolved** (only a `conn_id` string, no `conn_type`) → **flag for manual review**; do not
+  assume `sql_task`.
+
+#### PostgresOperator / MySqlOperator
+
+Provider-specific operators bind to their database hooks, so `PostgresOperator` and `MySqlOperator`
+identify known remote PostgreSQL and MySQL engines respectively. Do not reinterpret either one as a
+Databricks SQL task based on its `conn_id` or connection metadata. Route by SQL intent:
+
+- **Read-only SELECT** → Lakehouse Federation over the corresponding foreign catalog, or a connector
+  notebook.
+- **Remote DML/DDL** → a connector/API notebook, or migrate the target to Delta and rewrite the SQL.
+- **Recurring source→Delta ingestion** → Lakeflow Connect when the source and destination contract meet
+  its eligibility rules.
+
+**Airflow:**
+
+```python
+run_report = SQLExecuteQueryOperator(
+    task_id="generate_report",
+    conn_id="databricks_sql",
+    sql="""
+        CREATE OR REPLACE TABLE gold.daily_report AS
+        SELECT date, SUM(revenue) as total_revenue
+        FROM silver.transactions
+        WHERE date = '{{ ds }}'
+        GROUP BY date
+    """,
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: generate_report
+  sql_task:
+    warehouse_id: ${var.warehouse_id}
+    file:
+      path: ../src/generate_report.sql
+      source: WORKSPACE
+    parameters:
+      run_date: "{{job.parameters.run_date}}"
+```
+
+**Generated `src/generate_report.sql`:**
+
+```sql
+CREATE OR REPLACE TABLE gold.daily_report AS
+SELECT date, SUM(revenue) as total_revenue
+FROM silver.transactions
+WHERE date = :run_date
+GROUP BY date
+```
+
+---
+
+### Snowflake operators (snowflake provider)
+
+There is **no dedicated Snowflake managed connector** in Lakeflow Connect — Snowflake maps via
+**Lakehouse Federation** (read) and **query-based foreign-catalog ingestion** (recurring copy). Apply
+the Source-aware classification step; route by *intent*, not operator class.
+
+**Operator state (snowflake provider):** `SnowflakeOperator` was **removed in v6.0** (use
+`SQLExecuteQueryOperator` with a Snowflake connection); `S3ToSnowflakeOperator` was **removed in v5.0**
+(use `CopyFromExternalStageToSnowflakeOperator`). Current: `SnowflakeSqlApiOperator`,
+`Snowflake{Check,ValueCheck,IntervalCheck}Operator`, `CopyFromExternalStageToSnowflakeOperator`. The
+Snowpark TaskFlow decorator's **DAG syntax is `@task.snowpark`** (underlying API
+`airflow.providers.snowflake.decorators.snowpark.snowpark_task`) — **recognize both forms**.
+
+| Intent | Migration |
+|---|---|
+| Read-only Snowflake SQL | Databricks SQL over a Snowflake **foreign catalog** (federation) |
+| Read Snowflake, write Delta | CTAS / INSERT…SELECT via federation |
+| **Recurring** Snowflake→Delta copy | **query-based Lakeflow Connect via foreign catalog** (`ingest_from_uc_foreign_catalog`) — see `references/lakeflow-connect.md` |
+| Snowflake DML / DDL / `COPY` / `CALL` | keep remote via a connector/API notebook, or rewrite for Delta |
+| Snowflake checks (`Snowflake*CheckOperator`) | federation `sql_task` + `assert_true()` (see SQL checks below) |
+| External stage → Snowflake (`CopyFromExternalStageToSnowflakeOperator`) | preserve Snowflake `COPY`, or change the destination to Delta + Auto Loader / `COPY INTO` |
+| Snowpark (`@task.snowpark` / `snowpark_task`) | keep Snowpark remote from a notebook, or manually rewrite to PySpark/SQL (Snowpark ≠ PySpark) |
+
+**Federation toward Snowflake is read-only** — it cannot write Snowflake or run arbitrary Snowflake
+administration. Snowflake credentials become a **UC connection** (federation / foreign-catalog
+ingestion) or `dbutils.secrets` (a connector notebook using the Snowflake Python/Spark connector).
+
+---
+
+### SQL data-quality check operators
+
+**DABs task type:** `sql_task` (over the appropriate connection — Databricks SQL or a federated foreign
+catalog per the classification step)
+
+Common-SQL and provider **check** operators — `SQLColumnCheckOperator`, `SQLTableCheckOperator`,
+`SQLValueCheckOperator`, `SQLThresholdCheckOperator`, `SQLIntervalCheckOperator`, `SQLCheckOperator`,
+`Snowflake{Check,ValueCheck,IntervalCheck}Operator` — assert a condition, so they map to a
+`sql_task` that uses **`assert_true(...)`** so a failed assertion fails the task (and the run).
+
+> **`@task.sql` is NOT a check.** It wraps `SQLExecuteQueryOperator` and can run any SELECT/DML/DDL and
+> return results — route it by connection and SQL intent (the source-aware classification step),
+> preserving or flagging any consumed output. Use `assert_true()` only when the task actually represents
+> an assertion, not for every `@task.sql`.
+
+**`assert_true()` is only the failure mechanism, not the whole conversion.** Faithfully port the
+check's semantics or **flag** it: the comparison **tolerance**, `SQLIntervalCheckOperator`'s
+**interval ratios** and time window, any **partition/`WHERE`** clause, **null handling**, and
+**dynamic thresholds** (values computed from another query). If a check can't be expressed exactly in
+SQL, flag it in `MIGRATION_NOTES.md` rather than approximating.
+
+```sql
+-- SQLValueCheckOperator (row count within tolerance) ->
+SELECT assert_true(
+  abs((SELECT count(*) FROM silver.orders WHERE order_date = :run_date) - :expected) <= :tolerance
+)
+```
+
+`GenericTransfer` is not a check — route it by (source, destination) per the classification step:
+supported source→Delta (Connect / federation + CTAS / connector notebook), Delta→external
+(JDBC/connector write), external→external (preserve via an SDK/connector task).
+
+---
+
+### TriggerDagRunOperator
+
+**DABs task type:** `run_job_task`
+
+Map `trigger_dag_id` to the corresponding DABs job using bundle substitutions. Map `conf` to `job_parameters`.
+
+**Airflow:**
+
+```python
+trigger_downstream = TriggerDagRunOperator(
+    task_id="trigger_reporting_dag",
+    trigger_dag_id="reporting_pipeline",
+    conf={"source": "etl_pipeline"},
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: trigger_reporting_dag
+  run_job_task:
+    job_id: ${resources.jobs.reporting-pipeline-job.id}
+    job_parameters:
+      source: "etl_pipeline"
+```
+
+> NOTE: The target DAG must also be converted to a DABs job for `${resources.jobs...}` substitution to work. Otherwise, use a hardcoded `job_id`.
+
+---
+
+### dbt CLI Operators (DbtOperator / DbtRunOperator / DbtTestOperator / DbtSeedOperator / DbtSnapshotOperator / DbtBuildOperator)
+
+**DABs output:** dbt factory mode (default) or a single `dbt_task` (fallback)
+
+#### dbt conversion decision point
+
+**Default to dbt factory mode for every dbt workload.** It converts the dbt project into a separate, Python-generated Lakeflow job with one task per dbt object (model / seed / snapshot / test), giving per-model observability, retry-from-failed-model, parallel execution, and tests gating downstream models — the reasons customers orchestrated dbt with Airflow (and cosmos) in the first place. A single `dbt_task` runs the whole invocation as one opaque box.
+
+Factory mode changes the bundle toolchain: it adds a PyDABs `python:` block, a `pyproject.toml` + `.venv` (`uv`), a `Makefile`, and a `databricks-dbt-factory` dependency, and it requires the dbt project source so `dbt parse` can produce `manifest.json`. Present the choice and these implications in the Phase 1 summary; proceed with factory mode unless a disqualifier applies.
+
+**Factories from commands.** Enable only the factory types matching the union of dbt commands the original Airflow tasks ran (`FACTORY_TYPES` in the glue template) — a test-only workload must not start running models:
+
+| Detected command | Factories |
+|---|---|
+| `dbt run` | `model` |
+| `dbt seed` | `seed` |
+| `dbt snapshot` | `snapshot` |
+| `dbt test` | `test` |
+| `dbt build` | `model`, `seed`, `snapshot`, `test` |
+| `deps`/`docs` only | not factory-eligible — use the single-`dbt_task` fallback |
+| Multiple commands | union of the above |
+
+databricks-dbt-factory 0.3.1 selects every node by its full dot-joined FQN, emits one task per dbt test (including unit tests), and derives readable task keys (`_`, e.g. `orders_model`/`countries_seed`; bundled tests keyed `_test`) that are guaranteed unique and ≤100 chars — collisions are disambiguated by package/hash inside the factory. The glue post-processes that output: it prunes `depends_on` references to omitted node types (the factory emits dangling dependencies when a node type has no factory), and applies deploy-time fail-closed guards as defense-in-depth: generated selectors that do not resolve to exactly their own node — the check imports dbt's own `is_selected_node` matcher at deploy time, so it covers everything dbt's semantics cover (prefix matching, leaf shortcuts, versioned models, wildcard slurp, package-stripped retry) and also rejects a selector resolving to a single wrong node, or one whose FQN — package, any directory component, or name — contains anything outside `[A-Za-z0-9_.-]` (an allowlist checked over the full FQN, not just the leaf; hyphens are allowed since dbt path components use them; other characters would be reinterpreted by dbt's CLI selector grammar or corrupt the runner's `shlex.split`); plus a final check that the emitted task keys are unique (the factory already guarantees this, so this only trips on a factory regression). The runner also rejects any dbt command carrying its own `--vars` (both `--vars ` and `--vars=`) — vars must use the canonical `dbt_vars.json`/`dbt_vars` channel.
+
+**Task count and the 1,000-task per-job limit.** A single Databricks job holds at most 1,000 tasks; one-task-per-dbt-node can exceed that on large, test-heavy projects. After `make manifest`, run `make task-count` to compare unbundled vs bundled counts. When the unbundled count is over the warn threshold (900), set `BUNDLE_TESTS = True` in the glue: this collapses each resource's single-model tests into one bundled test task (`dbt test --select  --indirect-selection cautious`, keyed `_test`) — the single biggest reduction — while cross-model and zero-dep tests still get their own tasks. The tradeoff is coarser retry granularity: a model's tests rerun together, not per individual test. The bundled task targets the resource with `--indirect-selection cautious` so it still sweeps in that resource's tests; selector-exactness skips test nodes in bundled mode since individual tests are not selected on their own. If the count exceeds 1,000 even bundled, do not auto-fall-back: record the options in MIGRATION_NOTES (split the project by dbt tag into multiple factory jobs, await a dbt-factory sub-job-splitting API, or a user-chosen single `dbt_task`). The glue fails closed above 1,000 tasks at deploy time so an over-limit job is caught at `bundle validate` rather than by the Jobs API.
+
+**Vars.** Static `vars` (literal dicts) live in ONE committed file: `dbt_vars.json` at the bundle root (required; `{}` when none). `make manifest` feeds it to `dbt parse --vars` and the runner falls back to it at run time whenever the `dbt_vars` job parameter is an empty object — so parse-time and run-time always agree, and no JSON is ever inlined into shell or Python quoting. A runtime override that differs from the file also bypasses the parse-cache injection (the cache was compiled with static vars — hooks, materializations, and grants would silently keep static values); dbt re-parses in-task instead, at some startup cost. A non-empty runtime `dbt_vars` REPLACES the whole dict (dbt does not merge repeated `--vars`), so overriding callers must pass the complete set. Never smuggle vars through `EXTRA_DBT_COMMAND_OPTIONS` (two `--vars` flags: dbt silently uses the last one). Runtime overrides are safe only when they do not change the dbt graph (enabled nodes, dependencies, schemas, aliases), because the task graph was compiled at deploy time. Disqualifiers: a var that changes the graph, or dbt operator tasks passing conflicting vars dicts (no single canonical value exists) — fall back to `dbt_task`.
+
+Fall back to a single `dbt_task` when:
+
+| Disqualifier | Why |
+|---|---|
+| dbt project **source** unavailable to the conversion | Runtime dbt needs the full project files synced (models, `dbt_project.yml`, profiles, packages) — a manifest alone is NOT enough. Conversely, source without a manifest is fine: `make manifest` generates one. |
+| Invocation subsets the project (`--select` / `--exclude` / `--models`, or cosmos `RenderConfig(select=...)`) and the user does not confirm whole-project runs | **Selector caveat:** factory mode explodes the *entire* manifest. A selector-scoped Airflow task ran less than that — converting silently would change semantics. Surface it; convert only on explicit confirmation, and record the decision in `MIGRATION_NOTES.md`. |
+| `full_refresh=True` detected and not manually resolved | Never apply `--full-refresh` automatically (it is invalid for `dbt test` and changes materialization behavior). Make it an explicit manual-review decision. |
+| Vars that change the dbt graph, without confirmation that the graph is invariant | See Vars above. |
+| More than one dbt project in the bundle | v1 supports exactly one dbt project per bundle, colocated at the bundle root. Multiple projects require split bundles. |
+| User explicitly requests minimal toolchain change | Their call; note the observability trade-off in `MIGRATION_NOTES.md`. |
+
+**dbt Cloud (`DbtCloudRunJobOperator`) is NOT a `dbt_task` fallback** — `dbt_task` runs dbt Core and cannot trigger a dbt Cloud job. Route it to Tier 4 (notebook calling the dbt Cloud API, or migrate the project to Databricks).
+
+For factory mode, generate the artifacts described in **dbt factory mode — generated artifacts** under the cosmos section in Tier 2 (the mechanics are identical for CLI operators; extract `project_dir`, `profiles_dir`, `target`, `vars`, and selectors from the operator arguments instead of cosmos configs). Multiple dbt operator tasks over the same project (e.g. `dbt_seed >> dbt_run >> dbt_test`) collapse into ONE factory job with ONE `run_job_task` hop — the manifest explosion already covers seeds, models, snapshots, and tests, with ordering derived from the dbt DAG instead of the coarse seed→run→test chain. Note the semantic shift in `MIGRATION_NOTES.md`: tests run after each model and gate downstream nodes, instead of one test phase at the end.
+
+#### Fallback mapping: single `dbt_task`
+
+Map dbt commands to the `commands` list and map `project_dir` to `project_directory`.
+
+- If using Databricks SQL warehouse execution, set `warehouse_id` and omit `profiles_directory`.
+- If using a custom profile-based setup, set `profiles_directory` and omit `warehouse_id`.
+
+**Airflow:**
+
+```python
+dbt_run = DbtRunOperator(
+    task_id="dbt_transform",
+    project_dir="/opt/dbt/my_project",
+    profiles_dir="/opt/dbt/profiles",
+    select="tag:daily",
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: dbt_transform
+  dbt_task:
+    commands:
+      - "dbt deps"
+      - "dbt run --select tag:daily"
+    project_directory: ../dbt/my_project
+    warehouse_id: ${var.warehouse_id}
+  libraries:
+    - pypi:
+        package: "dbt-databricks>=1.0.0,<2.0.0"
+```
+
+Also treat `BashOperator`/`SSHOperator` commands matching `dbt (deps|seed|snapshot|run|test|build)` as dbt workloads subject to this decision point.
+
+---
+
+### HiveOperator / HivePartitionSensor (Hadoop)
+
+**DABs task type:** `sql_task` or `notebook_task`
+
+HiveQL queries map directly to Spark SQL via `sql_task`. Table references need conversion from `database.table` to `catalog.schema.table` (Unity Catalog). See `references/hadoop-migration-guide.md` for Hive-to-UC table mapping.
+
+**Airflow:**
+
+```python
+from airflow.providers.apache.hive.operators.hive import HiveOperator
+
+hive_etl = HiveOperator(
+    task_id="hive_aggregate",
+    hql="""
+        INSERT OVERWRITE TABLE analytics.daily_summary
+        SELECT date, COUNT(*) as total, SUM(amount) as revenue
+        FROM events.transactions
+        WHERE date = '{{ ds }}'
+        GROUP BY date
+    """,
+    hive_cli_conn_id="hive_default",
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: hive_aggregate
+  sql_task:
+    warehouse_id: ${var.warehouse_id}
+    file:
+      path: ../src/hive_aggregate.sql
+      source: WORKSPACE
+    parameters:
+      run_date: "{{job.parameters.run_date}}"
+```
+
+**Generated `src/hive_aggregate.sql`:**
+
+```sql
+-- Migrated from HiveQL. Table references updated to Unity Catalog.
+INSERT INTO catalog.analytics.daily_summary
+SELECT date, COUNT(*) as total, SUM(amount) as revenue
+FROM catalog.events.transactions
+WHERE date = :run_date
+GROUP BY date
+```
+
+> NOTE: `INSERT OVERWRITE TABLE` should be converted to `INSERT INTO` with `CREATE OR REPLACE TABLE` or `MERGE` depending on the use case. Delta tables do not support `INSERT OVERWRITE` in the same way as Hive.
+
+---
+
+### SSHOperator (Hadoop Edge Node)
+
+**DABs task type:** `spark_python_task`, `spark_jar_task`, or `notebook_task`
+
+SSHOperator is commonly used to SSH into a Hadoop edge node and run `spark-submit`. Extract the remote command and convert it to a direct DABs task. The SSH hop is eliminated since Databricks runs Spark natively. See `references/hadoop-migration-guide.md` for spark-submit parsing.
+
+**Airflow:**
+
+```python
+from airflow.providers.ssh.operators.ssh import SSHOperator
+
+ssh_spark = SSHOperator(
+    task_id="run_spark_on_hadoop",
+    ssh_conn_id="hadoop_edge",
+    command="spark-submit --master yarn --class com.example.ETL /opt/jars/etl.jar --date {{ ds }}",
+)
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: run_spark_on_hadoop
+  spark_jar_task:
+    main_class_name: com.example.ETL
+    parameters:
+      - "--date"
+      - "{{job.parameters.run_date}}"
+  libraries:
+    - jar: /Volumes/main/default/libs/etl.jar
+```
+
+---
+
+## Tier 2: Semantic Mappings (Require Interpretation)
+
+These operators require reasoning about intent to determine the best DABs equivalent.
+
+---
+
+### BranchPythonOperator / ShortCircuitOperator
+
+**DABs task type:** `condition_task` + `depends_on` with `outcome`
+
+For simple comparisons, map directly to `condition_task` fields (`left`, `op`, `right`). For complex logic, split into a `notebook_task` that sets a task value, followed by a `condition_task` that reads that value.
+
+**Airflow:**
+
+```python
+def choose_branch(**context):
+    if context["params"]["env"] == "prod":
+        return "run_full_pipeline"
+    return "run_sample_pipeline"
+
+branch = BranchPythonOperator(
+    task_id="check_environment",
+    python_callable=choose_branch,
+)
+```
+
+**DABs YAML (simple case):**
+
+```yaml
+- task_key: check_environment
+  condition_task:
+    left: "{{job.parameters.env}}"
+    op: EQUAL_TO
+    right: "prod"
+
+- task_key: run_full_pipeline
+  depends_on:
+    - task_key: check_environment
+      outcome: "true"
+  notebook_task:
+    notebook_path: ../src/full_pipeline.py
+
+- task_key: run_sample_pipeline
+  depends_on:
+    - task_key: check_environment
+      outcome: "false"
+  notebook_task:
+    notebook_path: ../src/sample_pipeline.py
+```
+
+**DABs YAML (complex logic -- two-step pattern):**
+
+```yaml
+- task_key: evaluate_branch
+  notebook_task:
+    notebook_path: ../src/evaluate_branch.py
+
+- task_key: check_branch_result
+  depends_on:
+    - task_key: evaluate_branch
+  condition_task:
+    left: "{{tasks.evaluate_branch.values.branch_decision}}"
+    op: EQUAL_TO
+    right: "full"
+
+- task_key: run_full_pipeline
+  depends_on:
+    - task_key: check_branch_result
+      outcome: "true"
+  notebook_task:
+    notebook_path: ../src/full_pipeline.py
+```
+
+---
+
+### PythonVirtualenvOperator / ExternalPythonOperator
+
+**DABs task type:** `python_wheel_task` or `notebook_task`
+
+If the function has custom dependencies, package it as a Python wheel with an `entry_point`. For simpler cases, use a `notebook_task` with `%pip install` commands at the top.
+
+**DABs YAML (wheel approach):**
+
+```yaml
+- task_key: custom_transform
+  python_wheel_task:
+    entry_point: run
+    package_name: custom_transform
+  libraries:
+    - whl: ../dist/custom_transform-*.whl
+```
+
+**DABs YAML (notebook approach):**
+
+```yaml
+- task_key: custom_transform
+  notebook_task:
+    notebook_path: ../src/custom_transform.py
+```
+
+**Generated `src/custom_transform.py`:**
+
+```python
+# Databricks notebook source
+# COMMAND ----------
+%pip install pandas==2.1.0 scikit-learn==1.3.0
+# COMMAND ----------
+import pandas as pd
+from sklearn.preprocessing import StandardScaler
+# ... extracted function body ...
+```
+
+---
+
+### SubDagOperator / TaskGroup
+
+**DABs equivalent:** flatten into individual tasks with `depends_on` chains, or extract to a separate job via `run_job_task`.
+
+Flatten the nested tasks into the parent job, preserving dependency order. Prefix task keys with the group name for clarity.
+
+> `SubDagOperator` is **removed in Airflow 3** — this mapping applies to Airflow 2 DAGs (and to `TaskGroup`, which remains). See `references/airflow3-migration.md`. For a `TaskGroup` fanned out over a collection with `@task_group.expand()`, use the **Mapped task group** pattern below, not this flatten.
+
+**Airflow:**
+
+```python
+with TaskGroup("data_quality") as quality_group:
+    check_nulls = PythonOperator(task_id="check_nulls", ...)
+    check_schema = PythonOperator(task_id="check_schema", ...)
+    check_nulls >> check_schema
+```
+
+**DABs YAML:**
+
+```yaml
+- task_key: data_quality__check_nulls
+  notebook_task:
+    notebook_path: ../src/check_nulls.py
+
+- task_key: data_quality__check_schema
+  depends_on:
+    - task_key: data_quality__check_nulls
+  notebook_task:
+    notebook_path: ../src/check_schema.py
+```
+
+---
+
+### Dynamic task mapping (`.expand()` / `.expand_kwargs()`)
+
+**DABs task type:** `for_each_task`
+
+Airflow dynamic task mapping fans one task out over a collection resolved at runtime. This maps to
+a `for_each_task`, whose nested task runs once per element with `{{input}}` (the element) or
+`{{input.}}` (a field) available in its parameters. See `references/dab-schema-reference.md`
+for the `for_each_task` schema, the `{{input}}` reference, and the `inputs` size limits.
+
+**Airflow:**
+
+```python
+@task
+def build_targets() -> list[str]:
+    return ["orders", "customers", "returns"]
+
+@task
+def checksum(table: str, catalog: str) -> None:
+    print(f"checksum {catalog}.{table}")
+
+checksum.partial(catalog="main").expand(table=build_targets())
+```
+
+**DABs YAML** (`.partial` kwargs become constant `base_parameters`; the `.expand` arg is `{{input}}`):
+
+```yaml
+- task_key: build_targets
+  notebook_task:
+    notebook_path: ../src/build_targets.py     # sets a JSON-array task value: values.tables
+- task_key: checksum
+  depends_on:
+    - task_key: build_targets
+  for_each_task:
+    inputs: "{{tasks.build_targets.values.tables}}"
+    concurrency: 3                               # default is 1 (sequential) — set to fan out
+    task:
+      task_key: checksum_iteration
+      notebook_task:
+        notebook_path: ../src/checksum.py
+        base_parameters:
+          table: "{{input}}"                     # the .expand arg
+          catalog: "main"                        # the .partial kwarg (constant)
+```
+
+**Support matrix** — how each mapping shape converts (or why it's flagged):
+
+| Airflow pattern | DABs mapping / disposition |
+|---|---|
+| `.expand(x=)` | `for_each_task`, `inputs` = JSON-array **literal** (≤5,000 chars); `{{input}}` in the nested task. |
+| `.expand(x=)` | Upstream task writes a JSON array **task value**; `inputs` = `{{tasks..values.}}` (≤48 KiB). |
+| `.partial(const=…).expand(x=…)` | `partial` kwargs → constant `base_parameters`; `expand` arg → `{{input}}`. |
+| `.expand_kwargs()` | Object elements; reference fields via `{{input.}}`. |
+| Multi-arg `.expand(a=…, b=…)` (Cartesian product) | **Flag.** `for_each_task` takes one `inputs` array — precompute the product into a single array of objects upstream, or manual review. |
+| Chained mapping (a mapped task's output feeds another mapped task) | **Flag.** A mapped task's per-iteration **outputs cannot be collected/consumed** by another mapped task; factor into a child job or manual review. |
+| Mapped-output **reduction** (a non-mapped task consuming all mapped results) | **Flag.** Downstream cannot read for-each iteration outputs — have each iteration **persist** its result (table/volume) and add a manual aggregation task that reads the persisted results, **not** the original input array. |
+| Collection non-deterministic at parse time | **Flag** for manual review. |
+| `zip` / `map` / filtered inputs | Precompute the final array upstream (task value / job parameter) and reference via `{{input}}`. |
+
+**Choose the `inputs` transport by size** (all must be JSON-serializable): a small **literal**
+array (≤5,000 chars) inline; a larger array through an upstream **task value** (≤48 KiB); or a
+**job-parameter** ref (≤10,000 chars). Oversize or non-JSON collections must be flagged in
+`MIGRATION_NOTES.md`, never silently truncated.
+
+---
+
+### Mapped task group (`@task_group.expand()` / `TaskGroup.partial().expand()`)
+
+**DABs equivalent:** `for_each_task` → `run_job_task` → a **child job** holding the group's subgraph
+
+A mapped task group fans a *multi-step subgraph* out over a collection. A `for_each_task` holds
+exactly one nested task and cannot nest another `for_each_task`, but the nested task **can** be a
+`run_job_task` — so move the group's subgraph into a child job and iterate over it:
+
+- **Parent job:** a `for_each_task` whose nested task is a `run_job_task` targeting the child job,
+  passing the element via `job_parameters` (`{{input}}` or `{{input.}}`).
+- **Child job:** the group's steps as `depends_on`-chained tasks, each reading the element from a
+  **job parameter**.
+
+**Airflow:**
+
+```python
+@task_group
+def region_pipeline(region: str):
+    ingested = ingest(region)
+    validated = validate(ingested)
+    publish(validated)
+
+region_pipeline.expand(region=["us", "eu", "apac"])
+```
+
+**Parent job YAML:**
+
+```yaml
+- task_key: region_pipeline
+  for_each_task:
+    inputs: '["us", "eu", "apac"]'
+    concurrency: 3
+    task:
+      task_key: region_pipeline_iteration
+      run_job_task:
+        job_id: ${resources.jobs.region_pipeline_job.id}
+        job_parameters:
+          region: "{{input}}"
+```
+
+**Child job YAML** (`region_pipeline_job`) — the subgraph, reading `region` from a job parameter:
+
+```yaml
+parameters:
+  - name: region
+    default: ""
+tasks:
+  - task_key: ingest
+    notebook_task:
+      notebook_path: ../src/region_ingest.py
+      base_parameters:
+        region: "{{job.parameters.region}}"
+  - task_key: validate
+    depends_on: [{ task_key: ingest }]
+    notebook_task:
+      notebook_path: ../src/region_validate.py
+      base_parameters:
+        region: "{{job.parameters.region}}"
+  - task_key: publish
+    depends_on: [{ task_key: validate }]
+    notebook_task:
+      notebook_path: ../src/region_publish.py
+      base_parameters:
+        region: "{{job.parameters.region}}"
+```
+
+**Rules this pattern requires** (see `references/dab-schema-reference.md`):
+
+- **Concurrency + queueing.** Set the parent `for_each_task.concurrency`, raise the child job's
+  `max_concurrent_runs` to at least that value, and set `queue: { enabled: true }` on the child
+  job — bundle/API jobs do **not** inherit the UI's default-on queueing, so without it excess
+  iterations are skipped rather than queued. Size `max_concurrent_runs` for overlapping parent
+  runs too (K parent runs × N iterations).
+- **Run Job nesting ≤ 3 levels.** `for_each → run_job → child` uses one level; if the child itself
+  calls Run Job, verify total depth stays within 3.
+- **No cross-iteration outputs.** Downstream consumption of per-element results is manual (persist
+  each child run's result to a table/volume, then aggregate those records separately).
+
+Record the subgraph→child-job decomposition and the observability shift (one child-job run per
+element) in `MIGRATION_NOTES.md`.
+
+---
+
+### Cosmos DbtDag / DbtTaskGroup (astronomer-cosmos)
+
+**DABs output:** dbt factory mode — a separate Python-generated job triggered via `run_job_task`
+
+Cosmos renders one Airflow task per dbt model/seed/test **at runtime** from the dbt manifest, so the individual tasks never appear in the DAG file — a `DbtDag`/`DbtTaskGroup` is statically unparseable task-by-task. Do not attempt to translate its tasks. Instead, swap the generator: `databricks-dbt-factory` reads the same `manifest.json` and renders the same per-model task graph natively as a Lakeflow job.
+
+> Cosmos and databricks-dbt-factory are independent projects with no integration between them — `manifest.json` (a stable dbt-core artifact) is the shared contract. Both are "manifest → orchestrator task graph" generators, which is why migration means swapping the generator rather than translating tasks. Equivalence is at the task-graph level, not feature-for-feature: cosmos-specific settings (per-model retries via `operator_args`, custom profile mappings, `ExecutionMode`) need manual mapping — record them in `MIGRATION_NOTES.md`.
+
+**Airflow:**
+
+```python
+from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig
+from cosmos.profiles import DatabricksTokenProfileMapping
+
+dbt_transform = DbtTaskGroup(
+    group_id="dbt_transform",
+    project_config=ProjectConfig("/opt/airflow/dbt/my_project"),
+    profile_config=ProfileConfig(
+        profile_name="my_project",
+        target_name="dev",
+        profile_mapping=DatabricksTokenProfileMapping(
+            conn_id="databricks_default",
+            profile_args={"catalog": "main", "schema": "analytics"},
+        ),
+    ),
+    render_config=RenderConfig(test_behavior=TestBehavior.AFTER_EACH),
+)
+```
+
+**Metadata to extract:**
+
+| Cosmos config | Use |
+|---|---|
+| `ProjectConfig` path / `manifest_path` | Locate the dbt project; colocate it at the bundle root (or point `MANIFEST_PATH` at it). |
+| `ProfileConfig.profile_name` / `target_name` | `dbt_profiles/profiles.yml` profile name and default target. |
+| `profile_mapping` class + `profile_args` (catalog/schema/http_path) | Warehouse hints for `dbt_profiles/profiles.yml`. Runner injects host/token — no Airflow connection needed. |
+| `RenderConfig.select` / `exclude` | **Selector caveat** — see the dbt conversion decision point in Tier 1. |
+| `RenderConfig.test_behavior` | `AFTER_EACH` (default) matches factory behavior: tests as tasks after each model, gating downstream. |
+| `operator_args` (retries, vars, `full_refresh`) | Manual mapping; record in `MIGRATION_NOTES.md`. |
+
+#### dbt factory mode — generated artifacts
+
+Factory mode adds these artifacts to the bundle (templates in `assets/templates/`):
+
+| Artifact | Template | Purpose |
+|---|---|---|
+| `resources/_dbt_job.py` | `dbt-factory-resources.py.tmpl` | PyDABs hook: reads `target//manifest.json`, enables factories per `FACTORY_TYPES`, prunes dangling deps, runs fail-closed checks, builds one task per dbt node, defines `dbt_vars`/`dbt_target` job parameters, writes `dbt_serverless_env.yaml` idempotently (pinning the venv's exact dbt-databricks and dbt-core — the exactness check imports the local dbt-core, so runtime must match). One module per dbt-bearing DAG. `` = dag_id sanitized to a Python identifier (non `[a-zA-Z0-9_]` chars -> `_`, e.g. `sales.daily` -> `sales_daily`) — raw dotted dag_ids break the module import. |
+| `resources/__init__.py` | — (empty file) | Makes `resources/` importable as a package. |
+| `databricks.yml` additions | `dbt-factory-databricks-additions.yml.tmpl` | `python:` block (one `resources._dbt_job:load_resources` entry per dbt-bearing DAG) + `sync.include`. |
+| `pyproject.toml` | `dbt-pyproject.toml.tmpl` | Pins `databricks-bundles`, `databricks-dbt-factory`, and EXACT `dbt-databricks`/`dbt-core` (dbt version/runtime parity, since uv.lock is git-ignored; transitive deps not locked). Shared across DAGs. |
+| `Makefile` | `dbt-Makefile.tmpl` | `TARGET ?= dev`; `setup` (uv sync) / `manifest` (dbt deps + parse `--target $(TARGET)` `--target-path target/$(TARGET)`) / `validate` / `deploy`. Per-target manifest paths keep dev-parsed artifacts (profile-resolved catalog/schema are baked into the manifest at parse time) out of prod deployments. |
+| `dbt_profiles/profiles.yml` | `dbt-profiles.yml.tmpl` | dev/prod outputs named after bundle targets; host/token injected by the runner notebook. |
+| `src/run_dbt_command.py` | `dbt-run-command.py.tmpl` | Runner notebook owned by the bundle: the 0.3.1 packaged runner extended with `dbt_vars` (appended as `--vars` argv, never string-interpolated; empty/`{}` falls back to `dbt_vars.json`) and per-target parse-cache lookup. Re-diff against the packaged runner when bumping the pin. |
+| `dbt_vars.json` | — (write `{}` or the DAG's static vars) | Single source of static dbt vars, committed at the bundle root; consumed by `make manifest` (parse time) and the runner (run time). REQUIRED — the runner fails if it is missing. |
+| dbt project at bundle root | — (copied) | `dbt_project.yml`, `models/`, `seeds/`, etc. **v1 constraint: exactly one dbt project per bundle, colocated at the bundle root.** Multiple dbt projects → split bundles. |
+| `.gitignore` additions | — | `.venv/`, `logs/`, `dbt_packages/`, `uv.lock`, `target/**`, `dbt_serverless_env.yaml`. `target/*/manifest.json` is a local hook input (not synced); `dbt_serverless_env.yaml` and `target/*/partial_parse.msgpack` are uploaded via `sync.include` despite being git-ignored. Exact `dbt-databricks`/`dbt-core` pins in `pyproject.toml` give dbt version/runtime parity (transitive deps unlocked). |
+
+**Two-job wiring** — the DAG's YAML job triggers the generated job where the cosmos group sat:
+
+```yaml
+- task_key: dbt_transform
+  depends_on:
+    - task_key: 
+  run_job_task:
+    job_id: ${resources.jobs._dbt_job.id}
+    job_parameters:
+      dbt_vars: "{{job.parameters.dbt_vars}}"
+```
+
+The parent job defines a `dbt_vars` parameter (default `"{}"`); the child job's own `dbt_vars` parameter reaches every runner task as a widget and is appended to each dbt command as `--vars` argv.
+
+Downstream tasks set `depends_on: [{task_key: dbt_transform}]`. The reference resolves because YAML and Python-registered resources share one namespace (see `references/dab-schema-reference.md`, Python-Defined Resources).
+
+Two rules for the YAML job in factory mode:
+
+- **Serverless companion tasks:** run the YAML job's own notebook tasks on serverless too — omit all cluster fields (classic `job_clusters` validate but fail at deploy on serverless-only workspaces, and the generated dbt job is serverless-only).
+- **Retries:** map Airflow retries onto the YAML job's own tasks only. Never set retries on the `run_job_task` hop — a retry there re-runs the entire dbt job. Per-model reruns use Lakeflow repair on the child job.
+
+See `examples/dbt-cosmos/` for a complete, validated conversion.
+
+---
+
+### DummyOperator / EmptyOperator
+
+**DABs equivalent:** omit entirely.
+
+Rewire `depends_on` references so that tasks downstream of the DummyOperator depend directly on its upstream tasks instead.
+
+**Airflow:**
+
+```python
+start = DummyOperator(task_id="start")
+end = DummyOperator(task_id="end")
+start >> [task_a, task_b] >> end >> task_c
+```
+
+**DABs YAML:**
+
+```yaml
+# "start" and "end" are omitted. Dependencies are rewired.
+- task_key: task_a
+  notebook_task:
+    notebook_path: ../src/task_a.py
+
+- task_key: task_b
+  notebook_task:
+    notebook_path: ../src/task_b.py
+
+- task_key: task_c
+  depends_on:
+    - task_key: task_a
+    - task_key: task_b
+  notebook_task:
+    notebook_path: ../src/task_c.py
+```
+
+---
+
+### EmailOperator
+
+**DABs equivalent:** `email_notifications` at job or task level (not a standalone task type).
+
+**DABs YAML:**
+
+```yaml
+# Applied at the job level or individual task level
+email_notifications:
+  on_success:
+    - "team@example.com"
+  on_failure:
+    - "oncall@example.com"
+```
+
+---
+
+### DatabricksWorkflowTaskGroup / DatabricksTaskOperator
+
+**DABs equivalent:** flatten into individual DABs job tasks.
+
+`DatabricksWorkflowTaskGroup` defines a multi-task Databricks workflow within Airflow, with each task defined by `DatabricksTaskOperator`. This is the closest Airflow construct to a DABs job. Each `DatabricksTaskOperator` already specifies a Databricks task type (`notebook_task`, `spark_python_task`, etc.), so the migration is nearly 1:1: extract each child task into a DABs task entry, preserve `depends_on` relationships, and map the group's shared cluster to a `job_cluster_key`.
+
+**Airflow:**
+
+```python
+from airflow.providers.databricks.operators.databricks import DatabricksTaskOperator
+from airflow.providers.databricks.operators.databricks_workflow import DatabricksWorkflowTaskGroup
+
+with DatabricksWorkflowTaskGroup(
+    group_id="etl_workflow",
+    databricks_conn_id="databricks_default",
+    job_clusters=[{
+        "job_cluster_key": "etl_cluster",
+        "new_cluster": {
+            "spark_version": "15.4.x-scala2.12",
+            "node_type_id": "i3.xlarge",
+            "num_workers": 4,
+        },
+    }],
+) as wf:
+    extract = DatabricksTaskOperator(
+        task_id="extract",
+        notebook_task={"notebook_path": "/Workspace/etl/extract"},
+        job_cluster_key="etl_cluster",
+    )
+    transform = DatabricksTaskOperator(
+        task_id="transform",
+        notebook_task={"notebook_path": "/Workspace/etl/transform"},
+        job_cluster_key="etl_cluster",
+    )
+    load = DatabricksTaskOperator(
+        task_id="load",
+        notebook_task={"notebook_path": "/Workspace/etl/load"},
+        job_cluster_key="etl_cluster",
+    )
+    extract >> transform >> load
+```
+
+**DABs YAML:**
+
+```yaml
+job_clusters:
+  - job_cluster_key: etl_cluster
+    new_cluster:
+      spark_version: "15.4.x-scala2.12"
+      node_type_id: ${var.node_type_id}
+      num_workers: 4
+
+tasks:
+  - task_key: extract
+    job_cluster_key: etl_cluster
+    notebook_task:
+      notebook_path: ../src/extract.py
+
+  - task_key: transform
+    depends_on:
+      - task_key: extract
+    job_cluster_key: etl_cluster
+    notebook_task:
+      notebook_path: ../src/transform.py
+
+  - task_key: load
+    depends_on:
+      - task_key: transform
+    job_cluster_key: etl_cluster
+    notebook_task:
+      notebook_path: ../src/load.py
+```
+
+---
+
+### DatabricksCreateJobsOperator
+
+**DABs equivalent:** absorbed by `databricks bundle deploy` — omit from job tasks.
+
+This operator programmatically creates Databricks jobs via the Jobs API. In a DABs migration, the job definition itself is the bundle YAML. Remove `DatabricksCreateJobsOperator` tasks from the task graph and instead ensure the job configuration from its `json` parameter is reflected in the generated `resources/_job.yml`. Add a note to `MIGRATION_NOTES.md` explaining that job creation is now handled by `databricks bundle deploy`.
+
+---
+
+### DatabricksReposCreateOperator / DatabricksReposUpdateOperator / DatabricksReposDeleteOperator
+
+**DABs equivalent:** not applicable — infrastructure/repo management, not a job task.
+
+These operators manage Databricks Repos (Git integration). They have no equivalent as DABs job tasks. If a DAG uses these to sync code before running notebooks, note in `MIGRATION_NOTES.md` that DABs handles code deployment natively via `databricks bundle deploy`. Remove these tasks from the job definition.
+
+---
+
+### KubernetesPodOperator / DockerOperator
+
+**DABs task type:** `spark_python_task` or `spark_jar_task` on a single-node cluster with `docker_image` (Databricks Container Services)
+
+These operators run a Docker image as an isolated task. On Databricks, the equivalent is a **single-node job cluster with a custom Docker image** via Databricks Container Services (DCS). The Docker image becomes the cluster environment, and a DABs task runs inside it.
+
+> **Limitations:** Databricks Container Services (DCS) is available on AWS, Azure, and GCP (workspace/region availability can vary). Not supported on serverless compute. Custom containers are not supported on standard/shared access mode; use dedicated/single-user style access mode. The container image must satisfy DCS prerequisites (include `bash`, `iproute2`, `coreutils`, `procps`, `sudo`, and a compatible JDK; Ubuntu-based images are common, and Alpine is also supported when required packages are installed). The image must start as root; clusters fail with `CONTAINER_LAUNCH_FAILURE` when the effective startup user is non-root. Databricks ignores image `ENTRYPOINT`/`CMD` and controls process launch.
+
+#### Decision tree
+
+Inspect the operator's `image`, `cmds`, and `arguments` fields to determine the conversion:
+
+1. **Python-based image** (image contains `python`, or `cmds` starts with `python`/`pip`):
+   → `spark_python_task` pointing to the script. Install deps in the Docker image or via `%pip`.
+
+2. **JVM-based image** (image contains `java`/`jdk`/`scala`, or `cmds` invokes a JAR):
+   → `spark_jar_task` with the JAR bundled in the image or uploaded to a UC volume.
+
+3. **Other runtime** (Go, Rust, Node, shell script, custom binary):
+   → `spark_python_task` with a thin Python wrapper (`entrypoint.py`) that calls `subprocess.run()` to invoke the binary. The binary must be installed in the Docker image.
+
+4. **Image is missing DCS prerequisites** (for example starts as non-root, missing required runtime tools, or incompatible base image setup):
+   → Flag in `MIGRATION_NOTES.md`: "Image must be updated for Databricks Container Services prerequisites (root startup user, required OS utilities, and Java runtime)."
+
+5. **K8s-specific features** (sidecar containers, init containers, persistent volume claims, service accounts):
+   → Flag in `MIGRATION_NOTES.md`: "No Databricks equivalent. Redesign or keep on K8s."
+
+6. **Workspace or policy blocks custom containers** (for example serverless, standard/shared access mode, or cluster policy forbids `docker_image`):
+   → Flag in `MIGRATION_NOTES.md`: "Custom container execution is blocked in the target workspace/policy; redesign task or run externally."
+
+#### Airflow (Python image):
+
+```python
+from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
+
+run_etl = KubernetesPodOperator(
+    task_id="run_etl_container",
+    image="myregistry.azurecr.io/etl-pipeline:2.1.0",
+    cmds=["python"],
+    arguments=["scripts/run_etl.py", "--date", "{{ ds }}"],
+    env_vars={"DB_SECRET": "{{ var.value.db_password }}"},
+    namespace="data-pipelines",
+    get_logs=True,
+)
+```
+
+**DABs YAML:**
+
+```yaml
+job_clusters:
+  - job_cluster_key: etl_container
+    new_cluster:
+      spark_version: "15.4.x-scala2.12"
+      node_type_id: ${var.node_type_id}
+      data_security_mode: SINGLE_USER
+      num_workers: 0
+      spark_conf:
+        spark.databricks.cluster.profile: singleNode
+        spark.master: local[*]
+      custom_tags:
+        ResourceClass: SingleNode
+      docker_image:
+        url: "myregistry.azurecr.io/etl-pipeline:2.1.0"
+        basic_auth:
+          username: "{{secrets/docker-scope/registry-user}}"
+          password: "{{secrets/docker-scope/registry-pass}}"
+
+tasks:
+  - task_key: run_etl_container
+    job_cluster_key: etl_container
+    spark_python_task:
+      python_file: ../src/run_etl.py
+      parameters:
+        - "--date"
+        - "{{job.parameters.run_date}}"
+```
+
+> NOTE: `env_vars` referencing Airflow Variables or secrets must be converted to `dbutils.secrets.get()` calls inside the script, or passed as `base_parameters`. K8s `namespace` and resource requests/limits have no DABs equivalent — cluster sizing is controlled by `node_type_id` and `num_workers`.
+
+#### Airflow (non-Python binary):
+
+```python
+run_go_binary = KubernetesPodOperator(
+    task_id="run_go_processor",
+    image="myregistry.azurecr.io/go-processor:1.0.0",
+    cmds=["./processor"],
+    arguments=["--input", "s3://bucket/data/", "--date", "{{ ds }}"],
+    namespace="data-pipelines",
+)
+```
+
+**DABs YAML:**
+
+```yaml
+job_clusters:
+  - job_cluster_key: go_processor_container
+    new_cluster:
+      spark_version: "15.4.x-scala2.12"
+      node_type_id: ${var.node_type_id}
+      data_security_mode: SINGLE_USER
+      num_workers: 0
+      spark_conf:
+        spark.databricks.cluster.profile: singleNode
+        spark.master: local[*]
+      custom_tags:
+        ResourceClass: SingleNode
+      docker_image:
+        url: "myregistry.azurecr.io/go-processor:1.0.0"
+        basic_auth:
+          username: "{{secrets/docker-scope/registry-user}}"
+          password: "{{secrets/docker-scope/registry-pass}}"
+
+tasks:
+  - task_key: run_go_processor
+    job_cluster_key: go_processor_container
+    spark_python_task:
+      python_file: ../src/run_go_processor.py
+      parameters:
+        - "--input"
+        - "s3://bucket/data/"
+        - "--date"
+        - "{{job.parameters.run_date}}"
+```
+
+**Generated `src/run_go_processor.py`:**
+
+```python
+# Databricks notebook source
+import subprocess
+import sys
+
+args = sys.argv[1:]
+result = subprocess.run(
+    ["./processor"] + args,
+    capture_output=True, text=True
+)
+print(result.stdout)
+if result.returncode != 0:
+    raise RuntimeError(f"Container process failed (exit {result.returncode}): {result.stderr}")
+```
+
+#### DockerOperator
+
+`DockerOperator` follows the same pattern as `KubernetesPodOperator`. Map `image` to `docker_image.url`, `command` to the task entrypoint, and `environment` to secrets or parameters. The Docker-in-Docker execution model is replaced by DCS running the image natively on the cluster node.
+
+---
+
+### Cloud & messaging operator families
+
+These provider families have **no single 1:1 DABs task** — route each **by intent** via the
+Source-aware classification step, not by class name. The recurring strategies:
+
+- **Remote query** (a SELECT against an external warehouse/engine) → Lakehouse Federation `sql_task`
+  over a foreign catalog, **only for a federatable source** (MySQL, PostgreSQL, SQL Server, Oracle,
+  Teradata, Redshift, Snowflake, BigQuery, Synapse, Salesforce Data 360, Databricks). **Athena, Trino,
+  Presto are NOT federatable** → JDBC/SDK/connector notebook.
+- **Recurring source→Delta ingestion** (eligible source) → **Lakeflow Connect** (`references/lakeflow-connect.md`).
+- **Remote compute that Databricks replaces** (EMR/Dataproc Spark, external Spark SQL) → migrate the
+  workload to a `notebook_task` / `sql_task` / pipeline on Databricks.
+- **Remote orchestration retained** (trigger an external job that stays external) → a `notebook_task`
+  driving the cloud SDK (boto3 / google-cloud / azure-sdk), with auth via `dbutils.secrets` or a UC
+  connection; preserve wait-for-completion only if the operator waited.
+- **Kafka consumption → Delta** → the **Lakeflow Connect managed Kafka connector** (continuous) where
+  eligible, else Structured Streaming in a notebook/pipeline.
+- **Messaging side-effects** (publish to SNS/SQS/Kafka, post to Slack/PagerDuty) → a `notebook_task`
+  using the SDK/webhook.
+
+| Family | Representative operators | Typical route |
+|---|---|---|
+| **AWS** | `AthenaOperator`, `EmrAddStepsOperator`/`Emr*`, `GlueJobOperator`, `BatchOperator`, `LambdaInvokeFunctionOperator`, `RedshiftDataOperator`, `SageMaker*`, `SqsPublishOperator`, `SnsPublishOperator` | Athena→JDBC/SDK (not federatable); Redshift→federation; EMR/Glue/Batch/Lambda/SageMaker→SDK notebook (retain remote) or migrate compute; SQS/SNS→SDK notebook |
+| **GCP** | `BigQueryInsertJobOperator`, `DataprocSubmitJobOperator`, `DataflowTemplatedJobStartOperator`, Cloud Run/Functions, `PubSub*` | BigQuery→federation or Connect; Dataproc→migrate to Databricks compute; Dataflow/Cloud Run/Functions→SDK notebook; Pub/Sub→SDK notebook or streaming |
+| **Azure** | `AzureDataFactoryRunPipelineOperator`, `AzureSynapseRunSparkBatchOperator`, Batch, Service Bus, MS Graph | ADF/Synapse→SDK notebook (retain) or migrate; Service Bus→SDK notebook |
+| **HTTP / files** | `HttpOperator`, `SFTPOperator`/`FTPOperator` | HTTP→`notebook_task` w/ `requests` (or the External-Orchestration HTTP operator when GA); SFTP/FTP→notebook w/ `paramiko`/`ftplib`, staging to a UC volume |
+| **Other SQL engines** | `TrinoOperator`/`PrestoOperator` (deprecated), `OracleOperator`/`MsSqlOperator`/`JdbcOperator` (use `SQLExecuteQueryOperator`), `SparkSqlOperator` | Oracle/MSSQL→federation; Trino/Presto→JDBC/SDK (not federatable); SparkSql→`sql_task`/notebook |
+| **Kafka** | `ConsumeFromTopicOperator`/`ProduceToTopicOperator` | Consume→managed Kafka connector (continuous) or Structured Streaming; Produce→SDK notebook |
+
+**Import-path honesty:** state an operator's import path as exact **only** when verified against the
+provider docs; otherwise describe it by pattern (`airflow.providers..operators.`) and
+tell the reader to confirm the module path. Many of these become native one-to-one targets under
+Lakeflow's External Orchestration (Python operator task) as it reaches GA — until then, notebook/SDK
+with a note is the faithful mapping.
+
+---
+
+## Tier 3: Sensor to Trigger Mappings
+
+Airflow sensors that wait for external conditions map to DABs job-level triggers.
+
+---
+
+### DatabricksSqlSensor / DatabricksSQLStatementsSensor
+
+**DABs equivalent:** `depends_on`, `trigger.table_update`, or polling task (intent-dependent)
+
+These sensors are blocking/wait semantics in Airflow, so they do not always become triggers.
+
+Use this decision order:
+1. If waiting on a statement already submitted by an upstream task (`DatabricksSQLStatementsSensor` with `statement_id`), convert to a normal task dependency (`depends_on`) on that upstream task. Do not convert to a trigger.
+2. If the sensor is effectively waiting for external table freshness or table updates, convert to `trigger.table_update`.
+3. If it checks an arbitrary SQL condition (for example, a business rule or feature flag), convert to a polling `notebook_task` (or `spark_python_task`) with timeout handling.
+
+Note: `DatabricksSQLStatementsSensor` can either submit a statement (`statement`) or wait on an existing statement (`statement_id`); preserve that intent during conversion.
+
+**Airflow:**
+
+```python
+from airflow.providers.databricks.sensors.databricks_sql import DatabricksSqlSensor
+
+wait_for_data = DatabricksSqlSensor(
+    task_id="wait_for_daily_data",
+    databricks_conn_id="databricks_default",
+    sql="SELECT COUNT(*) FROM silver.transactions WHERE date = '{{ ds }}'",
+    success=lambda result: result[0][0] > 0,
+    poke_interval=300,
+    timeout=3600,
+)
+```
+
+**DABs YAML (table trigger):**
+
+```yaml
+trigger:
+  table_update:
+    condition: ANY_UPDATED
+    table_names:
+      - "main.silver.transactions"
+    min_time_between_triggers_seconds: 300
+```
+
+> NOTE: If using `statement_id`, prefer `depends_on` over triggers. If the sensor checks an arbitrary SQL condition (not table freshness), convert to a polling compute task that raises on timeout. Add this decision to `MIGRATION_NOTES.md`.
+
+---
+
+### DatabricksPartitionSensor
+
+**DABs equivalent:** `trigger.table_update` or polling `notebook_task`
+
+Waits for a specific partition to appear in a Delta table. If the table is managed via Unity Catalog, convert to `trigger.table_update`. For complex partition-level checks, use a polling `notebook_task`.
+
+**Airflow:**
+
+```python
+from airflow.providers.databricks.sensors.databricks_partition import DatabricksPartitionSensor
+
+wait_for_partition = DatabricksPartitionSensor(
+    task_id="wait_for_partition",
+    databricks_conn_id="databricks_default",
+    table_name="main.silver.events",
+    partitions={"date": "2024-01-15"},
+    poke_interval=300,
+    timeout=3600,
+)
+```
+
+**DABs YAML (table trigger):**
+
+```yaml
+trigger:
+  table_update:
+    condition: ANY_UPDATED
+    table_names:
+      - "main.silver.events"
+    min_time_between_triggers_seconds: 300
+```
+
+> NOTE: DABs `trigger.table_update` fires on any table update, not partition-specific changes. If partition-level precision is required, use a polling `notebook_task` that checks `DESCRIBE DETAIL` or partition metadata. Add to `MIGRATION_NOTES.md`.
+
+---
+
+### HdfsSensor / WebHdfsSensor (Hadoop)
+
+**DABs equivalent:** job-level `trigger.file_arrival`
+
+HDFS file sensors wait for files to land on HDFS. After migrating to cloud storage, these convert to `trigger.file_arrival` pointing at the equivalent cloud path or UC external location. The HDFS path must be mapped to its cloud storage equivalent first. See `references/hadoop-migration-guide.md`.
+
+**Airflow:**
+
+```python
+from airflow.providers.apache.hdfs.sensors.hdfs import HdfsSensor
+
+wait_for_data = HdfsSensor(
+    task_id="wait_for_hdfs_file",
+    filepath="/data/landing/{{ ds }}/*.parquet",
+    hdfs_conn_id="hdfs_default",
+    poke_interval=120,
+    timeout=3600,
+)
+```
+
+**DABs YAML (job-level trigger):**
+
+```yaml
+trigger:
+  file_arrival:
+    url: s3://datalake-bucket/data/landing/
+    min_time_between_triggers_seconds: 120
+    wait_after_last_change_seconds: 60
+```
+
+> NOTE: The HDFS path `/data/landing/` must be mapped to its cloud storage equivalent. Add to MIGRATION_NOTES.md.
+
+---
+
+### S3KeySensor / GCSObjectExistenceSensor
+
+**DABs equivalent:** job-level `trigger.file_arrival`
+
+The sensor's bucket/key path maps to a Unity Catalog external location or volume URL.
+
+**Airflow:**
+
+```python
+wait_for_file = S3KeySensor(
+    task_id="wait_for_upload",
+    bucket_name="data-landing",
+    bucket_key="incoming/{{ ds }}/*.csv",
+    poke_interval=60,
+    timeout=3600,
+)
+```
+
+**DABs YAML (job-level trigger):**
+
+```yaml
+resources:
+  jobs:
+    process_upload_job:
+      name: process-upload-job
+      trigger:
+        file_arrival:
+          url: s3://data-landing/incoming/
+          min_time_between_triggers_seconds: 60
+      tasks:
+        - task_key: process_upload
+          notebook_task:
+            notebook_path: ../src/process_upload.py
+```
+
+---
+
+### ExternalTaskSensor
+
+**DABs equivalent:** `depends_on` (same job), `run_job_task` (cross-job), or `trigger.table_update`
+
+**Same job:** use `depends_on` on the task key.
+**Cross-job, table-driven:** use `trigger.table_update` to fire when a table is updated by the upstream job.
+**Cross-job, explicit:** use `run_job_task` in the upstream job to chain them.
+
+**DABs YAML (table trigger):**
+
+```yaml
+resources:
+  jobs:
+    downstream_job:
+      name: downstream-job
+      trigger:
+        table_update:
+          condition: ANY_UPDATED
+          table_names:
+            - "main.silver.transactions"
+          min_time_between_triggers_seconds: 300
+      tasks:
+        - task_key: process_transactions
+          notebook_task:
+            notebook_path: ../src/process_transactions.py
+```
+
+---
+
+### SqlSensor
+
+**DABs equivalent:** `trigger.table_update` or `notebook_task` with polling logic.
+
+If the SQL checks for table row existence or freshness, convert to a `trigger.table_update`. If the SQL checks an arbitrary condition, wrap it in a `notebook_task`.
+
+---
+
+### FileSensor
+
+**DABs equivalent:** `trigger.file_arrival`
+
+Same pattern as S3KeySensor -- map the file path to a Unity Catalog volume or external location URL.
+
+---
+
+### TimeSensor / TimeDeltaSensor
+
+**DABs equivalent:** absorbed into `schedule.quartz_cron_expression`.
+
+These sensors delay execution until a certain time. In DABs, schedule the job to run at that time directly using a cron expression. If the sensor is mid-pipeline (not at the start), note this in MIGRATION_NOTES.md as requiring manual handling.
+
+---
+
+## Tier 4: Unsupported / Manual Review Required
+
+These operators have no direct DABs equivalent. Flag them in `MIGRATION_NOTES.md`.
+
+| Airflow Operator | Suggested Fallback | Notes |
+|---|---|---|
+| Custom `BaseOperator` subclass | `notebook_task` | Extract operator logic into a notebook. Review `execute()` method. |
+| `HttpSensor` / `SimpleHttpOperator` | `notebook_task` wrapping `requests` | Use a notebook with the `requests` library for HTTP calls. |
+| `LivyOperator` | `spark_python_task` or `notebook_task` | Livy is unnecessary on Databricks; submit Spark code directly. See `hadoop-migration-guide.md`. |
+| `SqoopOperator` (import) | Lakeflow Connect pipeline | Managed RDBMS-to-lakehouse ingestion. A cursor `--incremental` (`append`/`lastmodified`) maps to **query-based** ingestion, NOT CDC; reserve CDC for a true log-based source. Not a DABs task -- create a pipeline resource. See `hadoop-migration-guide.md`. |
+| `SqoopOperator` (export) | `notebook_task` with JDBC write | `df.write.format("jdbc")` in a notebook. See `hadoop-migration-guide.md`. |
+| `PigOperator` | `notebook_task` or `sql_task` | Rewrite Pig Latin scripts as Spark SQL or PySpark. No Pig runtime on Databricks. |
+| `DbtCloudRunJobOperator` / `DbtCloudJobRunSensor` | `notebook_task` calling the dbt Cloud API, or full migration to dbt factory mode / `dbt_task` | dbt Cloud owns orchestration and compute — factory mode does not apply unless the dbt project itself migrates to Databricks. The notebook fallback needs dbt Cloud `account_id`/`job_id` and an API token in secrets. |
+| `BashOperator` (wrapping `spark-submit`) | `spark_python_task` or `spark_jar_task` | Parse the spark-submit command and convert. See `hadoop-migration-guide.md`. |
+| `SSHOperator` (wrapping `spark-submit`) | `spark_python_task` or `spark_jar_task` | Extract the remote command. SSH hop is eliminated. See `hadoop-migration-guide.md`. |
+| Airflow dynamic task mapping (`.expand()`, mapped TaskFlow tasks / task groups) | `for_each_task` | **Not Tier 4** — see the Tier-2 **Dynamic task mapping** and **Mapped task group** sections above for the full support matrix and the `for_each → run_job → child job` pattern. Listed here only as a pointer. |
+| XCom-heavy patterns | `dbutils.jobs.taskValues` | Replace `xcom_push`/`xcom_pull` with `dbutils.jobs.taskValues.set()` and dynamic value references `{{tasks..values.}}`. |
+| Airflow Variables | DABs variables or job parameters | Replace `Variable.get()` with `${var.}` in YAML or `dbutils.widgets.get()` in notebooks. |
+| Airflow Connections | Databricks secrets or UC connections | Replace `BaseHook.get_connection()` with `dbutils.secrets.get()` or Unity Catalog connection references. |
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/schedule-trigger-mapping.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/schedule-trigger-mapping.md
new file mode 100644
index 0000000..620b1fb
--- /dev/null
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/schedule-trigger-mapping.md
@@ -0,0 +1,349 @@
+# Airflow Schedule and Trigger Mapping Reference
+
+Maps Airflow scheduling mechanisms (cron expressions, presets, sensors) to Databricks Asset Bundles schedule and trigger configurations.
+
+---
+
+## Cron Expression Conversion
+
+Airflow uses **5-field Unix cron** (minute, hour, day-of-month, month, day-of-week).
+DABs uses **6-field Quartz cron** (second, minute, hour, day-of-month, month, day-of-week).
+
+### Key Differences
+
+| Feature | Airflow (Unix cron) | DABs (Quartz cron) |
+|---|---|---|
+| Fields | 5: `MIN HOUR DOM MON DOW` | 6-7: `SEC MIN HOUR DOM MON DOW [YEAR]` |
+| Seconds | Not supported | First field, usually `0` |
+| Day-of-week | `0-7` (Sun=0 or 7) or `SUN-SAT` | `1-7` (Sun=1) or `SUN-SAT` |
+| Mutual exclusion | Both DOM and DOW can be `*` | Use `?` for one when the other is set |
+| Timezone | `start_date` timezone or `schedule_interval` | `timezone_id` field (IANA format) |
+
+### Conversion Rule
+
+Prepend `0` for seconds. Replace `*` in day-of-week with `?` when day-of-month is specified (and vice versa).
+
+```
+Airflow:  MIN HOUR DOM MON DOW
+DABs:     0   MIN  HOUR DOM MON DOW
+```
+
+If both DOM and DOW are `*` in Airflow, set DOW to `?` in Quartz:
+`* * * * *` -> `0 * * * * ?`
+
+If DOW is numeric, shift by +1 for Quartz and normalize Sunday:
+- Airflow `0` or `7` (Sunday) -> Quartz `1`
+- Airflow `1-6` (Mon-Sat) -> Quartz `2-7`
+
+Prefer named days (`MON`..`SUN`) to avoid off-by-one conversion bugs.
+
+---
+
+## Airflow Preset to Quartz Cron
+
+| Airflow Preset | Airflow Cron | Quartz Cron | Description |
+|---|---|---|---|
+| `@once` | N/A | *(no schedule, manual trigger)* | Run once. Remove schedule, trigger manually. |
+| `@continuous` | N/A | `continuous.pause_status: UNPAUSED` | Continuous execution mode. |
+| `@hourly` | `0 * * * *` | `0 0 * * * ?` | Top of every hour |
+| `@daily` / `@midnight` | `0 0 * * *` | `0 0 0 * * ?` | Midnight daily |
+| `@weekly` | `0 0 * * 0` | `0 0 0 ? * 1` | Midnight Sunday |
+| `@monthly` | `0 0 1 * *` | `0 0 0 1 * ?` | Midnight first day of month |
+| `@yearly` / `@annually` | `0 0 1 1 *` | `0 0 0 1 1 ?` | Midnight Jan 1 |
+| `None` | N/A | *(no schedule)* | Manual trigger only |
+
+---
+
+## Common Cron Conversions
+
+| Description | Airflow | DABs Quartz |
+|---|---|---|
+| Every 15 minutes | `*/15 * * * *` | `0 */15 * * * ?` |
+| Every 6 hours | `0 */6 * * *` | `0 0 */6 * * ?` |
+| 8 AM daily | `0 8 * * *` | `0 0 8 * * ?` |
+| 8 AM weekdays | `0 8 * * 1-5` | `0 0 8 ? * 2-6` |
+| 6 PM last day of month | `0 18 28-31 * *` | `0 0 18 L * ?` |
+| Every Monday 9 AM | `0 9 * * 1` | `0 0 9 ? * 2` |
+
+> **Day-of-week note:** Airflow `1` = Monday, Quartz `2` = Monday. Also map Airflow `7` (Sunday) to Quartz `1`. Using named days avoids numeric ambiguity.
+
+---
+
+## DABs Schedule YAML
+
+```yaml
+schedule:
+  quartz_cron_expression: "0 0 8 * * ?"
+  timezone_id: "America/New_York"
+  pause_status: UNPAUSED                   # PAUSED or UNPAUSED
+```
+
+### Timezone Mapping
+
+Airflow timezone comes from `default_timezone` in `airflow.cfg` or the DAG's `start_date` timezone. Map to IANA timezone ID for DABs.
+
+| Common Airflow Value | DABs `timezone_id` |
+|---|---|
+| `UTC` | `UTC` |
+| `US/Eastern` | `America/New_York` |
+| `US/Pacific` | `America/Los_Angeles` |
+| `US/Central` | `America/Chicago` |
+| `Europe/London` | `Europe/London` |
+| `Asia/Tokyo` | `Asia/Tokyo` |
+
+---
+
+## Sensor to Trigger Mapping
+
+Airflow sensors that block execution until a condition is met map to DABs job-level triggers or are absorbed into task dependencies.
+
+### File-Based Sensors -> `trigger.file_arrival`
+
+| Airflow Sensor | Trigger Config |
+|---|---|
+| `S3KeySensor` | `trigger.file_arrival` with `url: s3://bucket/prefix/` |
+| `GCSObjectExistenceSensor` | `trigger.file_arrival` with `url: gs://bucket/prefix/` |
+| `FileSensor` | `trigger.file_arrival` with `url:` pointing to UC volume |
+
+**Airflow:**
+
+```python
+wait_for_data = S3KeySensor(
+    task_id="wait_for_data",
+    bucket_name="landing-zone",
+    bucket_key="data/{{ ds }}/*.parquet",
+    poke_interval=60,
+    timeout=3600,
+)
+```
+
+**DABs:**
+
+```yaml
+trigger:
+  file_arrival:
+    url: s3://landing-zone/data/
+    min_time_between_triggers_seconds: 60
+    wait_after_last_change_seconds: 60
+```
+
+**Key differences:**
+- Airflow sensors are task-level (block one task). DABs triggers are job-level (start the whole job).
+- Move the sensor to the job trigger. Downstream tasks that depended on the sensor now just run as the first task(s) in the job.
+
+---
+
+### Table-Based Sensors -> `trigger.table_update`
+
+| Airflow Sensor | Trigger Config |
+|---|---|
+| `ExternalTaskSensor` (if upstream writes to a table) | `trigger.table_update` monitoring the output table |
+| `SqlSensor` (if checking table freshness/existence) | `trigger.table_update` with `condition: ANY_UPDATED` |
+
+**Airflow:**
+
+```python
+wait_for_upstream = ExternalTaskSensor(
+    task_id="wait_for_upstream",
+    external_dag_id="upstream_etl",
+    external_task_id="write_silver_table",
+    timeout=3600,
+)
+```
+
+**DABs:**
+
+```yaml
+trigger:
+  table_update:
+    condition: ANY_UPDATED
+    table_names:
+      - "main.silver.transactions"
+    min_time_between_triggers_seconds: 300
+    wait_after_last_change_seconds: 60
+```
+
+> **Continuous Lakeflow Connect pipelines** (streaming connectors like Kafka/RabbitMQ, or any connector
+> documented continuous-only) are **not** driven by a `pipeline_task` hop. Run the pipeline standalone
+> and have the downstream job depend on a job-level `trigger.table_update` on the pipeline's destination
+> table — the same mechanism above. A **triggered** ingestion pipeline uses a `pipeline_task` instead.
+> See `references/lakeflow-connect.md`.
+
+---
+
+### Dependency-Based Sensors -> `depends_on` or `run_job_task`
+
+| Airflow Sensor | DABs Equivalent |
+|---|---|
+| `ExternalTaskSensor` (same job/bundle) | `depends_on` with `task_key` |
+| `ExternalTaskSensor` (cross-job) | Upstream job triggers downstream via `run_job_task` |
+
+---
+
+### Time-Based Sensors -> Schedule Adjustment
+
+| Airflow Sensor | DABs Equivalent |
+|---|---|
+| `TimeSensor` / `TimeDeltaSensor` at DAG start | Adjust `schedule.quartz_cron_expression` to the target time |
+| `TimeSensor` / `TimeDeltaSensor` mid-pipeline | Flag in MIGRATION_NOTES.md -- no direct equivalent |
+| `DayOfWeekSensor` | Adjust cron to run only on specified days |
+
+---
+
+## Airflow Timetable, Dataset, and Asset Scheduling
+
+Airflow DAGs can use non-cron schedule APIs that are not 1:1 with a Quartz cron. In **Airflow 3**,
+"Datasets" are renamed **Assets** (`airflow.sdk.Asset`); the scheduling mappings below apply to
+both `Dataset(...)` (Airflow 2) and `Asset(...)` (Airflow 3). See `references/airflow3-migration.md`.
+
+| Airflow Scheduling Pattern | DABs Mapping |
+|---|---|
+| `schedule=[Dataset(...)]` / `schedule=[Asset(...)]` (single) | `trigger.table_update` on the upstream Unity Catalog table — **only** when the asset resolves to a UC table (see resolution rule below); otherwise flag. |
+| `schedule=[asset_a, asset_b]` (list — Airflow: ALL updated) | `trigger.table_update` on both tables with `condition: ALL_UPDATED`. |
+| `schedule=(asset_a \| asset_b)` (OR) | `trigger.table_update` with `condition: ANY_UPDATED`. |
+| `schedule=(asset_a & asset_b)` (AND) | `trigger.table_update` with `condition: ALL_UPDATED`. |
+| `AssetOrTimeSchedule(timetable=..., assets=...)` (time **and** asset) | **Flag** — a single Lakeflow job takes either a `schedule` **or** a trigger, not both as a clean 1:1. Choose the dominant intent (or split), and record the tradeoff in `MIGRATION_NOTES.md`. |
+| Custom `Timetable` subclass | Flag for manual review and map to `schedule` or `trigger` based on business intent. |
+| `@continuous` | Use job-level `continuous` (not periodic trigger). |
+
+**Asset → UC-table resolution rule.** An Airflow `Asset` URI is an arbitrary string (it may be an
+S3 path, a file path, a custom scheme, or a bare name) — there is **no** official Airflow/Databricks
+convention that encodes a Unity Catalog table in it. So the default is to **flag**, and an asset
+maps to `trigger.table_update` only when the target table is stated unambiguously by one of:
+
+1. **Explicit metadata (recommended):** `Asset("orders-raw", extra={"databricks_table": "..
"})`. +2. **A user-supplied URI→table mapping** given in the conversion prompt. +3. **A skill-local scheme with exact parsing:** the URI is `x-databricks-table:..
`. + This is a convention of *this skill only* — document it as such in `MIGRATION_NOTES.md`; it is not + an Airflow or Databricks standard. + +Any other asset URI (`s3://…`, `file://…`, a bare string, or an ambiguous scheme) → **flag for +manual review** in `MIGRATION_NOTES.md`. Never guess a table from the URI. + +If a dataset/asset or timetable schedule cannot be deterministically mapped, add a required action +item in `MIGRATION_NOTES.md`. + +--- + +## Airflow `default_args` Mapping + +Common `default_args` fields and their DABs equivalents: + +| Airflow `default_args` | DABs Equivalent | +|---|---| +| `owner` | *(no direct mapping -- do not auto-map identity; document intended run identity in MIGRATION_NOTES.md)* | +| `retries` | `max_retries` on task | +| `retry_delay` | `min_retry_interval_millis` on task | +| `email` | `email_notifications.on_failure` | +| `email_on_failure` | `email_notifications.on_failure` | +| `email_on_retry` | *(no direct equivalent, note in migration notes)* | +| `depends_on_past` | *(no direct equivalent, note in migration notes)* | +| `start_date` | *(not needed -- DABs jobs start when deployed)* | +| `end_date` | *(no direct equivalent -- pause the schedule manually)* | +| `execution_timeout` | `timeout_seconds` on task | +| `sla` | *(no direct equivalent -- use monitoring/alerts)* | +| `catchup` | `catchup=True` → use native [Databricks backfill](https://docs.databricks.com/aws/en/jobs/backfill-jobs) to replay history (requires `{{ ds }}` mapped to a job parameter — see the execution-date section); `catchup=False` (the Airflow 3 default) → no backfill. Note the expectation in MIGRATION_NOTES.md. | + +--- + +## `trigger_rule` → `run_if` Mapping + +Lakeflow `run_if` takes exactly six values: `ALL_SUCCESS`, `ALL_DONE`, `NONE_FAILED`, +`AT_LEAST_ONE_SUCCESS`, `ALL_FAILED`, `AT_LEAST_ONE_FAILED`. Airflow has more trigger rules than +that, so some map exactly, some are approximations that must be recorded, and the rest have no +faithful mapping and must be flagged. + +**Exact:** + +| Airflow `trigger_rule` | `run_if` | +|---|---| +| `all_success` (default) | `ALL_SUCCESS` (omit — it is the default) | +| `all_done` | `ALL_DONE` | +| `all_failed` | `ALL_FAILED` | +| `one_success` | `AT_LEAST_ONE_SUCCESS` | +| `one_failed` | `AT_LEAST_ONE_FAILED` | + +**Approximate — map, but record the behavioral delta in `MIGRATION_NOTES.md`:** + +| Airflow `trigger_rule` | `run_if` | Delta to record | +|---|---|---| +| `none_failed` | `NONE_FAILED` | Confirm skip semantics for the specific fan-in. | +| `none_failed_min_one_success` | `NONE_FAILED` | Drops the "at least one succeeded" clause: the task **also runs when every upstream skipped**. `AT_LEAST_ONE_SUCCESS` is the wrong substitute — it allows the task to run while another upstream has failed. | +| `none_failed_or_skipped` (deprecated alias) | `NONE_FAILED` | Same as `none_failed`. | + +**Flag — no faithful mapping; use a `condition_task` or surface for manual review:** + +`always` / `dummy` (Airflow runs regardless of upstream state, *including upstreams that never ran*; +`ALL_DONE` still waits for upstreams to reach a terminal state), `none_skipped`, `all_skipped`, +`one_done`, and the setup/teardown-specific rules. `all_skipped` mapped to the default inverts to its +opposite condition — it would run only when upstreams *succeeded*. + +> An unrecognized or dynamically-computed `trigger_rule` must be flagged, never defaulted. A default +> of `ALL_SUCCESS` is indistinguishable from a correctly-mapped `all_success`, which hides the loss. + +--- + +## Execution date (`{{ ds }}` / `execution_date`) semantics and backfill + +Airflow's `{{ ds }}`/`execution_date` has **no single Databricks equivalent** — its correct mapping +depends on what the DAG *means* by it, and the wrong choice silently processes the wrong data (most +dangerously under backfill). Decide the semantics per DAG before mapping, and **ask the user when it +is ambiguous** — do not default silently. + +**Step 1 — classify the intent of each `{{ ds }}` use:** + +- **Wall-clock / "today's data"** — the task just wants the date the run happens on, with no + historical-replay meaning. Rare in scheduled ETL. → default the parameter to + `{{job.start_time.iso_date}}` (actual execution start). +- **Logical interval / partition key** — `{{ ds }}` identifies *which* data window is being processed + (a `WHERE date = '{{ ds }}'` filter, a partition path `.../{{ ds }}/...`, an incremental cursor). + This is the common case and the one that must survive backfill. → default the parameter to + **`{{job.trigger.time.iso_date}}`** (the scheduled trigger time), not `start_time` — `start_time` + drifts with queue delay and retries. + + > Airflow 2 vs Databricks convention: an Airflow 2 scheduled run's `logical_date` is the **start of + > the data interval** (typically one period *behind* the run's fire time), while Databricks + > `{{job.trigger.time}}` and Airflow 3's `CronTriggerTimetable` `logical_date` are the **fire time**. + > If the DAG's `{{ ds }}` relied on the Airflow 2 "process the previous interval" convention, + > confirm the intended window with the user and offset in code if needed; flag when unsure. + +> `{{job.trigger.time}}` is defined for **cron/scheduled** runs. If the DAG's schedule became an +> **event trigger** (`file_arrival`, `table_update`, `continuous`) — e.g. a cron+sensor collapsed to +> file arrival — there is no scheduled logical date: `{{job.trigger.time}}` is not a reliable source. +> Derive the partition from the event itself (e.g. parse the date from the arriving file path / +> `{{job.trigger.file_arrival.location}}`), fall back to `{{job.start_time.iso_date}}` as an +> approximation, or use backfill for exact historical windows — and flag the change. + +**Step 2 — always make it a real job parameter (backfill resilience).** Whichever default you choose, +`{{ ds }}` must map to a named **job parameter** (e.g. `run_date`), never a hardcoded date or an +inline `{{job.start_time...}}` buried in a task. Native [Databricks backfill](https://docs.databricks.com/aws/en/jobs/backfill-jobs) +replays a job over a historical range by **overriding an existing date/time job parameter** per +replayed window with `{{backfill.iso_date}}` (the start of that window's range). What makes a job +backfillable is that such a parameter **exists** to be overridden — a job that hardcodes the date or +computes it inline from `{{job.start_time...}}` gives backfill nothing to override, so history cannot +be replayed for the right window. (During a backfill the override wins regardless of the parameter's +default; the default only governs **normal** runs — which is why a logical/partition date should +default to `{{job.trigger.time.iso_date}}`, not `{{job.start_time.iso_date}}`, whose drift on delayed +or retried runs would process the wrong date.) So: expose `run_date`, default it to the Step-1 choice, +and record in `MIGRATION_NOTES.md` that a backfill should override `run_date` with `{{backfill.iso_date}}`. (Backfills always run the whole job; **pipeline tasks are not parameterized** +and run as-is, so a pipeline-only workload can't carry a backfill date — flag it.) + +## Jinja Template Variable Conversion + +Airflow Jinja variables used in operators/SQL need conversion to DABs dynamic value references. + +> Dynamic value references belong in PARAMETER values (`base_parameters`, `sql_task.parameters`, task `parameters` lists) — never inline in SQL files. SQL files use `:name` parameter markers whose values are supplied via `sql_task.parameters` (see `references/dab-schema-reference.md`). + +| Airflow Jinja | DABs Equivalent | Notes | +|---|---|---| +| `{{ ds }}` | `{{job.parameters.run_date}}` | Define `run_date` as a job parameter (so backfill can override it). Default `{{job.trigger.time.iso_date}}` for a logical/partition date on a scheduled job (correct on normal runs); `{{job.start_time.iso_date}}` only for wall-clock "today" semantics or an event-triggered job. See the semantics + backfill section above. | +| `{{ ds_nodash }}` | *(compute in notebook)* | No direct equivalent. Derive from `run_date` in code. | +| `{{ execution_date }}` | `{{job.parameters.run_date}}` | Same as `ds`; classify wall-clock vs logical per the section above. | +| `{{ prev_ds }}` | *(compute in notebook)* | No direct equivalent. Calculate in code. | +| `{{ next_ds }}` | *(compute in notebook)* | No direct equivalent. Calculate in code. | +| `{{ params.x }}` | `{{job.parameters.x}}` | Define as job parameter | +| `{{ var.value.x }}` | `${var.x}` | Define as bundle variable | +| `{{ task_instance.xcom_pull(...) }}` | `{{tasks..values.}}` | Use `dbutils.jobs.taskValues.set/get` | +| `{{ run_id }}` | `{{job.run_id}}` | Direct mapping | +| `{{ dag.dag_id }}` | `${bundle.name}` or hardcode | Bundle name is typically the DAG equivalent | +| `{{ macros.ds_add(ds, -1) }}` | *(compute in notebook)* | No macro support. Calculate in Python/SQL. | diff --git a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md index cac122a..09e78f2 100644 --- a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md +++ b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md @@ -22,7 +22,7 @@ Databricks `task_key`. `task_path` identifies the exact placeholder location, in "request_sha256": "copied from the envelope", "provider": { "name": "airflow-to-dabs", - "version": "0.2.0", + "version": "0.2.1", "repository": "https://github.com/park-peter/airflow-to-dabs" }, "model": {"name": "model identifier"}, @@ -72,4 +72,4 @@ automatic retry occurs. `{{tasks.upstream.values.x}}`, and `{{input}}` are valid in replacement parameter values, not in uploaded notebook or SQL source files; source files must read widgets or SQL named parameters. - Every source argument in the envelope has exactly one disposition and a non-empty rationale. -- The provider identity must match the pinned `airflow-to-dabs` v0.2.0 knowledge release. +- The provider identity must match the pinned `airflow-to-dabs` v0.2.1 knowledge release. diff --git a/src/flowx/agentic.py b/src/flowx/agentic.py index 271c328..39a4773 100644 --- a/src/flowx/agentic.py +++ b/src/flowx/agentic.py @@ -24,7 +24,7 @@ CONTRACT_VERSION = "1" PROVIDER_NAME = "airflow-to-dabs" -PROVIDER_VERSION = "0.2.0" +PROVIDER_VERSION = "0.2.1" PROVIDER_REPOSITORY = "https://github.com/park-peter/airflow-to-dabs" _ALLOWED_REPLACEMENT_KINDS = ("notebook", "sql", "spark_python") @@ -174,6 +174,8 @@ def prepare_airflow_resolutions( raise AgenticContractError(f"No Airflow DAG files found under {source_path}") source_hashes = {relative: _sha256_file(path) for relative, path in source_files} baseline_hash = _sha256_bytes(baseline_bytes) + provider_source = _provider_context_path() + provider_sha256 = _directory_sha256(provider_source) work_dir = output_dir / ".work" work_dir.mkdir(parents=True, exist_ok=True) @@ -195,7 +197,12 @@ def prepare_airflow_resolutions( "Airflow source no longer reproduces the deterministic report; rerun convert before prepare" ) - gaps = _build_gap_envelopes(baseline, baseline_hash=baseline_hash, source_hashes=source_hashes) + gaps = _build_gap_envelopes( + baseline, + baseline_hash=baseline_hash, + source_hashes=source_hashes, + provider_sha256=provider_sha256, + ) if not gaps: raise AgenticContractError("The deterministic report contains no eligible Airflow leaf gaps") if gap_id is not None and gap_id not in {gap["gap_id"] for gap in gaps}: @@ -206,15 +213,13 @@ def prepare_airflow_resolutions( (staging / "gaps.json").write_bytes(gaps_bytes) (staging / "candidates").mkdir() _copy_provider_context(staging / "provider") + if _directory_sha256(staging / "provider") != provider_sha256: + raise AgenticContractError("Pinned provider context changed while the agentic workspace was prepared") _write_json(staging / "candidate_index.json", {}) manifest = { "contract_version": CONTRACT_VERSION, "source": "airflow", - "provider": { - "name": PROVIDER_NAME, - "version": PROVIDER_VERSION, - "repository": PROVIDER_REPOSITORY, - }, + "provider": {**_provider_identity(), "sha256": provider_sha256}, "source_path": str(source_path), "source_kind": "file" if source_path.is_file() else "directory", "source_files": [ @@ -611,17 +616,9 @@ def _load_persisted_agentic_evidence(evidence_dir: Path) -> PersistedResolutionE raise AgenticContractError("agentic resolution baseline hash does not match its manifest") if _sha256_bytes(gaps_bytes) != manifest.get("gaps_sha256"): raise AgenticContractError("agentic gap-envelope hash does not match its manifest") - expected_provider = { - "name": PROVIDER_NAME, - "version": PROVIDER_VERSION, - "repository": PROVIDER_REPOSITORY, - } - if ( - manifest.get("contract_version") != CONTRACT_VERSION - or manifest.get("source") != "airflow" - or manifest.get("provider") != expected_provider - ): - raise AgenticContractError("agentic resolution manifest has an unsupported contract, source, or provider") + if manifest.get("contract_version") != CONTRACT_VERSION or manifest.get("source") != "airflow": + raise AgenticContractError("agentic resolution manifest has an unsupported contract or source") + provider_sha256 = _validate_manifest_provider(manifest.get("provider")) if accepted.get("contract_version") != CONTRACT_VERSION: raise AgenticContractError("accepted_resolutions.json has an unsupported contract_version") if reviewed.get("contract_version") != CONTRACT_VERSION: @@ -663,6 +660,7 @@ def _load_persisted_agentic_evidence(evidence_dir: Path) -> PersistedResolutionE baseline, baseline_hash=str(manifest["baseline_report_sha256"]), source_hashes=source_hashes, + provider_sha256=provider_sha256, ) if gaps != expected_gaps: raise AgenticContractError("persisted gap envelopes do not match the immutable baseline") @@ -768,6 +766,7 @@ def _build_gap_envelopes( *, baseline_hash: str, source_hashes: dict[str, str], + provider_sha256: str, ) -> list[dict[str, Any]]: envelopes: list[dict[str, Any]] = [] for pipeline in _pipeline_list(baseline): @@ -818,11 +817,6 @@ def _build_gap_envelopes( if not any(item.get("code") == "unsupported_trigger_rule" for item in related_findings): flowx_owned_arguments.add("trigger_rule") sanitized_definition = _sanitize_raw_definition(raw_definition) - provider = { - "name": PROVIDER_NAME, - "version": PROVIDER_VERSION, - "repository": PROVIDER_REPOSITORY, - } envelope = GapEnvelope( gap_id=str(matched_finding["fingerprint"]), pipeline_name=str(pipeline["name"]), @@ -836,7 +830,7 @@ def _build_gap_envelopes( baseline_report_sha256=baseline_hash, task_sha256=_sha256_bytes(_json_bytes(task)), graph_sha256=_graph_hash(pipeline), - provider_sha256=_sha256_bytes(_json_bytes(provider)), + provider_sha256=provider_sha256, finding_fingerprints=sorted( {str(item["fingerprint"]) for item in related_findings if isinstance(item.get("fingerprint"), str)} ), @@ -1359,13 +1353,9 @@ def _load_workspace(workspace: Path) -> tuple[dict[str, Any], list[dict[str, Any manifest = _read_json_object(workspace / "manifest.json") if manifest.get("contract_version") != CONTRACT_VERSION or manifest.get("source") != "airflow": raise AgenticContractError("Agentic workspace has an unsupported contract or source") - expected_provider = { - "name": PROVIDER_NAME, - "version": PROVIDER_VERSION, - "repository": PROVIDER_REPOSITORY, - } - if manifest.get("provider") != expected_provider: - raise AgenticContractError(f"Agentic workspace provider must be pinned to {PROVIDER_NAME} v{PROVIDER_VERSION}") + provider_sha256 = _validate_manifest_provider(manifest.get("provider")) + if _directory_sha256(workspace / "provider") != provider_sha256: + raise AgenticContractError("Pinned provider context was modified after prepare") gaps_bytes = (workspace / "gaps.json").read_bytes() if _sha256_bytes(gaps_bytes) != manifest.get("gaps_sha256"): raise AgenticContractError("Prepared GapEnvelope file was modified after prepare") @@ -1398,6 +1388,10 @@ def _workspace(output_dir: Path) -> Path: def _copy_provider_context(destination: Path) -> None: + shutil.copytree(_provider_context_path(), destination) + + +def _provider_context_path() -> Path: source = ( Path(__file__).resolve().parents[2] / "skills" @@ -1409,7 +1403,47 @@ def _copy_provider_context(destination: Path) -> None: raise AgenticContractError( f"provider_unavailable: pinned {PROVIDER_NAME} v{PROVIDER_VERSION} context is missing" ) - shutil.copytree(source, destination) + return source + + +def _provider_identity() -> dict[str, str]: + return {"name": PROVIDER_NAME, "version": PROVIDER_VERSION, "repository": PROVIDER_REPOSITORY} + + +def _validate_manifest_provider(value: Any) -> str: + if not isinstance(value, dict) or {key: value.get(key) for key in _provider_identity()} != _provider_identity(): + raise AgenticContractError(f"Agentic workspace provider must be pinned to {PROVIDER_NAME} v{PROVIDER_VERSION}") + sha256 = value.get("sha256") + if ( + set(value) != {*_provider_identity(), "sha256"} + or not isinstance(sha256, str) + or not re.fullmatch(r"[0-9a-f]{64}", sha256) + ): + raise AgenticContractError("Agentic workspace provider pin is invalid") + return sha256 + + +def _directory_sha256(directory: Path) -> str: + if not directory.is_dir(): + raise AgenticContractError(f"Pinned provider context is missing: {directory}") + files: list[Path] = [] + for path in directory.rglob("*"): + if path.is_symlink(): + raise AgenticContractError(f"Pinned provider context cannot contain symlinks: {path}") + if path.is_file(): + files.append(path) + if not files: + raise AgenticContractError("Pinned provider context contains no files") + digest = hashlib.sha256() + for path in sorted(files, key=lambda item: item.relative_to(directory).as_posix()): + relative = path.relative_to(directory).as_posix().encode() + content = path.read_bytes() + digest.update(relative) + digest.update(b"\0") + digest.update(str(len(content)).encode()) + digest.update(b"\0") + digest.update(content) + return digest.hexdigest() def _safe_relative_path(value: str) -> bool: diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 39a5f55..1dac06d 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -1759,7 +1759,13 @@ def _report_reconciliation_failures(report_path: Path) -> list[str]: return [f"legacy ADF translation at index {index} is missing pipeline or IR data"] return [] - if any( + airflow_agentic_report = report_path.name == "translation_report.agentic.json" and any( + isinstance(pipeline, dict) + and isinstance(pipeline.get("tags"), dict) + and pipeline["tags"].get("source") == "airflow" + for pipeline in pipelines + ) + if airflow_agentic_report or any( isinstance(pipeline, dict) and ( pipeline.get("reconciliation_status") == "verified_with_reviewed_resolutions" diff --git a/src/flowx/mcp/server.py b/src/flowx/mcp/server.py index 16fd03c..36c67fc 100644 --- a/src/flowx/mcp/server.py +++ b/src/flowx/mcp/server.py @@ -548,7 +548,7 @@ def flowx(command: str, parameters: dict[str, Any] | None = None) -> dict[str, A - "merge_agentic": source(req: "adf"), report_path(req), agentic_results_dir(req), output_path — merge ADF agent results. Airflow's legacy name-based merge is disabled; use resolve_agentic. - "resolve_agentic": source(req: "airflow"), action(req: prepare | stage | apply), output_dir, - airflow_source_path, report_path, candidates, replace, accept_gap | accept_gaps, accept_all, + airflow_source_path, report_path, gap_id, candidates, replace, accept_gap | accept_gaps, accept_all, review_complete, review_manifest, reset — prepare, stage, and explicitly apply fingerprint-bound Airflow leaf-gap resolutions. - "inspect": report_path(req) — return the full translation-option schema (every option with diff --git a/src/flowx/reporting/coverage.py b/src/flowx/reporting/coverage.py index 9fa0a19..8f15238 100644 --- a/src/flowx/reporting/coverage.py +++ b/src/flowx/reporting/coverage.py @@ -33,7 +33,8 @@ "other_activities", "deterministic_activities", "agentic_activities", - "unresolved_agentic_activities", + "resolved_agentic_count", + "unresolved_agentic_count", "agentic_resolution_outcomes", "agentic_provider_version", "unsupported_activities", @@ -43,7 +44,7 @@ "migration_status", "coverage_pct", "deterministic_coverage_pct", - "runnable_coverage_pct", + "code_attached_coverage_pct", "finding_count", "finding_fingerprints", "complexity_score", @@ -75,7 +76,7 @@ def _deterministic_coverage_pct(deterministic: int, total: int) -> float: return round(deterministic / total * 100, 1) -def _runnable_coverage_pct(deterministic: int, resolved: int, total: int) -> float: +def _code_attached_coverage_pct(deterministic: int, resolved: int, total: int) -> float: """Mechanically code-attached coverage over audited activity candidates.""" if total <= 0: return 0.0 @@ -133,11 +134,9 @@ def build_coverage_rows(metadata_dir: Path) -> list[dict[str, Any]]: excluded = int(pipeline.get("excluded_count", 0)) if has_audit else 0 total = int(pipeline.get("audited_activity_count", 0)) if has_audit else len(strategies) if is_airflow: + empty_outcomes = {"resolved": 0, "needs_input": 0, "deferred": 0, "declined": 0, "unreviewed": agentic} if agentic_summary: - outcomes = resolution_pipelines.get( - name, - {"resolved": 0, "needs_input": 0, "deferred": 0, "unreviewed": 0}, - ) + outcomes = resolution_pipelines.get(name, empty_outcomes) if not isinstance(outcomes, dict) or any( not isinstance(outcomes.get(key), int) for key in ("resolved", "needs_input", "deferred", "declined", "unreviewed") @@ -149,14 +148,15 @@ def build_coverage_rows(metadata_dir: Path) -> list[dict[str, Any]]: f"{agentic} agentic activities in pipeline {name!r}" ) else: - outcomes = {"resolved": 0, "needs_input": 0, "deferred": 0, "declined": 0, "unreviewed": agentic} + outcomes = empty_outcomes resolved_agentic = outcomes["resolved"] unresolved_agentic = agentic - resolved_agentic - runnable_coverage = _runnable_coverage_pct(deterministic, resolved_agentic, total) + code_attached_coverage = _code_attached_coverage_pct(deterministic, resolved_agentic, total) else: outcomes = {} + resolved_agentic = agentic unresolved_agentic = 0 - runnable_coverage = _coverage_pct(deterministic, agentic, total) + code_attached_coverage = _coverage_pct(deterministic, agentic, total) findings = pipeline.get("findings", []) fingerprints = [ finding["fingerprint"] @@ -184,7 +184,8 @@ def _csv_int(col: str, _csv_row: dict[str, str] = csv_row) -> int: "other_activities": _csv_int("other_activities"), "deterministic_activities": deterministic, "agentic_activities": agentic, - "unresolved_agentic_activities": unresolved_agentic, + "resolved_agentic_count": resolved_agentic, + "unresolved_agentic_count": unresolved_agentic, "agentic_resolution_outcomes": json.dumps(outcomes, sort_keys=True, separators=(",", ":")), "agentic_provider_version": provider_version if is_airflow else "", "unsupported_activities": unsupported, @@ -198,7 +199,7 @@ def _csv_int(col: str, _csv_row: dict[str, str] = csv_row) -> int: "migration_status": pipeline.get("migration_status", "included"), "coverage_pct": _coverage_pct(deterministic, agentic, total), "deterministic_coverage_pct": _deterministic_coverage_pct(deterministic, total), - "runnable_coverage_pct": runnable_coverage, + "code_attached_coverage_pct": code_attached_coverage, "finding_count": len(findings), "finding_fingerprints": json.dumps(fingerprints, separators=(",", ":")), "complexity_score": _csv_int("complexity_score"), diff --git a/src/flowx/reporting/dashboard_template.json b/src/flowx/reporting/dashboard_template.json index 39f3f72..2ee2135 100644 --- a/src/flowx/reporting/dashboard_template.json +++ b/src/flowx/reporting/dashboard_template.json @@ -8,13 +8,14 @@ "SUM(audited_activities) AS audited_activities, ", "SUM(deterministic_activities) AS deterministic_activities, ", "SUM(agentic_activities) AS agentic_activities, ", - "SUM(unresolved_agentic_activities) AS unresolved_agentic_activities, ", + "SUM(resolved_agentic_count) AS resolved_agentic_count, ", + "SUM(unresolved_agentic_count) AS unresolved_agentic_count, ", "SUM(unsupported_activities) AS unsupported_activities, ", "SUM(failed_activities) AS failed_activities, ", "SUM(excluded_activities) AS excluded_activities, ", "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(audited_activities),0),1) AS coverage_pct, ", "ROUND(100.0*SUM(deterministic_activities)/NULLIF(SUM(audited_activities),0),1) AS deterministic_coverage_pct, ", - "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities)-SUM(unresolved_agentic_activities))/NULLIF(SUM(audited_activities),0),1) AS runnable_coverage_pct ", + "ROUND(100.0*(SUM(deterministic_activities)+SUM(resolved_agentic_count))/NULLIF(SUM(audited_activities),0),1) AS code_attached_coverage_pct ", "FROM {{RESULTS_TABLE}} ", "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1)" ] @@ -33,13 +34,13 @@ "name": "latest_pipelines", "displayName": "Pipeline coverage (latest run)", "queryLines": [ - "SELECT pipeline, audited_activities, deterministic_activities, agentic_activities, unresolved_agentic_activities, ", + "SELECT pipeline, audited_activities, deterministic_activities, agentic_activities, resolved_agentic_count, unresolved_agentic_count, ", "unsupported_activities, failed_activities, excluded_activities, reconciliation_status, ", - "migration_status, coverage_pct, deterministic_coverage_pct, runnable_coverage_pct, ", + "migration_status, coverage_pct, deterministic_coverage_pct, code_attached_coverage_pct, ", "agentic_resolution_outcomes, agentic_provider_version, finding_count, collapsible_patterns, complexity_size ", "FROM {{RESULTS_TABLE}} ", "WHERE run_id = (SELECT run_id FROM {{RESULTS_TABLE}} ORDER BY run_date DESC LIMIT 1) ", - "ORDER BY runnable_coverage_pct ASC, audited_activities DESC" + "ORDER BY code_attached_coverage_pct ASC, audited_activities DESC" ] }, { @@ -49,7 +50,7 @@ "SELECT DATE_TRUNC('SECOND', run_date) AS run_ts, ", "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities))/NULLIF(SUM(audited_activities),0),1) AS coverage_pct, ", "ROUND(100.0*SUM(deterministic_activities)/NULLIF(SUM(audited_activities),0),1) AS deterministic_coverage_pct, ", - "ROUND(100.0*(SUM(deterministic_activities)+SUM(agentic_activities)-SUM(unresolved_agentic_activities))/NULLIF(SUM(audited_activities),0),1) AS runnable_coverage_pct, ", + "ROUND(100.0*(SUM(deterministic_activities)+SUM(resolved_agentic_count))/NULLIF(SUM(audited_activities),0),1) AS code_attached_coverage_pct, ", "SUM(audited_activities) AS audited_activities, SUM(failed_activities) AS failed_activities, ", "SUM(excluded_activities) AS excluded_activities ", "FROM {{RESULTS_TABLE}} ", @@ -85,7 +86,7 @@ "name": "subtitle", "multilineTextboxSpec": { "lines": [ - "Code-attached coverage counts deterministic and reviewed provider code that passed mechanical validation. It does not certify semantic correctness." + "Code attached — deterministic or reviewed agentic; semantic correctness not verified" ] } }, @@ -146,8 +147,8 @@ "datasetName": "latest_summary", "fields": [ { - "name": "runnable_coverage_pct", - "expression": "`runnable_coverage_pct`" + "name": "code_attached_coverage_pct", + "expression": "`code_attached_coverage_pct`" } ], "disaggregated": true @@ -159,7 +160,7 @@ "widgetType": "counter", "encodings": { "value": { - "fieldName": "runnable_coverage_pct", + "fieldName": "code_attached_coverage_pct", "displayName": "Code-attached %" } }, @@ -404,8 +405,8 @@ "expression": "`run_ts`" }, { - "name": "runnable_coverage_pct", - "expression": "`runnable_coverage_pct`" + "name": "code_attached_coverage_pct", + "expression": "`code_attached_coverage_pct`" } ], "disaggregated": true @@ -424,7 +425,7 @@ "displayName": "Run" }, "y": { - "fieldName": "runnable_coverage_pct", + "fieldName": "code_attached_coverage_pct", "scale": { "type": "quantitative" }, @@ -470,8 +471,12 @@ "expression": "`agentic_activities`" }, { - "name": "unresolved_agentic_activities", - "expression": "`unresolved_agentic_activities`" + "name": "resolved_agentic_count", + "expression": "`resolved_agentic_count`" + }, + { + "name": "unresolved_agentic_count", + "expression": "`unresolved_agentic_count`" }, { "name": "failed_activities", @@ -494,8 +499,8 @@ "expression": "`coverage_pct`" }, { - "name": "runnable_coverage_pct", - "expression": "`runnable_coverage_pct`" + "name": "code_attached_coverage_pct", + "expression": "`code_attached_coverage_pct`" }, { "name": "agentic_resolution_outcomes", @@ -540,7 +545,11 @@ "displayName": "Agentic" }, { - "fieldName": "unresolved_agentic_activities", + "fieldName": "resolved_agentic_count", + "displayName": "Resolved agentic" + }, + { + "fieldName": "unresolved_agentic_count", "displayName": "Unresolved agentic" }, { @@ -564,7 +573,7 @@ "displayName": "Translation path %" }, { - "fieldName": "runnable_coverage_pct", + "fieldName": "code_attached_coverage_pct", "displayName": "Code-attached %" }, { diff --git a/src/flowx/reporting/results.py b/src/flowx/reporting/results.py index 294ab62..7ec5f50 100644 --- a/src/flowx/reporting/results.py +++ b/src/flowx/reporting/results.py @@ -32,7 +32,8 @@ "other_activities": "INT", "deterministic_activities": "INT", "agentic_activities": "INT", - "unresolved_agentic_activities": "INT", + "resolved_agentic_count": "INT", + "unresolved_agentic_count": "INT", "agentic_resolution_outcomes": "STRING", "agentic_provider_version": "STRING", "unsupported_activities": "INT", @@ -42,7 +43,7 @@ "migration_status": "STRING", "coverage_pct": "DOUBLE", "deterministic_coverage_pct": "DOUBLE", - "runnable_coverage_pct": "DOUBLE", + "code_attached_coverage_pct": "DOUBLE", "finding_count": "INT", "finding_fingerprints": "STRING", "complexity_score": "INT", @@ -67,7 +68,7 @@ "complexity_size", } ) -_FLOAT_METRICS: frozenset[str] = frozenset({"coverage_pct", "deterministic_coverage_pct", "runnable_coverage_pct"}) +_FLOAT_METRICS: frozenset[str] = frozenset({"coverage_pct", "deterministic_coverage_pct", "code_attached_coverage_pct"}) def _sql_str(value: Any) -> str: diff --git a/tests/unit/test_airflow_agentic_resolution.py b/tests/unit/test_airflow_agentic_resolution.py index 3318ac5..ec98d1e 100644 --- a/tests/unit/test_airflow_agentic_resolution.py +++ b/tests/unit/test_airflow_agentic_resolution.py @@ -116,7 +116,7 @@ def _candidate(gap: dict, *, source: str = "print('Migrated from Airflow')\n", s "request_sha256": gap["request_sha256"], "provider": { "name": "airflow-to-dabs", - "version": "0.2.0", + "version": "0.2.1", "repository": "https://github.com/park-peter/airflow-to-dabs", }, "model": {"name": "test-model"}, @@ -191,29 +191,32 @@ def test_prepare_writes_versioned_fingerprint_bound_gap_without_changing_report( assert image["normalized_value"] == "python:3.11" assert (output / ".work" / "agentic" / "baseline.json").exists() assert (output / ".work" / "agentic" / "source" / "dag.py").read_bytes() == source.read_bytes() - assert (output / ".work" / "agentic" / "provider" / "PROFILE.md").exists() + assert (output / ".work" / "agentic" / "provider" / "providers" / "flowx-gap-resolver" / "PROFILE.md").exists() def test_prepare_can_select_one_gap_for_the_caller_without_losing_workspace_gaps(tmp_path: Path) -> None: source, output, gaps = _prepare(tmp_path, two_tasks=True) report = output / ".work" / "translation_report.json" - assert adapter_main( - [ - "resolve-agentic", - "prepare", - "--source", - "airflow", - "--source-path", - str(source), - "--report", - str(report), - "--output-dir", - str(output), - "--gap-id", - gaps[0]["gap_id"], - ] - ) == 0 + assert ( + adapter_main( + [ + "resolve-agentic", + "prepare", + "--source", + "airflow", + "--source-path", + str(source), + "--report", + str(report), + "--output-dir", + str(output), + "--gap-id", + gaps[0]["gap_id"], + ] + ) + == 0 + ) prepared = json.loads((output / ".work" / "agentic" / "gaps.json").read_text(encoding="utf-8")) manifest = json.loads((output / ".work" / "agentic" / "manifest.json").read_text(encoding="utf-8")) assert {gap["gap_id"] for gap in prepared} == {gap["gap_id"] for gap in gaps} @@ -591,7 +594,7 @@ def test_stage_rejects_unresolved_jinja_provider_drift_and_file_hash_mismatch(tm wrong_provider = _candidate(gaps[0]) wrong_provider["provider"]["version"] = "0.1.0" assert _stage(output, wrong_provider) == 1 - assert "pinned airflow-to-dabs v0.2.0" in capsys.readouterr().err + assert "pinned airflow-to-dabs v0.2.1" in capsys.readouterr().err bad_hash = _candidate(gaps[0]) bad_hash["generated_files"][0]["sha256"] = "0" * 64 @@ -599,6 +602,15 @@ def test_stage_rejects_unresolved_jinja_provider_drift_and_file_hash_mismatch(tm assert "sha256 does not match" in capsys.readouterr().err +def test_stage_rejects_provider_context_modified_after_prepare(tmp_path: Path, capsys) -> None: + _, output, gaps = _prepare(tmp_path) + profile = output / ".work" / "agentic" / "provider" / "providers" / "flowx-gap-resolver" / "PROFILE.md" + profile.write_text(profile.read_text(encoding="utf-8") + "\nmodified\n", encoding="utf-8") + + assert _stage(output, _candidate(gaps[0])) == 1 + assert "provider context was modified after prepare" in capsys.readouterr().err + + @pytest.mark.parametrize("field", ["task_sha256", "graph_sha256", "provider_sha256", "request_sha256"]) def test_stage_rejects_stale_request_identity(tmp_path: Path, capsys, field: str) -> None: _, output, gaps = _prepare(tmp_path) @@ -609,6 +621,28 @@ def test_stage_rejects_stale_request_identity(tmp_path: Path, capsys, field: str assert f"Candidate {field} does not match" in capsys.readouterr().err +@pytest.mark.parametrize( + ("field", "message"), + [ + ("baseline_report_sha256", "does not match the prepared baseline"), + ("source_sha256", "does not match its GapEnvelope"), + ("gap_id", "does not match a prepared gap"), + ], +) +def test_stage_rejects_stale_gap_source_and_report_identity( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + field: str, + message: str, +) -> None: + _, output, gaps = _prepare(tmp_path) + candidate = _candidate(gaps[0]) + candidate[field] = "0" * 64 + + assert _stage(output, candidate) == 1 + assert message in capsys.readouterr().err + + def test_stage_limits_artifact_size_and_allows_needs_input_disposition(tmp_path: Path, capsys) -> None: _, output, gaps = _prepare(tmp_path) oversized = _candidate(gaps[0], source="x = '" + "a" * (1024 * 1024) + "'\n") @@ -686,12 +720,20 @@ def test_stage_does_not_misclassify_airflow_input_names_as_dynamic_references(tm assert "unresolved Airflow Jinja" in capsys.readouterr().err -def test_pinned_v020_provider_fixtures_satisfy_the_flowx_contract() -> None: - root = Path(__file__).parents[2] / "skills" / "flowx-resolve-airflow-gaps" / "references" / "airflow-to-dabs-v0.2.0" +def test_pinned_v021_provider_fixtures_satisfy_the_flowx_contract() -> None: + root = ( + Path(__file__).parents[2] + / "skills" + / "flowx-resolve-airflow-gaps" + / "references" + / "airflow-to-dabs-v0.2.1" + / "providers" + / "flowx-gap-resolver" + ) provider = json.loads((root / "provider.json").read_text(encoding="utf-8")) - assert provider["provider"]["version"] == "0.2.0" - for outcome in ("notebook", "sql", "needs-input", "deferred"): + assert provider["provider"]["version"] == "0.2.1" + for outcome in ("notebook", "sql", "spark-python", "needs-input", "deferred"): gap = json.loads((root / "fixtures" / f"gap-{outcome}.json").read_text(encoding="utf-8")) candidate = json.loads((root / "fixtures" / f"resolution-{outcome}.json").read_text(encoding="utf-8")) manifest = {"baseline_report_sha256": gap["baseline_report_sha256"]} @@ -1101,6 +1143,34 @@ def test_accept_all_requires_an_exact_prior_review_manifest(tmp_path: Path, caps assert "does not exactly match" in capsys.readouterr().err +def test_identical_stage_and_allowlist_replay_are_byte_idempotent(tmp_path: Path) -> None: + _, output, gaps = _prepare(tmp_path) + candidate = _candidate(gaps[0]) + assert _stage(output, candidate) == 0 + candidate_path = output / ".work" / "agentic" / "candidates" / f"{gaps[0]['gap_id']}.json" + staged_bytes = candidate_path.read_bytes() + + assert _stage(output, candidate) == 0 + assert candidate_path.read_bytes() == staged_bytes + + args = [ + "resolve-agentic", + "apply", + "--source", + "airflow", + "--output-dir", + str(output), + "--accept-gap", + gaps[0]["gap_id"], + ] + assert adapter_main(args) == 0 + report = output / ".work" / "translation_report.agentic.json" + applied_bytes = report.read_bytes() + + assert adapter_main(args) == 0 + assert report.read_bytes() == applied_bytes + + def test_review_complete_declines_exact_staged_set_and_leaves_unstaged_gaps_unreviewed(tmp_path: Path) -> None: _, output, gaps = _prepare(tmp_path, two_tasks=True) assert _stage(output, _candidate(gaps[0])) == 0 @@ -1135,6 +1205,37 @@ def test_review_complete_declines_exact_staged_set_and_leaves_unstaged_gaps_unre ] outcomes = summarize_persisted_agentic_resolutions(evidence)["pipelines"]["agentic"] assert outcomes == {"resolved": 0, "needs_input": 0, "deferred": 0, "declined": 1, "unreviewed": 1} + assert package_main(["--report", str(report), "--output-dir", str(output)]) == 0 + + +def test_package_replays_review_complete_evidence_before_bundle_writes(tmp_path: Path, capsys) -> None: + _, output, gaps = _prepare(tmp_path) + assert _stage(output, _candidate(gaps[0])) == 0 + assert ( + adapter_main( + [ + "resolve-agentic", + "apply", + "--source", + "airflow", + "--output-dir", + str(output), + "--review-complete", + "--review-manifest", + str(_review_manifest(output)), + ] + ) + == 0 + ) + decisions_path = output / "metadata" / "agentic" / "review_decisions.json" + decisions = json.loads(decisions_path.read_text(encoding="utf-8")) + decisions["decisions"][0]["candidate_sha256"] = "0" * 64 + decisions_path.write_text(json.dumps(decisions), encoding="utf-8") + + report = output / ".work" / "translation_report.agentic.json" + assert package_main(["--report", str(report), "--output-dir", str(output)]) == 1 + assert "review decision hash does not match candidate" in capsys.readouterr().err + assert not (output / "databricks.yml").exists() def test_reset_uses_durable_baseline_after_source_change_and_work_pruning(tmp_path: Path) -> None: @@ -1288,7 +1389,7 @@ def test_package_rejects_agentic_report_tampering_before_bundle_writes(tmp_path: assert not (bundle / "databricks.yml").exists() -def test_reviewed_resolution_evidence_drives_honest_runnable_coverage(tmp_path: Path) -> None: +def test_reviewed_resolution_evidence_drives_honest_code_attached_coverage(tmp_path: Path) -> None: _, output, gaps = _prepare(tmp_path, two_tasks=True) assert _stage(output, _candidate(gaps[0]), name="resolved.json") == 0 assert _stage(output, _candidate(gaps[1], status="needs_input"), name="needs-input.json") == 0 @@ -1337,14 +1438,15 @@ def test_reviewed_resolution_evidence_drives_honest_runnable_coverage(tmp_path: row = build_coverage_rows(metadata)[0] assert summary == { - "provider_version": "0.2.0", + "provider_version": "0.2.1", "pipelines": {"agentic": {"resolved": 1, "needs_input": 1, "deferred": 0, "declined": 0, "unreviewed": 0}}, } assert row["coverage_pct"] == 100.0 assert row["deterministic_coverage_pct"] == 0.0 - assert row["runnable_coverage_pct"] == 50.0 - assert row["unresolved_agentic_activities"] == 1 - assert row["agentic_provider_version"] == "0.2.0" + assert row["code_attached_coverage_pct"] == 50.0 + assert row["resolved_agentic_count"] == 1 + assert row["unresolved_agentic_count"] == 1 + assert row["agentic_provider_version"] == "0.2.1" assert row["reconciliation_status"] == "verified_with_reviewed_resolutions" @@ -1411,7 +1513,8 @@ def test_reporting_keeps_non_resolver_source_gaps_unreviewed(tmp_path: Path) -> "declined": 0, "unreviewed": 1, } - assert row["unresolved_agentic_activities"] == 1 + assert row["resolved_agentic_count"] == 1 + assert row["unresolved_agentic_count"] == 1 def test_reporting_rejects_duplicate_hash_valid_agentic_evidence(tmp_path: Path) -> None: diff --git a/tests/unit/test_airflow_provider_sync.py b/tests/unit/test_airflow_provider_sync.py new file mode 100644 index 0000000..72880cf --- /dev/null +++ b/tests/unit/test_airflow_provider_sync.py @@ -0,0 +1,58 @@ +"""Tests for the vendored airflow-to-dabs provider pin.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).parents[2] +SCRIPT = ROOT / "scripts" / "sync_airflow_provider.py" +PROVIDER = ROOT / "skills" / "flowx-resolve-airflow-gaps" / "references" / "airflow-to-dabs-v0.2.1" + + +def _check(destination: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), "--check", "--destination", str(destination)], + check=False, + capture_output=True, + text=True, + ) + + +def test_committed_airflow_provider_pin_is_valid() -> None: + result = _check(PROVIDER) + + assert result.returncode == 0, result.stderr + pin = json.loads(result.stdout) + assert pin == { + "commit": "75196aef85ebb2736b926f2d4db13ec7d5c2551c", + "content_sha256": "e1e7395204b3f2759722b9320cea08c6b58284a48fd8062686b36bf676b657fd", + "tag": "v0.2.1", + } + + +def test_airflow_provider_pin_rejects_modified_content(tmp_path: Path) -> None: + destination = tmp_path / PROVIDER.name + shutil.copytree(PROVIDER, destination) + profile = destination / "providers" / "flowx-gap-resolver" / "PROFILE.md" + profile.write_text(profile.read_text(encoding="utf-8") + "\nmodified\n", encoding="utf-8") + + result = _check(destination) + + assert result.returncode != 0 + assert "content digest does not match" in result.stderr + + +def test_airflow_provider_pin_rejects_noncanonical_json(tmp_path: Path) -> None: + destination = tmp_path / PROVIDER.name + shutil.copytree(PROVIDER, destination) + manifest = destination / "providers" / "flowx-gap-resolver" / "provider.json" + manifest.write_text(manifest.read_text(encoding="utf-8") + "\n", encoding="utf-8") + + result = _check(destination) + + assert result.returncode != 0 + assert "JSON is not canonical" in result.stderr diff --git a/tests/unit/test_reporting_coverage.py b/tests/unit/test_reporting_coverage.py index 43264c6..17c43d7 100644 --- a/tests/unit/test_reporting_coverage.py +++ b/tests/unit/test_reporting_coverage.py @@ -76,8 +76,9 @@ def test_build_coverage_rows_joins_inventory_and_csv(tmp_path: Path): assert alpha["unsupported_activities"] == 1 # coverage = (det + agentic) / total = 3/4 = 75.0 assert alpha["coverage_pct"] == 75.0 - assert alpha["runnable_coverage_pct"] == 75.0 - assert alpha["unresolved_agentic_activities"] == 0 + assert alpha["code_attached_coverage_pct"] == 75.0 + assert alpha["resolved_agentic_count"] == 1 + assert alpha["unresolved_agentic_count"] == 0 assert alpha["agentic_resolution_outcomes"] == "{}" # complexity columns come from the CSV assert alpha["datasets"] == 2 and alpha["linked_services"] == 1 @@ -134,8 +135,9 @@ def test_audited_counts_drive_translation_and_deterministic_coverage(tmp_path: P assert verified["audited_activities"] == 8 assert verified["coverage_pct"] == 100.0 assert verified["deterministic_coverage_pct"] == 87.5 - assert verified["runnable_coverage_pct"] == 87.5 - assert verified["unresolved_agentic_activities"] == 1 + assert verified["code_attached_coverage_pct"] == 87.5 + assert verified["resolved_agentic_count"] == 0 + assert verified["unresolved_agentic_count"] == 1 assert json.loads(verified["agentic_resolution_outcomes"]) == { "resolved": 0, "needs_input": 0, @@ -151,7 +153,7 @@ def test_audited_counts_drive_translation_and_deterministic_coverage(tmp_path: P assert failed["failed_activities"] == 1 assert failed["coverage_pct"] == 88.9 assert failed["deterministic_coverage_pct"] == 77.8 - assert failed["runnable_coverage_pct"] == 77.8 + assert failed["code_attached_coverage_pct"] == 77.8 assert failed["reconciliation_status"] == "failed" @@ -183,5 +185,5 @@ def test_excluded_activities_remain_in_coverage_denominator(tmp_path: Path) -> N assert row["excluded_activities"] == 3 assert row["coverage_pct"] == 0.0 assert row["deterministic_coverage_pct"] == 0.0 - assert row["runnable_coverage_pct"] == 0.0 + assert row["code_attached_coverage_pct"] == 0.0 assert row["migration_status"] == "excluded" diff --git a/tests/unit/test_reporting_dashboard.py b/tests/unit/test_reporting_dashboard.py index cf80aa2..002f19a 100644 --- a/tests/unit/test_reporting_dashboard.py +++ b/tests/unit/test_reporting_dashboard.py @@ -22,8 +22,9 @@ def test_build_serialized_dashboard_injects_table_and_is_valid_json(): assert "excluded_activities" in joined assert "reconciliation_status" in joined assert "deterministic_coverage_pct" in joined - assert "runnable_coverage_pct" in joined - assert "unresolved_agentic_activities" in joined + assert "code_attached_coverage_pct" in joined + assert "resolved_agentic_count" in joined + assert "unresolved_agentic_count" in joined assert "agentic_resolution_outcomes" in joined assert "agentic_provider_version" in joined assert spec["pages"][0]["pageType"] == "PAGE_TYPE_CANVAS" @@ -31,6 +32,10 @@ def test_build_serialized_dashboard_injects_table_and_is_valid_json(): widget_names = {w["widget"]["name"] for w in spec["pages"][0]["layout"]} assert {"kpi-coverage", "by-size", "coverage-trend", "pipeline-table"} <= widget_names assert "mechanically validated" in serialized + subtitle = next(item["widget"] for item in spec["pages"][0]["layout"] if item["widget"]["name"] == "subtitle") + assert subtitle["multilineTextboxSpec"]["lines"] == [ + "Code attached — deterministic or reviewed agentic; semantic correctness not verified" + ] dataset_fields = {} for dataset in spec["datasets"]: diff --git a/tests/unit/test_reporting_results.py b/tests/unit/test_reporting_results.py index 5a9ca00..4c45ade 100644 --- a/tests/unit/test_reporting_results.py +++ b/tests/unit/test_reporting_results.py @@ -24,8 +24,9 @@ def test_create_table_sql_has_run_metadata_and_all_columns(): assert "excluded_activities INT" in sql assert "reconciliation_status STRING" in sql assert "deterministic_coverage_pct DOUBLE" in sql - assert "runnable_coverage_pct DOUBLE" in sql - assert "unresolved_agentic_activities INT" in sql + assert "code_attached_coverage_pct DOUBLE" in sql + assert "resolved_agentic_count INT" in sql + assert "unresolved_agentic_count INT" in sql assert "agentic_resolution_outcomes STRING" in sql assert "agentic_provider_version STRING" in sql assert "finding_fingerprints STRING" in sql @@ -41,7 +42,7 @@ def test_schema_evolution_sql_adds_only_missing_metric_columns() -> None: assert sql.startswith("ALTER TABLE cat.sch.tbl ADD COLUMNS") assert "audited_activities INT" in sql assert "deterministic_coverage_pct DOUBLE" in sql - assert "runnable_coverage_pct DOUBLE" in sql + assert "code_attached_coverage_pct DOUBLE" in sql assert "pipeline STRING" not in sql assert "\n coverage_pct DOUBLE" not in sql @@ -60,9 +61,10 @@ def test_insert_sql_stamps_run_metadata_and_escapes(): "other_activities": 2, "deterministic_activities": 2, "agentic_activities": 1, - "unresolved_agentic_activities": 1, + "resolved_agentic_count": 0, + "unresolved_agentic_count": 1, "agentic_resolution_outcomes": '{"unreviewed":1}', - "agentic_provider_version": "0.2.0", + "agentic_provider_version": "0.2.1", "unsupported_activities": 0, "failed_activities": 0, "excluded_activities": 0, @@ -70,7 +72,7 @@ def test_insert_sql_stamps_run_metadata_and_escapes(): "migration_status": "included", "coverage_pct": 100.0, "deterministic_coverage_pct": 66.7, - "runnable_coverage_pct": 66.7, + "code_attached_coverage_pct": 66.7, "finding_count": 1, "finding_fingerprints": '["abc"]', "complexity_score": 7, @@ -88,7 +90,8 @@ def test_insert_sql_stamps_run_metadata_and_escapes(): "other_activities": 1, "deterministic_activities": 0, "agentic_activities": 0, - "unresolved_agentic_activities": 0, + "resolved_agentic_count": 0, + "unresolved_agentic_count": 0, "agentic_resolution_outcomes": "{}", "agentic_provider_version": "", "unsupported_activities": 1, @@ -98,7 +101,7 @@ def test_insert_sql_stamps_run_metadata_and_escapes(): "migration_status": "included", "coverage_pct": 0.0, "deterministic_coverage_pct": 0.0, - "runnable_coverage_pct": 0.0, + "code_attached_coverage_pct": 0.0, "finding_count": 0, "finding_fingerprints": "[]", "complexity_score": 3, @@ -269,6 +272,6 @@ def test_write_results_evolves_an_existing_legacy_schema_before_insert(tmp_path: assert statements[2].startswith("ALTER TABLE cat.sch.tbl ADD COLUMNS") assert "audited_activities INT" in statements[2] assert "deterministic_coverage_pct DOUBLE" in statements[2] - assert "runnable_coverage_pct DOUBLE" in statements[2] + assert "code_attached_coverage_pct DOUBLE" in statements[2] assert "agentic_resolution_outcomes STRING" in statements[2] assert statements[3].startswith("INSERT INTO cat.sch.tbl") From a121b11e301de347412ecdbd43ab84d7443643bf Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Sun, 9 Aug 2026 14:57:34 -0700 Subject: [PATCH 62/77] cleanup --- README.md | 4 +- docs/content/docs/guide.mdx | 4 +- docs/content/docs/options.mdx | 13 +--- skills/flowx-convert/SKILL.md | 12 ++-- .../flowx-convert/sources/airflow-coverage.md | 9 +-- skills/flowx-convert/sources/airflow.md | 11 +--- skills/flowx-migrate/SKILL.md | 9 +-- skills/flowx-resolve-airflow-gaps/SKILL.md | 62 +++++-------------- .../references/contract-v1.md | 28 +++------ 9 files changed, 37 insertions(+), 115 deletions(-) diff --git a/README.md b/README.md index 3820152..412e7bd 100644 --- a/README.md +++ b/README.md @@ -166,9 +166,7 @@ execution) and maps ~35 operator/sensor families to the shared IR. Highlights: `params={...}` → job parameters, `>>` / `<<` / `set_upstream` / TaskGroup edges. Operators without a deterministic mapping become a failing placeholder and are recorded in -`gaps.json` for review. Eligible leaf gaps can use the fingerprint-bound resolver backed by the -pinned [`airflow-to-dabs` v0.2.1](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.1) -provider profile; flowx retains ownership of parsing, graph identity, policy, IR, and packaging. Full matrix: +`gaps.json` for review. Eligible leaf gaps can use the fingerprint-bound resolver backed by the pinned [`airflow-to-dabs` v0.2.1](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.1) provider profile; flowx retains ownership of parsing, graph identity, policy, IR, and packaging. Full matrix: [`skills/flowx-convert/sources/airflow-coverage.md`](skills/flowx-convert/sources/airflow-coverage.md). Airflow discovery independently audits DAG declarations, task candidates, dependency declarations, diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx index ffd7fb1..c1a7275 100644 --- a/docs/content/docs/guide.mdx +++ b/docs/content/docs/guide.mdx @@ -85,9 +85,7 @@ and `run_by` (`record-results`) — and install a published AI/BI coverage dashb For Airflow, `activities` is the independent source-audit count rather than the number of tasks the translator happened to emit. Reporting distinguishes deterministic, agentic, failed, and excluded -candidates and carries reconciliation status, translation-path coverage, deterministic coverage, -unresolved agentic outcomes, and mechanically validated code-attached coverage. Code attachment is -not a certification that provider-authored code is semantically correct. +candidates and carries reconciliation status, translation-path coverage, deterministic coverage, unresolved agentic outcomes, and mechanically validated code-attached coverage. Code attachment is not a certification that provider-authored code is semantically correct. diff --git a/docs/content/docs/options.mdx b/docs/content/docs/options.mdx index 6cd6397..84af359 100644 --- a/docs/content/docs/options.mdx +++ b/docs/content/docs/options.mdx @@ -115,20 +115,11 @@ phase surfaces three optional inputs — `results_table`, `results_warehouse_id` - **`record-results`** writes one row **per pipeline per run** to the supplied Unity Catalog table (`catalog.schema.table`), combining the complexity columns above with the audited/deterministic/agentic/failed/excluded coverage breakdown, reconciliation and migration - status, finding fingerprints, translation-path coverage, deterministic coverage, unresolved - agentic count, reviewed-resolution outcomes/provider version, and code-attached coverage. - The corresponding result columns are `resolved_agentic_count`, `unresolved_agentic_count`, and - `code_attached_coverage_pct`. - Airflow's audited count remains the denominator even for failed or excluded candidates. - Code-attached coverage counts deterministic tasks plus accepted `resolved` provider candidates; - it means the generated code passed mechanical contract validation, not that its semantics were - certified. Every row is stamped with a shared + status, finding fingerprints, translation-path coverage, deterministic coverage, unresolved agentic count, reviewed-resolution outcomes/provider version, and code-attached coverage. The corresponding result columns are `resolved_agentic_count`, `unresolved_agentic_count`, and `code_attached_coverage_pct`. Airflow's audited count remains the denominator even for failed or excluded candidates. Code-attached coverage counts deterministic tasks plus accepted `resolved` provider candidates; it means the generated code passed mechanical contract validation, not that its semantics were certified. Every row is stamped with a shared **`run_id`** (UUID), **`run_date`** (`CURRENT_TIMESTAMP()`), and **`run_by`** (`CURRENT_USER()`), so coverage is trackable across runs and users. - **`install-dashboard`** creates and publishes an AI/BI (Lakeview) dashboard over that table — - KPI counters (pipelines, audited activities, and mechanically validated code-attached coverage), - failed/excluded totals, a pipelines-by-complexity bar chart, a code-attached-coverage trend, and a - per-pipeline table that retains translation-path and deterministic coverage. + KPI counters (pipelines, audited activities, and mechanically validated code-attached coverage), failed/excluded totals, a pipelines-by-complexity bar chart, a code-attached-coverage trend, and a per-pipeline table that retains translation-path and deterministic coverage. The SQL warehouse is auto-detected (preferring a running serverless warehouse) when `results_warehouse_id` is left blank. Both run via the Databricks SDK and degrade gracefully diff --git a/skills/flowx-convert/SKILL.md b/skills/flowx-convert/SKILL.md index a9d5546..10155d0 100644 --- a/skills/flowx-convert/SKILL.md +++ b/skills/flowx-convert/SKILL.md @@ -20,8 +20,7 @@ phase 2 of the flowx migration workflow; it produces a transient translation rep Translation is **source-specific** (ADF activity translators vs. Airflow operator mapping), so this skill routes to the right source guide. The shared mechanics — how to run the phase, the report -contract, and the `inspect`/`modify` machinery — live here. The legacy `merge_agentic` command is -ADF-only; Airflow uses `resolve-agentic prepare|stage|apply` instead. +contract, and the `inspect`/`modify` machinery — live here. The legacy `merge_agentic` command is ADF-only; Airflow uses `resolve-agentic prepare|stage|apply` instead. ## Step 1 — Identify the source (required) @@ -36,8 +35,7 @@ There is no default source. Every phase invocation passes `--source ` expl ## Step 2 — Follow the source guide Read the matching `sources/.md` and follow it. ADF has a rich deterministic-first + -agentic-gap flow with just-in-time configuration. Airflow converts deterministically first and may -then use the separately reviewed, fingerprint-bound `flowx-resolve-airflow-gaps` workflow. +agentic-gap flow with just-in-time configuration. Airflow converts deterministically first and may then use the separately reviewed, fingerprint-bound `flowx-resolve-airflow-gaps` workflow. ## How to run this phase — MCP tool or venv CLI @@ -66,15 +64,13 @@ across ADF and Airflow. ## Shared adapter commands -`inspect` and `modify` operate on the report rather than raw source definitions. The ADF guide uses -them heavily; Airflow currently needs only the base conversion: +`inspect` and `modify` operate on the report rather than raw source definitions. The ADF guide uses them heavily; Airflow currently needs only the base conversion: - `inspect ` — emit the full just-in-time option schema (each option annotated with a `show_when` condition). Walk it locally; ask an option only when its `show_when` is satisfied. - `modify --output-dir --answer OPTION_ID=VALUE ...` — validate and apply collected answers, writing `.work/translation_report.stamped.json` + `metadata/configuration.json`. -- `merge_agentic --report --agentic-results ` — **ADF only**. Fold agent-produced - per-activity translations into an ADF report. Airflow's legacy name-based merge is disabled. +- `merge_agentic --report --agentic-results ` — **ADF only**. Fold agent-produced per-activity translations into an ADF report. Airflow's legacy name-based merge is disabled. ## Output artifacts (shared, transient under `/.work/`) diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md index c5c0909..013a9db 100644 --- a/skills/flowx-convert/sources/airflow-coverage.md +++ b/skills/flowx-convert/sources/airflow-coverage.md @@ -40,17 +40,12 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't | Multiple DAGs | Every DAG, including multiple declarations and repeated static `@dag` factory invocations in one Python file, becomes a sibling job in one shared Airflow bundle so `TriggerDagRunOperator` resource references resolve. Narrow classic factories shaped as one DAG declaration followed by `return dag` are expanded with statically bindable arguments. | Any operator not listed becomes a `PlaceholderActivity` **and** a `gaps.json` entry carrying the -operator's raw source for review. The legacy `merge_agentic` command is disabled for Airflow; -eligible one-task leaf gaps may use the fingerprint-bound `flowx-resolve-airflow-gaps` workflow. The +operator's raw source for review. The legacy `merge_agentic` command is disabled for Airflow; eligible one-task leaf gaps may use the fingerprint-bound `flowx-resolve-airflow-gaps` workflow. The safe fallback is a flagged, failing task rather than a silent omission. Callables that read Airflow task context (`**context` / `ti`) or XCom, and runtime-branching decorators, take the same route rather than emitting code that fails at runtime. -The resolver consumes the pinned `airflow-to-dabs` v0.2.1 Flowx provider profile. It receives one -flowx-produced gap envelope and cannot express graph or task-policy changes. Accepted `resolved` -candidates contribute to mechanically validated code-attached coverage, but remain agentic and do -not increase deterministic coverage. `needs_input`, `deferred`, and unreviewed candidates remain -linked failing placeholders. +The resolver consumes the pinned `airflow-to-dabs` v0.2.1 Flowx provider profile. It receives one flowx-produced gap envelope and cannot express graph or task-policy changes. Accepted `resolved` candidates contribute to mechanically validated code-attached coverage, but remain agentic and do not increase deterministic coverage. `needs_input`, `deferred`, and unreviewed candidates remain linked failing placeholders. ## Not yet supported diff --git a/skills/flowx-convert/sources/airflow.md b/skills/flowx-convert/sources/airflow.md index c47a93d..9ed05b3 100644 --- a/skills/flowx-convert/sources/airflow.md +++ b/skills/flowx-convert/sources/airflow.md @@ -3,11 +3,9 @@ Source guide for `--source airflow`. Translate parsed Airflow DAGs into Databricks IR. See the parent `SKILL.md` for how to run the phase and the report contract. -Airflow translation is **deterministic-first**. The static parse maps ~35 operator/sensor families -directly to IR (Tier 1-3). Operators with no deterministic mapping become +Airflow translation is **deterministic-first**. The static parse maps ~35 operator/sensor families directly to IR (Tier 1-3). Operators with no deterministic mapping become `PlaceholderActivity` tasks and are recorded in `gaps.json` with their raw source for review. The -placeholder remains a deliberate runtime failure until it is resolved manually or through the -fingerprint-bound `flowx-resolve-airflow-gaps` workflow. +placeholder remains a deliberate runtime failure until it is resolved manually or through the fingerprint-bound `flowx-resolve-airflow-gaps` workflow. **Before converting, check [`sources/airflow-coverage.md`](airflow-coverage.md)** — the verified support matrix (classic operators, TaskFlow, sensors, TaskGroups, dbt factory) and the constructs @@ -42,10 +40,7 @@ PythonOperator callable or BashOperator command, carrying `generated_source`) or If convert wrote `/.work/gaps.json`, each entry describes an unmapped construct whose generated Job task points to a notebook that raises `NotImplementedError`. Review every gap before -deployment. The shared `merge_agentic` command is disabled for Airflow and rejects -`--source airflow`; keep the placeholder, exclude the DAG, or invoke the -`flowx-resolve-airflow-gaps` skill. That workflow binds one leaf resolution to the finding -fingerprint and revalidates graph and policy invariants before package. +deployment. The shared `merge_agentic` command is disabled for Airflow and rejects `--source airflow`; keep the placeholder, exclude the DAG, or invoke the `flowx-resolve-airflow-gaps` skill. That workflow binds one leaf resolution to the finding fingerprint and revalidates graph and policy invariants before package. ## Step 4 — Proceed to package diff --git a/skills/flowx-migrate/SKILL.md b/skills/flowx-migrate/SKILL.md index a8b5593..e12b05c 100644 --- a/skills/flowx-migrate/SKILL.md +++ b/skills/flowx-migrate/SKILL.md @@ -96,9 +96,7 @@ To accept all defaults and skip the prompts, pass `"interactive": false`. (Re-ca For step-by-step control, run the commands in order (the app reuses `output_dir` across calls, so only `discover` needs the source input). `source` ("adf" | "airflow") is required for -discover/convert and for `inputs discover`/`inputs convert`; for Airflow, swap `adf_definitions` for -`airflow_source_path`. `merge_agentic` is ADF-only. `package` and `inputs package` are -source-independent: +discover/convert and for `inputs discover`/`inputs convert`; for Airflow, swap `adf_definitions` for `airflow_source_path`. `merge_agentic` is ADF-only. `package` and `inputs package` are source-independent: ``` flowx(command="inputs", parameters={"phase": "discover", "source": "adf"}) # source req for discover/convert @@ -111,10 +109,7 @@ flowx(command="package", parameters={"output_dir": ..., "catalog": ..., "schema" flowx(command="record_results", parameters={...}) / flowx(command="install_dashboard", parameters={...}) ``` -For an Airflow report with eligible leaf placeholders, use the `flowx-resolve-airflow-gaps` skill -between convert and package. It calls `resolve_agentic` with `action="prepare"`, stages one or more -provider candidates, and applies only the gap fingerprints the user explicitly accepts. Package -must then receive `/.work/translation_report.agentic.json` as `report_path`. +For an Airflow report with eligible leaf placeholders, use the `flowx-resolve-airflow-gaps` skill between convert and package. It calls `resolve_agentic` with `action="prepare"`, stages one or more provider candidates, and applies only the gap fingerprints the user explicitly accepts. Package must then receive `/.work/translation_report.agentic.json` as `report_path`. The server's `output_dir` is ephemeral and not reachable from your workspace, so **have `migrate`/ `package` write the DAB to the target via the SDK** — pass `"output_volume_path": "/Volumes/…"` or diff --git a/skills/flowx-resolve-airflow-gaps/SKILL.md b/skills/flowx-resolve-airflow-gaps/SKILL.md index 9955532..7d617c1 100644 --- a/skills/flowx-resolve-airflow-gaps/SKILL.md +++ b/skills/flowx-resolve-airflow-gaps/SKILL.md @@ -1,20 +1,13 @@ --- name: flowx-resolve-airflow-gaps -description: > - Resolve source-reconciled Airflow leaf gaps through the fingerprint-bound flowx contract. Use - after Airflow conversion emits PlaceholderActivity tasks and before packaging the reviewed report. +description: Resolve source-reconciled Airflow leaf gaps through the fingerprint-bound flowx contract. Use after Airflow conversion emits PlaceholderActivity tasks and before packaging the reviewed report. --- # Resolve Airflow Leaf Gaps -Use this workflow only for Airflow reports whose deterministic conversion succeeded with gaps. -Flowx owns source parsing, task identity, dependencies, task policy, IR, and packaging. This skill -reasons about one prepared gap at a time using the migration knowledge from -[`park-peter/airflow-to-dabs` v0.2.1](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.1). -It must not parse the DAG independently or generate a second bundle. +Use this workflow only for Airflow reports whose deterministic conversion succeeded with gaps. Flowx owns source parsing, task identity, dependencies, task policy, IR, and packaging. This skill reasons about one prepared gap at a time using the migration knowledge from [`park-peter/airflow-to-dabs` v0.2.1](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.1). It must not parse the DAG independently or generate a second bundle. -Read [`references/contract-v1.md`](references/contract-v1.md) and the pinned -[`airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md`](references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md) before authoring a resolution. The profile and every referenced knowledge file are vendored from the exact upstream tag and commit under `references/airflow-to-dabs-v0.2.1/`. If required knowledge is unavailable, return `needs_input` or `deferred`; never infer missing operator semantics. +Read [`references/contract-v1.md`](references/contract-v1.md) and the pinned [`airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md`](references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md) before authoring a resolution. The profile and every referenced knowledge file are vendored from the exact upstream tag and commit under `references/airflow-to-dabs-v0.2.1/`. If required knowledge is unavailable, return `needs_input` or `deferred`; never infer missing operator semantics. ## 1. Prepare immutable gap envelopes @@ -26,12 +19,9 @@ Read [`references/contract-v1.md`](references/contract-v1.md) and the pinned --output-dir ``` -Preparation reparses the source, proves that it reproduces the deterministic report, and writes an -immutable baseline, source snapshot, manifest, and `GapEnvelope v1` objects under -`/.work/agentic/`. If the source or report no longer agrees, rerun convert first. +Preparation reparses the source, proves that it reproduces the deterministic report, and writes an immutable baseline, source snapshot, manifest, and `GapEnvelope v1` objects under `/.work/agentic/`. If the source or report no longer agrees, rerun convert first. -With MCP, call `flowx(command="resolve_agentic", parameters={"action": "prepare", "source": -"airflow", "airflow_source_path": ..., "report_path": ..., "output_dir": ...})`. +With MCP, call `flowx(command="resolve_agentic", parameters={"action": "prepare", "source": "airflow", "airflow_source_path": ..., "report_path": ..., "output_dir": ...})`. ## 2. Produce one candidate per gap @@ -41,19 +31,11 @@ Read the prepared envelope rather than reopening or reparsing the DAG. Return on - `needs_input`: a concrete question or prerequisite blocks a safe migration. - `deferred`: the gap is outside the leaf-only contract and remains a linked failing placeholder. -`KubernetesPodOperator` commonly returns `needs_input` when the image, secrets, storage, networking, -or compute assumptions cannot be preserved from the envelope alone. Do not present it as the default -successful example. +`KubernetesPodOperator` commonly returns `needs_input` when the image, secrets, storage, networking, or compute assumptions cannot be preserved from the envelope alone. Do not present it as the default successful example. -Every source argument must appear exactly once in `argument_disposition` as `consumed`, -`preserved_by_flowx`, `ignored`, or `needs_input`. Every disposition needs a rationale; an ignored argument must -state the specific semantic loss. Never include task names, task keys, dependencies, retries, -timeouts, clusters, schedules, or other graph/policy fields in the replacement. +Every source argument must appear exactly once in `argument_disposition` as `consumed`, `preserved_by_flowx`, `ignored`, or `needs_input`. Every disposition needs a rationale; an ignored argument must state the specific semantic loss. Never include task names, task keys, dependencies, retries, timeouts, clusters, schedules, or other graph/policy fields in the replacement. -Generated code must be self-contained, contain no Airflow import statements, and contain no -template expressions. Python notebooks must start with `# Databricks notebook source`. Put -Databricks dynamic references in replacement parameters and read them through notebook widgets or -SQL named parameters. Comments may mention Airflow for provenance. +Generated code must be self-contained, contain no Airflow import statements, and contain no template expressions. Python notebooks must start with `# Databricks notebook source`. Put Databricks dynamic references in replacement parameters and read them through notebook widgets or SQL named parameters. Comments may mention Airflow for provenance. ## 3. Stage candidates @@ -64,18 +46,13 @@ SQL named parameters. Comments may mention Airflow for provenance. --candidate [--candidate ...] ``` -Stage validates fingerprints, source/report hashes, the pinned provider version, argument -disposition, generated-file hashes, Python imports, templates, and the constrained replacement -schema. Tampering after staging is a hard failure. Identical content is idempotent; use `--replace` -to replace different content for an already-staged gap. Stage returns an immutable, hash-addressed -review-manifest path for the complete staged candidate set. +Stage validates fingerprints, source/report hashes, the pinned provider version, argument disposition, generated-file hashes, Python imports, templates, and the constrained replacement schema. Tampering after staging is a hard failure. Identical content is idempotent; use `--replace` to replace different content for an already-staged gap. Stage returns an immutable, hash-addressed review-manifest path for the complete staged candidate set. MCP accepts candidate objects inline with `action="stage"` and `candidates=[...]`. ## 4. Review and explicitly apply -Show the user each candidate's code, prerequisites, warnings, semantic deltas, ignored arguments, -provider version, and model provenance. Apply only the fingerprints the user accepts: +Show the user each candidate's code, prerequisites, warnings, semantic deltas, ignored arguments, provider version, and model provenance. Apply only the fingerprints the user accepts: ```bash "$PY" -m flowx.adapter resolve-agentic apply \ @@ -84,12 +61,7 @@ provider version, and model provenance. Apply only the fingerprints the user acc --accept-gap [--accept-gap ...] ``` -`--accept-all` is only for replaying candidates already staged in a prior step; never combine it -with live candidate generation. It requires `--review-manifest ` and rejects the operation if -the reviewed candidate IDs or hashes no longer exactly match the staged set. Apply always rebuilds from the immutable deterministic baseline, -then proves task count, location, keys, dependencies, policy, and enclosing control flow are -unchanged. It writes `.work/translation_report.agentic.json` and keeps accepted evidence under -`metadata/agentic/` so package pruning does not destroy provenance. +`--accept-all` is only for replaying candidates already staged in a prior step; never combine it with live candidate generation. It requires `--review-manifest ` and rejects the operation if the reviewed candidate IDs or hashes no longer exactly match the staged set. Apply always rebuilds from the immutable deterministic baseline, then proves task count, location, keys, dependencies, policy, and enclosing control flow are unchanged. It writes `.work/translation_report.agentic.json` and keeps accepted evidence under `metadata/agentic/` so package pruning does not destroy provenance. To decline every staged candidate after reviewing that exact set, use: @@ -101,14 +73,9 @@ To decline every staged candidate after reviewing that exact set, use: --review-manifest ``` -This records the staged candidates as declined. Prepared gaps without a staged candidate remain -unreviewed; the flag makes no claim about artifacts that did not exist. +This records the staged candidates as declined. Prepared gaps without a staged candidate remain unreviewed; the flag makes no claim about artifacts that did not exist. -Use a reduced `--accept-gap` allowlist to reject selected candidates while retaining others. Use -`--reset` to discard all accepted resolutions and start over from the deterministic baseline. A -normal apply after a source edit is a hard failure: rerun convert and prepare instead of applying -stale results. Reset is the recovery path and restores the durable baseline even after source drift -or normal `.work/` pruning. +Use a reduced `--accept-gap` allowlist to reject selected candidates while retaining others. Use `--reset` to discard all accepted resolutions and start over from the deterministic baseline. A normal apply after a source edit is a hard failure: rerun convert and prepare instead of applying stale results. Reset is the recovery path and restores the durable baseline even after source drift or normal `.work/` pruning. Package the reviewed report explicitly: @@ -118,5 +85,4 @@ Package the reviewed report explicitly: --output-dir ``` -Package replays the kept baseline and accepted candidates before writing bundle files. Missing, -modified, or inconsistent evidence fails preflight. +Package replays the kept baseline and accepted candidates before writing bundle files. Missing, modified, or inconsistent evidence fails preflight. diff --git a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md index 09e78f2..ed86112 100644 --- a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md +++ b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md @@ -1,11 +1,8 @@ # Airflow Agentic Gap Contract v1 -The provider receives a `GapEnvelope` produced by flowx. It does not receive authority to alter the -captured graph. +The provider receives a `GapEnvelope` produced by flowx. It does not receive authority to alter the captured graph. -`capture_identity` is flowx's source-capture identity and may differ from the collision-safe -Databricks `task_key`. `task_path` identifies the exact placeholder location, including a nested -`for_each` body; providers must copy neither field into the replacement payload. +`capture_identity` is flowx's source-capture identity and may differ from the collision-safe Databricks `task_key`. `task_path` identifies the exact placeholder location, including a nested `for_each` body; providers must copy neither field into the replacement payload. ## Resolution shape @@ -48,28 +45,19 @@ Databricks `task_key`. `task_path` identifies the exact placeholder location, in } ``` -SQL uses `{"kind": "sql", "file": "task.sql", "parameters": {}}` and a single generated file -whose language is `sql`. +SQL uses `{"kind": "sql", "file": "task.sql", "parameters": {}}` and a single generated file whose language is `sql`. -Spark Python uses `{"kind": "spark_python", "file": "task.py", "parameters": ["--arg", "value"]}` -and a single generated Python file. It is emitted as a Databricks `spark_python_task`. +Spark Python uses `{"kind": "spark_python", "file": "task.py", "parameters": ["--arg", "value"]}` and a single generated Python file. It is emitted as a Databricks `spark_python_task`. -`needs_input` and `deferred` omit `replacement` and `generated_files` and add a non-empty `reason`. -They are terminal reviewed outcomes: the linked `NotImplementedError` placeholder remains and no -automatic retry occurs. +`needs_input` and `deferred` omit `replacement` and `generated_files` and add a non-empty `reason`. They are terminal reviewed outcomes: the linked `NotImplementedError` placeholder remains and no automatic retry occurs. ## Hard boundaries - Only `notebook`, `sql`, and `spark_python` leaf replacements are allowed in v1. -- The replacement cannot express `name`, `task_key`, `depends_on`, retries, timeouts, compute, - libraries, schedules, or control-flow fields. +- The replacement cannot express `name`, `task_key`, `depends_on`, retries, timeouts, compute, libraries, schedules, or control-flow fields. - Generated file paths are relative and cannot contain `..`. - Every generated file is inline and hash-bound; external workspace paths are not accepted. -- Python payloads may mention Airflow in comments but may not contain `import airflow` or - `from airflow ...` statements. Notebook payloads must start with `# Databricks notebook source`; - Spark Python scripts are ordinary valid Python files. -- Airflow Jinja is rejected. Databricks dynamic references such as `{{job.parameters.x}}`, - `{{tasks.upstream.values.x}}`, and `{{input}}` are valid in replacement parameter values, not in - uploaded notebook or SQL source files; source files must read widgets or SQL named parameters. +- Python payloads may mention Airflow in comments but may not contain `import airflow` or `from airflow ...` statements. Notebook payloads must start with `# Databricks notebook source`; Spark Python scripts are ordinary valid Python files. +- Airflow Jinja is rejected. Databricks dynamic references such as `{{job.parameters.x}}`, `{{tasks.upstream.values.x}}`, and `{{input}}` are valid in replacement parameter values, not in uploaded notebook or SQL source files; source files must read widgets or SQL named parameters. - Every source argument in the envelope has exactly one disposition and a non-empty rationale. - The provider identity must match the pinned `airflow-to-dabs` v0.2.1 knowledge release. From 7d393155048ec980050a4ec93679bc301b2bc762 Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 10 Aug 2026 12:57:55 -0700 Subject: [PATCH 63/77] Gate CI on the integration suite CI ran unit tests only, so integration regressions reached reviewers. Run make integration in the build job, deselect the live-Azure suite that needs az login plus factory access, and xfail the six pre-existing ADF translation gaps non-strictly so they report XPASS once fixed. --- .github/workflows/push.yml | 1 + Makefile | 5 ++++- README.md | 11 ++++++----- tests/integration/test_end_to_end.py | 10 ++++++++++ tests/integration/test_golden_output.py | 10 ++++++++++ 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 39cfd50..7cd1702 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -19,6 +19,7 @@ jobs: run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock - run: uv sync --frozen --extra mcp - run: make test + - run: make integration - name: Verify requirements.txt is in sync with the lockfile run: | make requirements diff --git a/Makefile b/Makefile index a99a435..e76581c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean dev ci test integration fmt help docs-install docs-clean docs-build docs-serve lock-dependencies requirements precommit +.PHONY: clean dev ci test integration integration-live fmt help docs-install docs-clean docs-build docs-serve lock-dependencies requirements precommit clean: rm -rf .venv .pytest_cache .ruff_cache .mypy_cache __pycache__ @@ -15,6 +15,9 @@ test: PYTHONPATH=src uv run pytest tests/unit -v integration: + PYTHONPATH=src uv run pytest tests/integration -v -m "not slow and not integration" + +integration-live: PYTHONPATH=src uv run pytest tests/integration -v -m "not slow" fmt: diff --git a/README.md b/README.md index 412e7bd..8c47dab 100644 --- a/README.md +++ b/README.md @@ -227,11 +227,12 @@ for deployment (SDK notebook or CLI script) and Genie Code registration. ## Development ```bash -make dev # Install dependencies (uses uv) -make test # Run unit tests -make integration # Run integration tests -make fmt # Format + lint (ruff + mypy) -make clean # Remove build artifacts +make dev # Install dependencies (uses uv) +make test # Run unit tests +make integration # Run integration tests (excludes the live-Azure suite; gates CI) +make integration-live # Also run tests needing live ADF access (az login + factory access) +make fmt # Format + lint (ruff + mypy) +make clean # Remove build artifacts ``` ### Prerequisites diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py index 1945974..e6035c7 100644 --- a/tests/integration/test_end_to_end.py +++ b/tests/integration/test_end_to_end.py @@ -24,6 +24,13 @@ from flowx.sources.adf.loader import build_inventory from flowx.sources.adf.translate import translate_pipeline +# ADF Copy translation gaps that predate the Airflow source work. Non-strict so the integration suite can +# gate CI, and so each test reports XPASS rather than failing once the ADF fix lands. +adf_translation_gap = pytest.mark.xfail( + reason="pre-existing ADF translation gap, tracked separately from the Airflow source", + strict=False, +) + # --------------------------------------------------------------------------- # TestTranslateAllPipelines — simulates "translate all pipelines" # --------------------------------------------------------------------------- @@ -89,6 +96,7 @@ def test_bundle_all_pipelines(self, adf_definitions, tmp_path): class TestTranslateSpecificPipeline: """Tests simulating 'translate a specific pipeline' prompt.""" + @adf_translation_gap def test_translate_copy_csv_pipeline(self, adf_definitions, pipeline_by_name): """Copy CSV to Delta pipeline translates correctly.""" pipeline = pipeline_by_name("pipeline_copy_csv_to_delta") @@ -171,6 +179,7 @@ def test_translate_mixed_agentic_pipeline(self, adf_definitions, pipeline_by_nam class TestActivityTypeTranslation: """Tests for specific activity type translation accuracy.""" + @adf_translation_gap def test_copy_activity_source_sink(self, adf_definitions, pipeline_by_name): """Copy activity preserves source/sink properties.""" pipeline = pipeline_by_name("pipeline_copy_csv_to_delta") @@ -236,6 +245,7 @@ def test_databricks_yml_structure(self, adf_definitions, tmp_path): assert "targets" in content assert set(content["targets"].keys()) == {"dev", "staging", "prod"} + @adf_translation_gap def test_job_yaml_task_keys_match_activities(self, adf_definitions, pipeline_by_name, tmp_path): """Job YAML has unique task keys matching the pipeline activities.""" pipeline = pipeline_by_name("pipeline_all_activity_types") diff --git a/tests/integration/test_golden_output.py b/tests/integration/test_golden_output.py index 9c35cf3..9330f1c 100644 --- a/tests/integration/test_golden_output.py +++ b/tests/integration/test_golden_output.py @@ -23,6 +23,13 @@ from flowx.sources.adf.loader import load_adf_definitions from flowx.sources.adf.translate import translate_pipeline +# ADF SetVariable translation gaps that predate the Airflow source work. Non-strict so the integration +# suite can gate CI, and so each test reports XPASS rather than failing once the ADF fix lands. +adf_translation_gap = pytest.mark.xfail( + reason="pre-existing ADF translation gap, tracked separately from the Airflow source", + strict=False, +) + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -334,6 +341,7 @@ def test_condition_chain_unique_keys(self, bundle_dirs): class TestSetVariableCoverage: """pl_test_setvariable_coverage: literal, notebook_code, dab_ref kinds.""" + @adf_translation_gap def test_translates_all_five(self, translated_pipelines): report = translated_pipelines.get("pl_test_setvariable_coverage") if report is None: @@ -341,6 +349,7 @@ def test_translates_all_five(self, translated_pipelines): svs = [t for t in report.pipeline.tasks if isinstance(t, SetVariableActivity)] assert len(svs) == 5 + @adf_translation_gap def test_utcnow_uses_notebook_code(self, translated_pipelines): report = translated_pipelines.get("pl_test_setvariable_coverage") if report is None: @@ -356,6 +365,7 @@ def test_utcnow_uses_notebook_code(self, translated_pipelines): assert utcnow_sv is not None, "Expected SetVariable for runTimestamp" assert utcnow_sv.value_kind == "notebook_code" + @adf_translation_gap def test_pipeline_param_uses_dab_ref(self, translated_pipelines): report = translated_pipelines.get("pl_test_setvariable_coverage") if report is None: From f64b1fb8091de6449b983d1cfcb5028459086b5a Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Mon, 10 Aug 2026 12:58:17 -0700 Subject: [PATCH 64/77] Recognize Airflow 3 authoring syntax and harden the gap contract Detect Airflow 3 (airflow.sdk, providers.standard) and strong 1.10 imports statically. Lower Asset/Dataset schedules to ANY_UPDATED/ALL_UPDATED table triggers when each asset declares a Databricks table, and fail closed on AssetOrTimeSchedule, mixed boolean expressions, unmapped assets, and ambiguous 1.10 schedule defaults. Route native async @task callables to linked leaf gaps. Reject dynamic Airflow imports with literal module names or executed source, and reserve flowx and Databricks task-schema notebook parameter keys. --- .../flowx-convert/sources/airflow-coverage.md | 7 +- skills/flowx-resolve-airflow-gaps/SKILL.md | 2 +- .../references/contract-v1.md | 3 +- src/flowx/agentic.py | 146 ++++++++++- src/flowx/sources/airflow/audit.py | 15 +- src/flowx/sources/airflow/loader.py | 241 +++++++++++++++++- tests/unit/test_airflow_agentic_resolution.py | 66 ++++- .../test_airflow_version_compatibility.py | 196 ++++++++++++++ 8 files changed, 646 insertions(+), 30 deletions(-) create mode 100644 tests/unit/test_airflow_version_compatibility.py diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md index 013a9db..91eb89c 100644 --- a/skills/flowx-convert/sources/airflow-coverage.md +++ b/skills/flowx-convert/sources/airflow-coverage.md @@ -11,6 +11,7 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't | Construct | Result | | --- | --- | +| Airflow authoring versions | Airflow 3 `airflow.sdk` and `airflow.providers.standard` imports, modern Airflow 2 imports, and strong Airflow 1.10 legacy operator/sensor imports are resolved statically without importing Airflow. | | `PythonOperator` (classic) | Notebook task; callable `def` preserved, transitive helpers/constants/non-Airflow imports carried, `op_args`/`op_kwargs` passed as JSON widgets, return value via `dbutils.jobs.taskValues.set`. | | `PythonVirtualenvOperator` / `ExternalPythonOperator` | Notebook task with a `%pip install` cell for `requirements`. | | `BranchPythonOperator` / `ShortCircuitOperator` | Failing placeholder + review gap (runtime branch selection can't be lowered statically). | @@ -21,7 +22,7 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't | `TriggerDagRunOperator` | `run_job_task` referencing the target DAG by sanitized job name. | | `EmailOperator` | Placeholder recommending job-level email notifications. | | dbt CLI operators (`DbtRun/Test/Seed/Snapshot/Build/Deps`) and Cosmos `DbtDag` / `DbtTaskGroup` | Single `DbtFactoryActivity`, **static explosion** (default) or **PyDABs** (`--dbt-mode pydabs`); see [dbt factory](#dbt-factory-mode). | -| **TaskFlow API** (`@dag`, `@task`, `@task.virtualenv`) | Canonical, aliased, and qualified Airflow decorators are resolved statically. Each `@task` invocation → a task; implicit XCom data flow (`transform(extract())`) → a notebook that reads upstream return values via `dbutils.jobs.taskValues.get`, calls the function, and publishes its own. `@task.branch` / `@task.short_circuit`, or a callable reading task context/XCom, route to a placeholder + gap. | +| **TaskFlow API** (`@dag`, `@task`, `@task.virtualenv`) | Canonical, aliased, and qualified Airflow decorators are resolved statically. Each synchronous `@task` invocation → a task; implicit XCom data flow (`transform(extract())`) → a notebook that reads upstream return values via `dbutils.jobs.taskValues.get`, calls the function, and publishes its own. Native async `@task` callables, `@task.branch` / `@task.short_circuit`, or a callable reading task context/XCom route to a linked placeholder + agentic leaf gap. | | File sensors (`S3KeySensor`, `GCSObjectExistenceSensor`, `FileSensor`, `HdfsSensor`, `WebHdfsSensor`) | With no schedule, a root sensor whose descendants cover every non-sensor task → `file_arrival` trigger; otherwise a `dbutils.fs` polling notebook task. | | Table/SQL sensors (`DatabricksPartitionSensor`, `DatabricksSqlSensor`, `DatabricksSQLStatementsSensor`, `SqlSensor`) | With no schedule, a root literal-table sensor whose descendants cover every non-sensor task → `table_update` trigger; otherwise a `spark.sql` polling notebook task. | | `ExternalTaskSensor` | Placeholder explaining logical-run-aware migration options; polling the latest Databricks job run is not equivalent to Airflow's matching logical run. | @@ -33,7 +34,7 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't | Dependencies | `>>` / `<<` chains (incl. list/tuple fan-out and inline TaskFlow calls) and `set_upstream` / `set_downstream`. | | **TaskGroups** (context-manager `with TaskGroup(...)`) | Static nesting → task-key namespacing (`group__subgroup__task`); group-level edges (`group_a >> group_b`, `task >> group`) expand to leaf→root edges between member tasks. | | **`@task_group`** (decorator form) | Placeholder + gap with dependency edges preserved; a decorator group is a sub-pipeline flowx doesn't lower deterministically. | -| Schedule | Cron `schedule_interval` → Quartz (Unix DOW 0–6 → Quartz 1–7); exact sub-hour `timedelta` → Quartz, longer intervals → periodic, `@continuous` → continuous mode. | +| Schedule | Cron `schedule_interval` → Quartz (Unix DOW 0–6 → Quartz 1–7); exact sub-hour `timedelta` → Quartz, longer intervals → periodic, `@continuous` → continuous mode. Airflow 3 Asset/Dataset lists and uniform `&` / `|` expressions map to `ALL_UPDATED` / `ANY_UPDATED` table triggers when each asset declares `extra={"databricks_table": "catalog.schema.table"}` or an `x-databricks-table:` URI. | | `trigger_rule` | Exact supported rules map to `run_if`; `none_failed_min_one_success` maps to `NONE_FAILED` with the all-skipped delta recorded. Rules without an equivalent become linked placeholders. | | Job parameters | `params={...}` / `Param(default=...)` → job parameters with defaults; `{{ params.x }}` / `{{ var.value.x }}` / `{{ dag_run.conf['x'] }}` → `{{job.parameters.x}}`. | | `Variable.get` in a callable | Rewritten to `dbutils.widgets.get`; a callable using an Airflow `Connection` object routes to a placeholder because one secret string cannot preserve the object API. | @@ -63,6 +64,8 @@ decisions. - **Dynamic DAG factories** — classic DAG factories outside the documented single-declaration shape, non-literal factory arguments, and non-literal `dag_id` overrides fail reconciliation and block package output rather than emitting a filename-derived empty Job. +- **Unresolved Airflow 3 schedules** — `AssetOrTimeSchedule`, mixed Asset boolean expressions, custom timetables, and Assets without explicit Databricks table metadata become `AirflowSourceSemantics` gaps. Job-level trigger and schedule changes are outside the leaf-only agentic contract. +- **Ambiguous Airflow 1.10 schedule defaults** — assigned DAGs and legacy imports are supported, but a DAG using strong 1.10 syntax that omits `schedule_interval` becomes an `AirflowSourceSemantics` gap because historical default schedule and catchup behavior cannot be inferred safely from source alone. - **Sensors beyond the mapped families** (`S3PrefixSensor`, custom sensors, etc.) → placeholder + gap. A file sensor with a non-literal path, or a table/SQL sensor with no literal `sql` / `table_name`, also falls back to a placeholder. diff --git a/skills/flowx-resolve-airflow-gaps/SKILL.md b/skills/flowx-resolve-airflow-gaps/SKILL.md index 7d617c1..538db74 100644 --- a/skills/flowx-resolve-airflow-gaps/SKILL.md +++ b/skills/flowx-resolve-airflow-gaps/SKILL.md @@ -35,7 +35,7 @@ Read the prepared envelope rather than reopening or reparsing the DAG. Return on Every source argument must appear exactly once in `argument_disposition` as `consumed`, `preserved_by_flowx`, `ignored`, or `needs_input`. Every disposition needs a rationale; an ignored argument must state the specific semantic loss. Never include task names, task keys, dependencies, retries, timeouts, clusters, schedules, or other graph/policy fields in the replacement. -Generated code must be self-contained, contain no Airflow import statements, and contain no template expressions. Python notebooks must start with `# Databricks notebook source`. Put Databricks dynamic references in replacement parameters and read them through notebook widgets or SQL named parameters. Comments may mention Airflow for provenance. +Generated code must be self-contained, must not import Airflow through import statements or literal dynamic imports, and must contain no template expressions. Python notebooks must start with `# Databricks notebook source`. Put Databricks dynamic references in replacement parameters and read them through notebook widgets or SQL named parameters. Notebook parameter keys must avoid Flowx and Databricks task-schema namespaces. Comments, docstrings, and inert strings may mention Airflow for provenance; static validation is runtime hygiene, not a Python security sandbox. ## 3. Stage candidates diff --git a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md index ed86112..9cc9b06 100644 --- a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md +++ b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md @@ -57,7 +57,8 @@ Spark Python uses `{"kind": "spark_python", "file": "task.py", "parameters": ["- - The replacement cannot express `name`, `task_key`, `depends_on`, retries, timeouts, compute, libraries, schedules, or control-flow fields. - Generated file paths are relative and cannot contain `..`. - Every generated file is inline and hash-bound; external workspace paths are not accepted. -- Python payloads may mention Airflow in comments but may not contain `import airflow` or `from airflow ...` statements. Notebook payloads must start with `# Databricks notebook source`; Spark Python scripts are ordinary valid Python files. +- Python payloads may mention Airflow in comments, docstrings, and other inert string literals but may not import Airflow through import statements or statically identifiable dynamic imports with literal module names or executed source. This validation enforces runtime compatibility and hygiene; it is not a Python security sandbox, and every accepted payload remains reviewed arbitrary Python. Notebook payloads must start with `# Databricks notebook source`; Spark Python scripts are ordinary valid Python files. +- Notebook `base_parameters` keys must use letters, digits, underscores, dots, and hyphens, starting with a letter or underscore. Flowx-owned names beginning with `__flowx` and Databricks task identity, graph, policy, and task-type field names are reserved case-insensitively. - Airflow Jinja is rejected. Databricks dynamic references such as `{{job.parameters.x}}`, `{{tasks.upstream.values.x}}`, and `{{input}}` are valid in replacement parameter values, not in uploaded notebook or SQL source files; source files must read widgets or SQL named parameters. - Every source argument in the envelope has exactly one disposition and a non-empty rationale. - The provider identity must match the pinned `airflow-to-dabs` v0.2.1 knowledge release. diff --git a/src/flowx/agentic.py b/src/flowx/agentic.py index 39a4773..ad62d4b 100644 --- a/src/flowx/agentic.py +++ b/src/flowx/agentic.py @@ -53,6 +53,31 @@ _NESTED_TASK_FIELDS = ("inner_activities", "if_true_activities", "if_false_activities", "default_activities") _AIRFLOW_TEMPLATE = re.compile(r"{{\s*([^{}]+?)\s*}}|{%\s*([^{}]+?)\s*%}") _DAB_TEMPLATE_PREFIXES = ("job.", "tasks.", "input.", "backfill.") +_NOTEBOOK_PARAMETER_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_.-]*\Z") +_RESERVED_NOTEBOOK_PARAMETER_KEYS = frozenset( + { + *_COMMON_TASK_FIELDS, + "condition_task", + "dbt_task", + "disable_auto_optimization", + "email_notifications", + "environment_key", + "for_each_task", + "job_cluster_key", + "new_cluster", + "notebook_task", + "notification_settings", + "pipeline_task", + "python_wheel_task", + "retry_on_timeout", + "run_if", + "run_job_task", + "spark_jar_task", + "spark_python_task", + "sql_task", + "webhook_notifications", + } +) _UNSET = object() @@ -1047,6 +1072,8 @@ def _validate_replacement(candidate: dict[str, Any], gap: dict[str, Any]) -> Non isinstance(key, str) and isinstance(val, str) for key, val in parameters.items() ): raise AgenticContractError(f"Replacement {parameters_field} must be a string-to-string object") + if kind == "notebook": + _validate_notebook_parameter_keys(parameters) files = candidate.get("generated_files") if not isinstance(files, list) or len(files) != 1 or not isinstance(files[0], dict): raise AgenticContractError("Resolved v1 candidate requires exactly one inline generated file") @@ -1077,15 +1104,118 @@ def _validate_replacement(candidate: dict[str, Any], gap: dict[str, Any]) -> Non module = ast.parse(content) except SyntaxError as error: raise AgenticContractError(f"Generated notebook is not valid Python: {error}") from error - for node in ast.walk(module): - if isinstance(node, ast.Import) and any( - alias.name == "airflow" or alias.name.startswith("airflow.") for alias in node.names - ): - raise AgenticContractError("Generated notebook must not import Airflow") - if isinstance(node, ast.ImportFrom) and ( - node.module == "airflow" or str(node.module).startswith("airflow.") + _reject_airflow_imports(module) + + +def _validate_notebook_parameter_keys(parameters: dict[str, str]) -> None: + for key in parameters: + if not _NOTEBOOK_PARAMETER_KEY.fullmatch(key): + raise AgenticContractError(f"Unsafe notebook base_parameters key: {key!r}") + normalized = key.casefold() + if normalized.startswith("__flowx") or normalized in _RESERVED_NOTEBOOK_PARAMETER_KEYS: + raise AgenticContractError(f"Reserved notebook base_parameters key: {key!r}") + + +def _reject_airflow_imports(module: ast.AST) -> None: + pending = [module] + while pending: + tree = pending.pop() + nodes = list(ast.walk(tree)) + importlib_names = {"importlib"} + import_module_names: set[str] = set() + builtins_names = {"builtins"} + builtin_import_names = {"__import__"} + execution_names = {"exec", "eval"} + + for node in nodes: + if isinstance(node, ast.Import): + for alias in node.names: + if _is_airflow_module(alias.name): + raise AgenticContractError("Generated notebook must not import Airflow") + if alias.name == "importlib": + importlib_names.add(alias.asname or alias.name) + elif alias.name == "builtins": + builtins_names.add(alias.asname or alias.name) + elif isinstance(node, ast.ImportFrom): + if _is_airflow_module(node.module): + raise AgenticContractError("Generated notebook must not import Airflow") + for alias in node.names: + bound_name = alias.asname or alias.name + if node.module == "importlib" and alias.name == "import_module": + import_module_names.add(bound_name) + elif node.module == "builtins" and alias.name == "__import__": + builtin_import_names.add(bound_name) + elif node.module == "builtins" and alias.name in {"exec", "eval"}: + execution_names.add(bound_name) + + for node in nodes: + if not isinstance(node, ast.Call): + continue + target = node.func + literal = _literal_call_argument(node) + if literal is None: + continue + if _is_module_import_call( + target, + importlib_names, + import_module_names, + builtins_names, + builtin_import_names, ): - raise AgenticContractError("Generated notebook must not import Airflow") + if _is_airflow_module(literal): + raise AgenticContractError("Generated notebook must not import Airflow") + continue + if _is_execution_call(target, execution_names, builtins_names): + try: + pending.append(ast.parse(literal, mode="exec")) + except SyntaxError: + continue + + +def _is_airflow_module(value: str | None) -> bool: + return value == "airflow" or bool(value and value.startswith("airflow.")) + + +def _literal_call_argument(node: ast.Call) -> str | None: + value: ast.AST | None = node.args[0] if node.args else None + if value is None: + for keyword in node.keywords: + if keyword.arg in {"name", "source", "object"}: + value = keyword.value + break + if isinstance(value, ast.Constant) and isinstance(value.value, str): + return value.value + return None + + +def _is_module_import_call( + target: ast.expr, + importlib_names: set[str], + import_module_names: set[str], + builtins_names: set[str], + builtin_import_names: set[str], +) -> bool: + if isinstance(target, ast.Name): + return target.id in import_module_names or target.id in builtin_import_names + return ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and ( + (target.attr == "import_module" and target.value.id in importlib_names) + or (target.attr == "__import__" and target.value.id in builtins_names) + ) + ) + + +def _is_execution_call(target: ast.expr, execution_names: set[str], builtins_names: set[str]) -> bool: + if isinstance(target, ast.Name): + return target.id in execution_names + return ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id in builtins_names + and target.attr in {"exec", "eval"} + ) def _apply_to_baseline(baseline: dict[str, Any], resolutions: list[StagedResolution]) -> dict[str, Any]: diff --git a/src/flowx/sources/airflow/audit.py b/src/flowx/sources/airflow/audit.py index de05bb6..3d906c0 100644 --- a/src/flowx/sources/airflow/audit.py +++ b/src/flowx/sources/airflow/audit.py @@ -95,7 +95,8 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None) -> No self.taskflow_defs = { node.name for node in ast.walk(module) - if isinstance(node, ast.FunctionDef) and _has_decorator(node, _TASK_DECORATORS, self.aliases) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and _has_decorator(node, _TASK_DECORATORS, self.aliases) } self.dag_defs = { node.name @@ -134,6 +135,12 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> None: if not isinstance(statement, ast.FunctionDef): self.visit(statement) + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + """Treats decorated async callables as TaskFlow definitions, not DAG-body statements.""" + if node.name in self.taskflow_defs: + return + self.generic_visit(node) + def visit_With(self, node: ast.With) -> None: for item in node.items: if isinstance(item.context_expr, ast.Call) and _leaf(item.context_expr.func, self.aliases) == "DAG": @@ -403,7 +410,11 @@ def _decorator_name(node: ast.expr, aliases: dict[str, str]) -> str: return canonical -def _has_decorator(function: ast.FunctionDef, names: set[str], aliases: dict[str, str]) -> bool: +def _has_decorator( + function: ast.FunctionDef | ast.AsyncFunctionDef, + names: set[str], + aliases: dict[str, str], +) -> bool: return any(_decorator_name(decorator, aliases) in names for decorator in function.decorator_list) diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py index 702bb9f..f35e7a6 100644 --- a/src/flowx/sources/airflow/loader.py +++ b/src/flowx/sources/airflow/loader.py @@ -63,6 +63,7 @@ class _TaskFlowTask: task_id: str def_name: str decorator: str + is_async: bool = False positional_deps: dict[int, str] = field(default_factory=dict) keyword_deps: dict[str, str] = field(default_factory=dict) positional_values: dict[int, str] = field(default_factory=dict) @@ -373,6 +374,123 @@ def _construct_name(node: ast.expr, aliases: dict[str, str]) -> str: return canonical.rsplit(".", 1)[-1] if canonical else "" +def _airflow_generation(module: ast.Module) -> str: + """Infers version-specific authoring syntax only when imports are unambiguous.""" + imported_modules: list[str] = [] + for statement in module.body: + if isinstance(statement, ast.Import): + imported_modules.extend(item.name for item in statement.names) + elif isinstance(statement, ast.ImportFrom) and statement.module: + imported_modules.append(statement.module) + if any(name == "airflow.sdk" or name.startswith("airflow.sdk.") for name in imported_modules) or any( + name == "airflow.providers.standard" or name.startswith("airflow.providers.standard.") + for name in imported_modules + ): + return "3" + legacy_module = re.compile(r"^airflow\.(?:operators|sensors)\.[^.]+_(?:operator|sensor)$") + if any(name.startswith("airflow.contrib.") or legacy_module.fullmatch(name) for name in imported_modules): + return "1.10" + return "unknown" + + +def _asset_definitions(module: ast.Module, aliases: dict[str, str]) -> dict[str, ast.Call]: + """Returns module-level Asset/Dataset objects that a DAG schedule may reference.""" + definitions: dict[str, ast.Call] = {} + for statement in module.body: + if not ( + isinstance(statement, ast.Assign) + and len(statement.targets) == 1 + and isinstance(statement.targets[0], ast.Name) + and isinstance(statement.value, ast.Call) + ): + continue + if _construct_name(statement.value.func, aliases) in {"Asset", "Dataset"}: + definitions[statement.targets[0].id] = statement.value + return definitions + + +def _asset_table_name(call: ast.Call) -> str | None: + kwargs = {keyword.arg: keyword.value for keyword in call.keywords if keyword.arg} + extra = ops.literal_value(kwargs.get("extra")) + if isinstance(extra, dict): + table_name = extra.get("databricks_table") + if isinstance(table_name, str) and table_name.strip(): + return table_name.strip() + uri = ops.literal_str(call.args[0]) if call.args else ops.literal_str(kwargs.get("uri")) + prefix = "x-databricks-table:" + if uri and uri.startswith(prefix): + table_name = uri[len(prefix) :].lstrip("/").strip() + return table_name or None + return None + + +def _asset_expression( + node: ast.expr, + aliases: dict[str, str], + definitions: dict[str, ast.Call], +) -> tuple[list[str], str, str | None] | None: + if isinstance(node, ast.Name): + definition = definitions.get(node.id) + return _asset_expression(definition, aliases, definitions) if definition is not None else None + if isinstance(node, ast.Call) and _construct_name(node.func, aliases) in {"Asset", "Dataset"}: + table_name = _asset_table_name(node) + return ([table_name], "leaf", None) if table_name else ([], "leaf", "unresolved_asset_schedule") + if isinstance(node, (ast.List, ast.Tuple)): + children = [_asset_expression(item, aliases, definitions) for item in node.elts] + if not children or any(child is None for child in children): + return None + resolved = [child for child in children if child is not None] + error = next((child[2] for child in resolved if child[2] is not None), None) + if error: + return [], "all", error + if any(child[1] == "any" for child in resolved): + return [], "all", "unsupported_asset_schedule_expression" + return [table for child in resolved for table in child[0]], "all", None + if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.BitAnd, ast.BitOr)): + left = _asset_expression(node.left, aliases, definitions) + right = _asset_expression(node.right, aliases, definitions) + if left is None or right is None: + return None + error = left[2] or right[2] + mode = "all" if isinstance(node.op, ast.BitAnd) else "any" + if error: + return [], mode, error + if any(child_mode not in {"leaf", mode} for child_mode in (left[1], right[1])): + return [], mode, "unsupported_asset_schedule_expression" + return [*left[0], *right[0]], mode, None + return None + + +def _asset_schedule_from_node( + node: ast.expr, + aliases: dict[str, str], + definitions: dict[str, ast.Call], +) -> tuple[dict[str, object] | None, str | None]: + if any( + isinstance(candidate, ast.Call) and _construct_name(candidate.func, aliases) == "AssetOrTimeSchedule" + for candidate in ast.walk(node) + ): + return None, "unsupported_asset_or_time_schedule" + expression = _asset_expression(node, aliases, definitions) + if expression is None: + return None, "unsupported_dag_schedule" + table_names, mode, error = expression + if error: + return None, error + table_names = list(dict.fromkeys(table_names)) + if not table_names: + return None, "unresolved_asset_schedule" + return ( + { + "kind": "table_update", + "table_names": table_names, + "condition": "ANY_UPDATED" if mode in {"leaf", "any"} else "ALL_UPDATED", + "pause_status": "UNPAUSED", + }, + None, + ) + + def _safe_static_value(node: ast.expr, constants: dict[str, Any]) -> Any: """Evaluates the small literal expression subset used by static DAG factories.""" if isinstance(node, ast.Constant): @@ -563,6 +681,8 @@ class _DagVisitor(ast.NodeVisitor): def __init__(self, module: ast.Module, *, target_dag_variable: str | None = None) -> None: self._aliases = _import_aliases(module) + self.airflow_generation = _airflow_generation(module) + self.asset_definitions = _asset_definitions(module, self._aliases) self._target_dag_variable = target_dag_variable # Classic python_callable resolution starts at module scope. Nested functions are only visible # from their lexical parent and must never overwrite a same-named module function. @@ -617,9 +737,9 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None = None self.partial_mapped: set[str] = set() # Disambiguates synthetic vars for operators instantiated without an assignment. self._bare_operator_counter = 0 - # TaskFlow: function name -> (FunctionDef, decorator dotted-name) for @task-decorated defs. + # TaskFlow: function name -> (definition, decorator dotted-name) for @task-decorated defs. # Pre-scanned so a @task def defined after the @dag body that uses it is still resolved. - self.taskflow_defs: dict[str, tuple[ast.FunctionDef, str]] = {} + self.taskflow_defs: dict[str, tuple[ast.FunctionDef | ast.AsyncFunctionDef, str]] = {} # @task_group def names -- a group is a sub-pipeline, not a single renderable task, so an # invocation routes to a placeholder + gap rather than being expanded here. self.taskgroup_defs: set[str] = set() @@ -646,7 +766,11 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None = None self.is_taskflow_dag: bool = False def functions(self) -> dict[str, ast.FunctionDef]: - taskflow = {name: definition for name, (definition, _decorator) in self.taskflow_defs.items()} + taskflow = { + name: definition + for name, (definition, _decorator) in self.taskflow_defs.items() + if isinstance(definition, ast.FunctionDef) + } return {**self._functions, **taskflow} def functions_for(self, task_var: str) -> dict[str, ast.FunctionDef]: @@ -700,6 +824,11 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> None: self._dag_scope_depth -= 1 self._scope_stack.pop() + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + """Claims native async TaskFlow definitions without treating their bodies as DAG structure.""" + if self._dag_scope_depth: + self._claimed_statement_ids.add(id(node)) + def visit_Assign(self, node: ast.Assign) -> None: if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name) and isinstance(node.value, ast.Call): var = node.targets[0].id @@ -961,8 +1090,13 @@ def _register_taskflow_call(self, call: ast.Call, var: str) -> bool: def_name, mapped, override_id = self._taskflow_def_name(call) if def_name is None: return False - _fn, decorator = self.taskflow_defs[def_name] - task = _TaskFlowTask(task_id=override_id or var, def_name=def_name, decorator=decorator) + function, decorator = self.taskflow_defs[def_name] + task = _TaskFlowTask( + task_id=override_id or var, + def_name=def_name, + decorator=decorator, + is_async=isinstance(function, ast.AsyncFunctionDef), + ) self.taskflow_tasks[var] = task self.calls[var] = call self.capture_source_nodes[var] = call @@ -1219,6 +1353,8 @@ def _read_dag_kwargs(self, call: ast.Call) -> None: kwargs = {kw.arg: _bind_constants(kw.value, self._constants) for kw in call.keywords if kw.arg} self.dag_id = ops.literal_str(kwargs.get("dag_id")) self._apply_dag_kwargs(kwargs) + if self.airflow_generation == "1.10" and not {"schedule", "schedule_interval"} & kwargs.keys(): + self.unresolved_constructs.append(("ambiguous_airflow_1_10_default_schedule", call)) def _apply_dag_kwargs(self, kwargs: dict[str, ast.expr]) -> None: self.captured_dag_settings.update(kwargs) @@ -1512,15 +1648,15 @@ def _mapping_chain_args(node: ast.expr) -> list[ast.expr]: return args -def _iter_functions(module: ast.Module) -> list[ast.FunctionDef]: - """All FunctionDefs in *module*, including those nested inside a ``@dag`` function body. +def _iter_functions(module: ast.Module) -> list[ast.FunctionDef | ast.AsyncFunctionDef]: + """All function definitions, including those nested inside a ``@dag`` function body. TaskFlow ``@task`` defs are often nested inside the ``@dag`` function, so a top-level-only scan - would miss them. Async defs are skipped (flowx renders sync notebooks). + would miss them. Async definitions are captured so they can become explicit agentic leaf gaps. """ - found: list[ast.FunctionDef] = [] + found: list[ast.FunctionDef | ast.AsyncFunctionDef] = [] for node in ast.walk(module): - if isinstance(node, ast.FunctionDef): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): found.append(node) return found @@ -1628,7 +1764,7 @@ def _decorator_kwargs( def _has_decorator( - func: ast.FunctionDef, + func: ast.FunctionDef | ast.AsyncFunctionDef, names: frozenset[str], aliases: dict[str, str] | None = None, ) -> bool: @@ -2125,6 +2261,30 @@ def _task_key(var: str, task_id: str) -> str: # mid-DAG (an ordering gate, not the DAG's entry condition), the sensor is retained as a polling # task instead of being silently dropped. schedule = _schedule_from_interval(visitor.schedule_interval, node=visitor.schedule_node, timezone=visitor.timezone) + schedule_proof: dict[str, Any] | None = None + schedule_node = visitor.schedule_node + explicit_none_schedule = isinstance(schedule_node, ast.Constant) and schedule_node.value is None + if schedule is None and schedule_node is not None and not explicit_none_schedule: + schedule, schedule_gap = _asset_schedule_from_node(schedule_node, visitor._aliases, visitor.asset_definitions) + if schedule is not None: + schedule_span = _span(schedule_node) + table_names = schedule["table_names"] + condition = schedule["condition"] + assert isinstance(table_names, list) + assert isinstance(condition, str) + schedule_proof = { + "code": "asset_schedule_lowered", + "table_names": list(table_names), + "condition": condition, + "source_span": { + "line": schedule_span.line, + "column": schedule_span.column, + "end_line": schedule_span.end_line, + "end_column": schedule_span.end_column, + }, + } + elif schedule_gap is not None: + visitor.unresolved_constructs.append((schedule_gap, schedule_node)) has_schedule = schedule is not None # Dummy/Empty operators are structural and can be removed after dependency rewiring. @@ -2432,6 +2592,32 @@ def append_task(activity: Activity, capture_id: str) -> None: dep_keys = {var_to_task_key[u] for u in upstreams.get(var, []) if u in var_to_task_key} dep_keys.discard(task_key) depends_on = [Dependency(task_key=k) for k in sorted(dep_keys)] or None + definition, _decorator = visitor.taskflow_defs[tf.def_name] + if tf.is_async: + mapping_call = visitor.calls.get(var) + raw_definition = { + "operator": "@task.async.expand" if var in visitor.mapped else "@task.async", + "source": ast.get_source_segment(source, definition) or "", + "invocation": ast.get_source_segment(source, visitor.capture_source_nodes[var]) or "", + } + if mapping_call is not None and var in visitor.mapped: + raw_definition["mapping"] = ast.get_source_segment(source, mapping_call) or "" + activity = PlaceholderActivity( + name=tf.task_id, + task_key=task_key, + original_type="@task.async.expand" if var in visitor.mapped else "@task.async", + comment=( + f"Native async TaskFlow callable {tf.def_name!r} requires an async-aware Databricks " + "implementation; resolve this captured leaf without changing its graph identity." + ), + raw_definition=raw_definition, + ) + activity.depends_on = depends_on + if var in visitor.mapped and tf.expand_items_json is not None: + append_task(_wrap_taskflow_in_for_each(activity, tf, task_key, depends_on), var) + else: + append_task(activity, var) + continue if var in visitor.mapped and tf.expand_items_json is None: # .expand over a non-literal iterable (e.g. an upstream task's output) can't be lowered to # a static for_each inputs array -- route to the agentic-gap round instead of silently @@ -2535,6 +2721,7 @@ def append_task(activity: Activity, capture_id: str) -> None: dbt_vars=dbt_vars, semantic_findings=semantic_findings, sensor_lift_proof=sensor_lift_proof, + schedule_proof=schedule_proof, argument_proofs=argument_proofs, expected_ir_edges=expected_ir_edges, placeholder_capture_ids=placeholder_capture_ids, @@ -2619,6 +2806,7 @@ def _reconcile_pipeline( dbt_vars: list[str], semantic_findings: list[dict[str, Any]], sensor_lift_proof: dict[str, Any] | None, + schedule_proof: dict[str, Any] | None, argument_proofs: list[dict[str, Any]], expected_ir_edges: set[tuple[str, str]], placeholder_capture_ids: dict[int, str], @@ -2645,6 +2833,8 @@ def _reconcile_pipeline( ) if sensor_lift_proof is not None: transformations.append(sensor_lift_proof) + if schedule_proof is not None: + transformations.append(schedule_proof) captured_task_count = len(visitor.operators) + len(visitor.taskflow_tasks) + len(visitor.taskgroup_calls) unresolved = list(audit.unresolved) @@ -2944,13 +3134,38 @@ def source_reference(capture_id: str) -> str: ) ) + unresolved_messages = { + "unresolved_asset_schedule": ( + "An Airflow Asset/Dataset schedule lacks an explicit Databricks table mapping. Add " + "extra={'databricks_table': '..
'} or use an " + "x-databricks-table: URI." + ), + "unsupported_asset_or_time_schedule": ( + "Airflow AssetOrTimeSchedule combines time and asset triggers, but a Databricks Job can " + "use only one job-level trigger." + ), + "unsupported_asset_schedule_expression": ( + "The Airflow Asset/Dataset boolean expression cannot be represented by one Databricks " + "ANY_UPDATED or ALL_UPDATED table trigger." + ), + "unsupported_dag_schedule": ( + "The Airflow DAG schedule or timetable has no proven static Databricks Jobs mapping." + ), + "ambiguous_airflow_1_10_default_schedule": ( + "This DAG uses strong Airflow 1.10 syntax and omits schedule_interval. Historical default " + "schedule and catchup behavior cannot be inferred safely without the deployed Airflow version." + ), + } for candidate in unresolved: findings.append( source_audit.finding( source_file=source_file, code=candidate.code, severity="gap", - message="Dynamic Airflow control flow could not be expanded safely by the static parser.", + message=unresolved_messages.get( + candidate.code, + "Dynamic Airflow control flow could not be expanded safely by the static parser.", + ), candidate=candidate, ) ) @@ -3163,7 +3378,7 @@ def _wrap_in_for_each( def _wrap_taskflow_in_for_each( - activity: NotebookActivity, + activity: Activity, tf: _TaskFlowTask, task_key: str, depends_on: list[Dependency] | None, diff --git a/tests/unit/test_airflow_agentic_resolution.py b/tests/unit/test_airflow_agentic_resolution.py index ec98d1e..a0e72ec 100644 --- a/tests/unit/test_airflow_agentic_resolution.py +++ b/tests/unit/test_airflow_agentic_resolution.py @@ -555,15 +555,75 @@ def test_stage_rejects_graph_identity_fields(tmp_path: Path, capsys): assert not list((output / ".work" / "agentic" / "candidates").glob("*.json")) -def test_stage_rejects_airflow_import_but_allows_airflow_in_comments(tmp_path: Path, capsys): +@pytest.mark.parametrize( + "source", + [ + "import importlib\nimportlib.import_module('airflow')\n", + "import importlib as imports\nimports.import_module('airflow.providers.cncf.kubernetes')\n", + "from importlib import import_module\nimport_module('airflow')\n", + "from importlib import import_module as load_module\nload_module('airflow.models')\n", + "__import__('airflow')\n", + "import builtins\nbuiltins.__import__('airflow.providers.amazon')\n", + "from builtins import __import__ as load_module\nload_module('airflow')\n", + "exec('import airflow')\n", + "eval(\"__import__('airflow.providers.google')\")\n", + ], +) +def test_stage_rejects_literal_dynamic_airflow_imports(tmp_path: Path, capsys, source: str) -> None: + _, output, gaps = _prepare(tmp_path) + + assert _stage(output, _candidate(gaps[0], source=source)) == 1 + assert "must not import Airflow" in capsys.readouterr().err + + +def test_stage_rejects_airflow_import_but_allows_airflow_in_nonexecuted_text(tmp_path: Path, capsys) -> None: _, output, gaps = _prepare(tmp_path) bad = _candidate(gaps[0], source="# Airflow provenance\nfrom airflow import DAG\n") assert _stage(output, bad) == 1 assert "must not import Airflow" in capsys.readouterr().err - good = _candidate(gaps[0], source="# Migrated from Airflow\nprint('ok')\n") - assert _stage(output, good) == 0 + for source in ( + "# import airflow documents migration provenance\nprint('ok')\n", + '"""The source DAG used import airflow."""\nprint(\'ok\')\n', + "message = 'import airflow'\nprint(message)\n", + "import importlib\nimportlib.import_module('airflowish')\n", + ): + assert _stage(output, _candidate(gaps[0], source=source), replace=True) == 0 + + +@pytest.mark.parametrize( + "key", + [ + "task_key", + "depends_on", + "__flowx_op_args", + "__FLOWX_custom", + "", + " ", + "line\nbreak", + "{{job.parameters.env}}", + ], +) +def test_stage_rejects_unsafe_notebook_parameter_keys(tmp_path: Path, capsys, key: str) -> None: + _, output, gaps = _prepare(tmp_path) + candidate = _candidate(gaps[0]) + candidate["replacement"]["base_parameters"] = {key: "value"} + + assert _stage(output, candidate) == 1 + assert "notebook base_parameters key" in capsys.readouterr().err + + +def test_stage_accepts_safe_notebook_parameter_keys(tmp_path: Path) -> None: + _, output, gaps = _prepare(tmp_path) + candidate = _candidate(gaps[0]) + candidate["replacement"]["base_parameters"] = { + "env": "dev", + "input-path": "/Volumes/input", + "config.env": "prod", + } + + assert _stage(output, candidate) == 0 def test_stage_requires_complete_argument_disposition_and_ignored_rationale(tmp_path: Path, capsys): diff --git a/tests/unit/test_airflow_version_compatibility.py b/tests/unit/test_airflow_version_compatibility.py new file mode 100644 index 0000000..48c0ce5 --- /dev/null +++ b/tests/unit/test_airflow_version_compatibility.py @@ -0,0 +1,196 @@ +"""Version-specific Airflow source compatibility and fail-closed behavior.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from flowx.models.ir import ForEachActivity, NotebookActivity, PlaceholderActivity +from flowx.sources.airflow.loader import load_airflow_dag + + +def _load(tmp_path: Path, source: str): + dag = tmp_path / "dag.py" + dag.write_text(source, encoding="utf-8") + return load_airflow_dag(dag) + + +def test_airflow_3_sdk_and_standard_provider_paths_translate_deterministically(tmp_path: Path) -> None: + pipeline = _load( + tmp_path, + "from airflow.sdk import DAG, task\n" + "from airflow.providers.standard.operators.bash import BashOperator\n" + "@task\n" + "def extract():\n" + " return 1\n" + "with DAG(dag_id='airflow_3', schedule='@daily', catchup=False) as dag:\n" + " start = BashOperator(task_id='start', bash_command='echo start')\n" + " result = extract()\n" + " start >> result\n", + ) + + assert pipeline.reconciliation_status == "verified" + assert [task.task_key for task in pipeline.tasks] == ["start", "result"] + assert all(isinstance(task, NotebookActivity) for task in pipeline.tasks) + assert (pipeline.schedule or {})["quartz_cron_expression"] == "0 0 0 * * ?" + + +def test_airflow_3_omitted_schedule_preserves_manual_only_default(tmp_path: Path) -> None: + pipeline = _load( + tmp_path, + "from airflow.sdk import dag\n" + "from airflow.providers.standard.operators.bash import BashOperator\n" + "@dag(dag_id='manual_airflow_3')\n" + "def build():\n" + " BashOperator(task_id='work', bash_command='echo work')\n" + "build()\n", + ) + + assert pipeline.reconciliation_status == "verified" + assert pipeline.schedule is None + assert pipeline.tags == {"source": "airflow", "dag_id": "manual_airflow_3"} + + +@pytest.mark.parametrize( + ("schedule", "condition"), + [ + ("[orders, customers]", "ALL_UPDATED"), + ("orders & customers", "ALL_UPDATED"), + ("orders | customers", "ANY_UPDATED"), + ], +) +def test_airflow_3_asset_schedules_with_uc_metadata_become_table_triggers( + tmp_path: Path, + schedule: str, + condition: str, +) -> None: + pipeline = _load( + tmp_path, + "from airflow.sdk import DAG, Asset\n" + "from airflow.providers.standard.operators.bash import BashOperator\n" + "orders = Asset('orders', extra={'databricks_table': 'main.raw.orders'})\n" + "customers = Asset('customers', extra={'databricks_table': 'main.raw.customers'})\n" + f"with DAG(dag_id='assets', schedule={schedule}) as dag:\n" + " BashOperator(task_id='work', bash_command='echo work')\n", + ) + + assert pipeline.reconciliation_status == "verified" + assert pipeline.schedule == { + "kind": "table_update", + "table_names": ["main.raw.orders", "main.raw.customers"], + "condition": condition, + "pause_status": "UNPAUSED", + } + assert any(item["code"] == "asset_schedule_lowered" for item in pipeline.audit["transformations"]) + + +@pytest.mark.parametrize( + ("schedule", "finding_code"), + [ + ("[Asset('s3://landing/orders')]", "unresolved_asset_schedule"), + ( + "AssetOrTimeSchedule(timetable=CronTriggerTimetable('0 0 * * *'), assets=[Asset('orders')])", + "unsupported_asset_or_time_schedule", + ), + ], +) +def test_airflow_3_unrepresentable_schedules_become_source_semantic_gaps( + tmp_path: Path, + schedule: str, + finding_code: str, +) -> None: + pipeline = _load( + tmp_path, + "from airflow.sdk import DAG, Asset\n" + "from airflow.timetables.assets import AssetOrTimeSchedule\n" + "from airflow.timetables.trigger import CronTriggerTimetable\n" + "from airflow.providers.standard.operators.bash import BashOperator\n" + f"with DAG(dag_id='asset_gap', schedule={schedule}) as dag:\n" + " BashOperator(task_id='work', bash_command='echo work')\n", + ) + + assert pipeline.reconciliation_status == "verified_with_gaps" + assert pipeline.schedule is None + assert pipeline.tasks[0].task_key == "__flowx_source_gaps" + assert any(item["code"] == finding_code for item in pipeline.not_translatable) + + +def test_airflow_3_async_taskflow_becomes_an_agentic_leaf_gap(tmp_path: Path) -> None: + pipeline = _load( + tmp_path, + "from airflow.sdk import dag, task\n" + "@task\n" + "async def fetch():\n" + " return 1\n" + "@dag(dag_id='async_task', schedule=None)\n" + "def build():\n" + " fetch()\n" + "build()\n", + ) + + assert pipeline.reconciliation_status == "verified_with_gaps" + assert len(pipeline.tasks) == 1 + task = pipeline.tasks[0] + assert isinstance(task, PlaceholderActivity) + assert task.original_type == "@task.async" + assert "async def fetch" in (task.raw_definition or {})["source"] + assert any(item["code"] == "operator_placeholder" for item in pipeline.not_translatable) + + +def test_airflow_3_mapped_async_taskflow_preserves_the_static_for_each(tmp_path: Path) -> None: + pipeline = _load( + tmp_path, + "from airflow.sdk import dag, task\n" + "@task\n" + "async def fetch(region):\n" + " return region\n" + "@dag(dag_id='mapped_async_task', schedule=None)\n" + "def build():\n" + " fetch.expand(region=['us-west-2', 'eu-west-1'])\n" + "build()\n", + ) + + assert pipeline.reconciliation_status == "verified_with_gaps" + assert len(pipeline.tasks) == 1 + mapped = pipeline.tasks[0] + assert isinstance(mapped, ForEachActivity) + assert [json.loads(item) for item in json.loads(mapped.items_expression)] == ["us-west-2", "eu-west-1"] + assert len(mapped.inner_activities) == 1 + task = mapped.inner_activities[0] + assert isinstance(task, PlaceholderActivity) + assert task.original_type == "@task.async.expand" + assert "fetch.expand" in (task.raw_definition or {})["mapping"] + + +def test_airflow_1_10_assigned_dag_and_legacy_imports_translate_deterministically(tmp_path: Path) -> None: + pipeline = _load( + tmp_path, + "from airflow import DAG\n" + "from airflow.operators.bash_operator import BashOperator\n" + "from airflow.operators.dummy_operator import DummyOperator\n" + "dag = DAG(dag_id='airflow_1_10', schedule_interval='@daily', catchup=False)\n" + "start = DummyOperator(task_id='start', dag=dag)\n" + "work = BashOperator(task_id='work', bash_command='echo work', dag=dag)\n" + "start >> work\n", + ) + + assert pipeline.reconciliation_status == "verified" + assert [task.task_key for task in pipeline.tasks] == ["work"] + assert (pipeline.schedule or {})["quartz_cron_expression"] == "0 0 0 * * ?" + + +def test_airflow_1_10_implicit_daily_schedule_fails_loudly(tmp_path: Path) -> None: + pipeline = _load( + tmp_path, + "from airflow import DAG\n" + "from airflow.operators.bash_operator import BashOperator\n" + "dag = DAG(dag_id='implicit_legacy_schedule')\n" + "work = BashOperator(task_id='work', bash_command='echo work', dag=dag)\n", + ) + + assert pipeline.reconciliation_status == "verified_with_gaps" + assert pipeline.schedule is None + assert pipeline.tasks[0].task_key == "__flowx_source_gaps" + assert any(item["code"] == "ambiguous_airflow_1_10_default_schedule" for item in pipeline.not_translatable) From 5016589b0de7f4550fabdd7ddf47f8b6fe4fb439 Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Tue, 11 Aug 2026 08:27:52 -0700 Subject: [PATCH 65/77] Harden Airflow template value semantics --- .../flowx-convert/sources/airflow-coverage.md | 7 +- src/flowx/bundler/prereqs_writer.py | 6 +- .../sources/airflow/callable_notebook.py | 82 ++++- src/flowx/sources/airflow/loader.py | 33 +- src/flowx/sources/airflow/templating.py | 337 +++++++++++++----- tests/unit/test_airflow_operators.py | 128 ++++++- .../unit/test_airflow_production_readiness.py | 5 +- tests/unit/test_airflow_templating.py | 134 ++++++- 8 files changed, 588 insertions(+), 144 deletions(-) diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md index 91eb89c..760eaf5 100644 --- a/skills/flowx-convert/sources/airflow-coverage.md +++ b/skills/flowx-convert/sources/airflow-coverage.md @@ -35,9 +35,9 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't | **TaskGroups** (context-manager `with TaskGroup(...)`) | Static nesting → task-key namespacing (`group__subgroup__task`); group-level edges (`group_a >> group_b`, `task >> group`) expand to leaf→root edges between member tasks. | | **`@task_group`** (decorator form) | Placeholder + gap with dependency edges preserved; a decorator group is a sub-pipeline flowx doesn't lower deterministically. | | Schedule | Cron `schedule_interval` → Quartz (Unix DOW 0–6 → Quartz 1–7); exact sub-hour `timedelta` → Quartz, longer intervals → periodic, `@continuous` → continuous mode. Airflow 3 Asset/Dataset lists and uniform `&` / `|` expressions map to `ALL_UPDATED` / `ANY_UPDATED` table triggers when each asset declares `extra={"databricks_table": "catalog.schema.table"}` or an `x-databricks-table:` URI. | -| `trigger_rule` | Exact supported rules map to `run_if`; `none_failed_min_one_success` maps to `NONE_FAILED` with the all-skipped delta recorded. Rules without an equivalent become linked placeholders. | -| Job parameters | `params={...}` / `Param(default=...)` → job parameters with defaults; `{{ params.x }}` / `{{ var.value.x }}` / `{{ dag_run.conf['x'] }}` → `{{job.parameters.x}}`. | -| `Variable.get` in a callable | Rewritten to `dbutils.widgets.get`; a callable using an Airflow `Connection` object routes to a placeholder because one secret string cannot preserve the object API. | +| `trigger_rule` | Exact supported rules map to `run_if`; `none_failed_min_one_success` and its legacy `none_failed_or_skipped` spelling map to `NONE_FAILED` with the all-skipped delta recorded. Rules without an equivalent become linked placeholders. | +| Job parameters | `params={...}` / `Param(default=...)` → job parameters with defaults. User `params.x` keeps the name `x`; logical-date macros, `var.value.x`, `dag_run.conf['x']`, and `run_id` use collision-free `__flowx_airflow_*` bindings. User parameter names beginning with `__flowx_` become explicit gaps. | +| `Variable.get` in a callable | `Variable.get('literal_name')` is rewritten to a collision-free `__flowx_airflow_variable_*` widget. Dynamic keys, Airflow defaults/deserialization options, other Airflow runtime imports, and Airflow `Connection` objects route to placeholders rather than emitting notebooks that require Airflow. | | Multiple DAGs | Every DAG, including multiple declarations and repeated static `@dag` factory invocations in one Python file, becomes a sibling job in one shared Airflow bundle so `TriggerDagRunOperator` resource references resolve. Narrow classic factories shaped as one DAG declaration followed by `return dag` are expanded with statically bindable arguments. | Any operator not listed becomes a `PlaceholderActivity` **and** a `gaps.json` entry carrying the @@ -66,6 +66,7 @@ decisions. package output rather than emitting a filename-derived empty Job. - **Unresolved Airflow 3 schedules** — `AssetOrTimeSchedule`, mixed Asset boolean expressions, custom timetables, and Assets without explicit Databricks table metadata become `AirflowSourceSemantics` gaps. Job-level trigger and schedule changes are outside the leaf-only agentic contract. - **Ambiguous Airflow 1.10 schedule defaults** — assigned DAGs and legacy imports are supported, but a DAG using strong 1.10 syntax that omits `schedule_interval` becomes an `AirflowSourceSemantics` gap because historical default schedule and catchup behavior cannot be inferred safely from source alone. +- **Unsafe inline template contexts** — SQL Jinja embedded in a string, quoted identifier, typed literal, or adjacent identifier fragment and shell Jinja in a non-expanding quoted heredoc, ANSI-C quote, or escaped position route to a placeholder instead of emitting a value with changed lexical semantics. - **Sensors beyond the mapped families** (`S3PrefixSensor`, custom sensors, etc.) → placeholder + gap. A file sensor with a non-literal path, or a table/SQL sensor with no literal `sql` / `table_name`, also falls back to a placeholder. diff --git a/src/flowx/bundler/prereqs_writer.py b/src/flowx/bundler/prereqs_writer.py index e79fc40..96d2132 100644 --- a/src/flowx/bundler/prereqs_writer.py +++ b/src/flowx/bundler/prereqs_writer.py @@ -135,7 +135,7 @@ class Prereqs: # manifest_path, note}). The user must `pip install databricks-dbt-factory` before deploy. pydabs_dbt_factories: list[dict[str, Any]] = field(default_factory=list) # Airflow catchup=True jobs; each entry is the SetupTask config dict ({pipeline}). History is - # replayed via a native Databricks backfill overriding the run_date parameter, not a DABs setting. + # replayed via a native Databricks backfill overriding the reserved Airflow date parameter. airflow_backfills: list[dict[str, Any]] = field(default_factory=list) def is_empty(self) -> bool: @@ -695,8 +695,8 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: "The DAG(s) below set `catchup=True`, so Airflow backfilled missed intervals. There is " "no equivalent DABs schedule setting. To replay history, run a " "[native Databricks backfill](https://docs.databricks.com/aws/en/jobs/backfill-jobs), which " - "overrides the `run_date` job parameter with `{{backfill.iso_date}}` per replayed window " - "(the run_date parameter is emitted for exactly this reason)." + "overrides the `__flowx_airflow_run_date` job parameter with `{{backfill.iso_date}}` per " + "replayed window (the parameter is emitted for exactly this reason)." ) lines.append("") for entry in sorted(prereqs.airflow_backfills, key=lambda config: config.get("pipeline", "")): diff --git a/src/flowx/sources/airflow/callable_notebook.py b/src/flowx/sources/airflow/callable_notebook.py index 764fc90..510cf0d 100644 --- a/src/flowx/sources/airflow/callable_notebook.py +++ b/src/flowx/sources/airflow/callable_notebook.py @@ -11,6 +11,7 @@ import ast import builtins +from collections.abc import Sequence from flowx.sources.airflow import templating @@ -166,7 +167,12 @@ def render_definitions(func: ast.FunctionDef, source: str, *, note: str) -> str: prelude = "\n".join(lines) + "\n" # Rewrite Variable.get / BaseHook.get_connection in the emitted definitions. - rewritten, _params, _notes = templating.rewrite_airflow_calls(prelude) + variable_binding = imports.get("Variable") + rewrite_variable = variable_binding is not None and variable_binding[1] in _AIRFLOW_IMPORT_ROOTS + rewritten, _params, _notes = templating.rewrite_airflow_calls( + prelude, + rewrite_variable=rewrite_variable, + ) return rewritten @@ -276,12 +282,86 @@ def airflow_runtime_reason(func: ast.FunctionDef, source: str) -> str | None: connections = sorted(templating.airflow_connection_names(closure_source)) if connections: return f"callable reads Airflow connection '{connections[0]}' as a Connection object" + import_reason = _airflow_runtime_import_reason( + module, + enclosing_statements, + closure_nodes, + ) + if import_reason is not None: + return import_reason unresolved_names = _unresolved_closure_names(func, source) if unresolved_names: return f"captures nonliteral closure '{unresolved_names[0]}'" return None +def _airflow_runtime_import_reason( + module: ast.Module, + enclosing_statements: list[ast.stmt], + closure_nodes: Sequence[ast.AST], +) -> str | None: + """Returns why an Airflow-bound name in emitted callable code cannot run without Airflow.""" + for closure_node in closure_nodes: + for imported in ast.walk(closure_node): + if isinstance(imported, ast.Import): + roots = {alias.name.split(".")[0] for alias in imported.names} + elif isinstance(imported, ast.ImportFrom): + roots = {(imported.module or "").split(".")[0]} + else: + continue + unavailable = sorted(roots & _AIRFLOW_IMPORT_ROOTS) + if unavailable: + return f"callable contains Airflow runtime import {unavailable[0]!r}" + imports = _import_bindings(module, enclosing_statements) + decorator_name_ids = { + id(name) + for closure_node in closure_nodes + for function in ast.walk(closure_node) + if isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef)) + for decorator in function.decorator_list + for name in ast.walk(decorator) + if isinstance(name, ast.Name) + } + loaded_names = [ + name + for closure_node in closure_nodes + for name in ast.walk(closure_node) + if isinstance(name, ast.Name) and isinstance(name.ctx, ast.Load) and id(name) not in decorator_name_ids + ] + airflow_names = { + name.id for name in loaded_names if name.id in imports and imports[name.id][1] in _AIRFLOW_IMPORT_ROOTS + } + if "Variable" in airflow_names: + supported_variable_ids: set[int] = set() + for closure_node in closure_nodes: + for call in (node for node in ast.walk(closure_node) if isinstance(node, ast.Call)): + function = call.func + if not ( + isinstance(function, ast.Attribute) + and function.attr == "get" + and isinstance(function.value, ast.Name) + and function.value.id == "Variable" + ): + continue + supported_variable_ids.add(id(function.value)) + literal_key = ( + call.args[0].value + if len(call.args) == 1 + and not call.keywords + and isinstance(call.args[0], ast.Constant) + and isinstance(call.args[0].value, str) + else None + ) + if literal_key is None or not literal_key.isascii() or not literal_key.isidentifier(): + return "callable calls Variable.get() with a dynamic key or Airflow-only options" + if any(name.id == "Variable" and id(name) not in supported_variable_ids for name in loaded_names): + return "callable uses Airflow Variable outside the supported Variable.get('literal_name') form" + airflow_names.remove("Variable") + if airflow_names: + return f"callable uses Airflow runtime import {sorted(airflow_names)[0]!r}" + return None + + def _unresolved_closure_names(func: ast.FunctionDef, source: str) -> list[str]: """Returns loaded names that the generated standalone definition cannot resolve.""" module = ast.parse(source) diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py index f35e7a6..091b83b 100644 --- a/src/flowx/sources/airflow/loader.py +++ b/src/flowx/sources/airflow/loader.py @@ -717,7 +717,7 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None = None self.schedule_node: ast.expr | None = None self.timezone: str | None = None # DAG catchup= flag: True means Airflow backfills missed intervals, which maps to a native - # Databricks backfill overriding the run_date parameter rather than any DABs schedule setting. + # Databricks backfill overriding the reserved Airflow date parameter. self.catchup: bool = False self.default_args: dict[str, ast.expr] = {} # DAG-level params={...} defaults (param name -> literal default), so emitted job parameters @@ -1378,6 +1378,9 @@ def _apply_dag_kwargs(self, kwargs: dict[str, ast.expr]) -> None: if isinstance(params, ast.Dict): for key, val in zip(params.keys, params.values): if isinstance(key, ast.Constant) and isinstance(key.value, str): + if key.value.startswith(templating.FLOWX_INTERNAL_PARAMETER_PREFIX): + self.unresolved_constructs.append(("reserved_airflow_parameter_name", key)) + continue self.dag_params[key.value] = _param_default(val) def visit_Expr(self, node: ast.Expr) -> None: @@ -2691,8 +2694,8 @@ def append_task(activity: Activity, capture_id: str) -> None: # Declare every job parameter -- those referenced in templates plus any from the DAG's # params={...} -- each with a default (Databricks requires one): the params={...} default when - # present; a logical-date parameter (run_date/execution_date/...) its schedule-aware time ref so a - # native backfill can override it per window; else an empty string so the bundle still validates. + # present; a reserved logical-date parameter its schedule-aware time ref so a native backfill can + # override it per window; else an empty string so the bundle still validates. param_names = referenced_params | set(visitor.dag_params) parameters = [ {"name": name, "default": _declared_param_default(name, visitor.dag_params, schedule)} @@ -2701,7 +2704,7 @@ def append_task(activity: Activity, capture_id: str) -> None: tags = {"source": "airflow", "dag_id": visitor.dag_id or ""} if visitor.catchup: # Airflow catchup=True has no DABs schedule setting; it maps to running a native Databricks - # backfill, which overrides the run_date job parameter with {{backfill.iso_date}} per window. + # backfill, which overrides the reserved logical-date parameter per replayed window. tags["airflow_catchup"] = "true" expected_ir_edges = {(dependency.task_key, task.task_key) for task in tasks for dependency in task.depends_on or []} pipeline = Pipeline( @@ -3155,6 +3158,10 @@ def source_reference(capture_id: str) -> str: "This DAG uses strong Airflow 1.10 syntax and omits schedule_interval. Historical default " "schedule and catchup behavior cannot be inferred safely without the deployed Airflow version." ), + "reserved_airflow_parameter_name": ( + "Airflow DAG parameter names beginning with '__flowx_' are reserved for flowx runtime bindings. " + "Rename the DAG parameter before migration." + ), } for candidate in unresolved: findings.append( @@ -3523,10 +3530,9 @@ def _reader(dep_var: str) -> str: def _declared_param_default(name: str, dag_params: dict[str, Any], schedule: dict[str, object] | None) -> Any: """Returns the Databricks-required default for a declared job parameter. - A DAG ``params={...}`` default wins. A macro-derived parameter (``run_date`` etc. from an Airflow - ``{{ ds }}``/``execution_date`` macro, or ``run_id``) gets its schedule-aware / inline default so - the value resolves at run time (and a native backfill can override a logical date). Everything else - defaults to an empty string. + A DAG ``params={...}`` default wins. A reserved macro-derived parameter gets its schedule-aware or + inline default so the value resolves at run time and native backfills can override logical dates. + Everything else defaults to an empty string. """ if dag_params.get(name) is not None: return dag_params[name] @@ -3559,11 +3565,16 @@ def _convert_activity_templates(activity: Activity) -> set[str]: for value in sql_params.values(): referenced |= set(_JOB_PARAM_REF.findall(value)) # generated_source was already rewritten (Variable.get -> dbutils.widgets.get); collect the - # widget names so the pipeline declares them as job parameters. Skip the internal __flowx_* - # widgets (op_args/op_kwargs) -- those are fed by the task's base_parameters, not job params. + # widget names so the pipeline declares them as job parameters. Airflow runtime widgets use the + # reserved __flowx_airflow_* namespace and must be declared; other __flowx_* widgets are task-local. generated = getattr(activity, "generated_source", None) if isinstance(generated, str): - referenced |= {name for name in _WIDGET_GET.findall(generated) if not name.startswith("__flowx_")} + referenced |= { + name + for name in _WIDGET_GET.findall(generated) + if not name.startswith(templating.FLOWX_INTERNAL_PARAMETER_PREFIX) + or name.startswith(templating.FLOWX_AIRFLOW_PARAMETER_PREFIX) + } return referenced diff --git a/src/flowx/sources/airflow/templating.py b/src/flowx/sources/airflow/templating.py index a2bc606..a4d99fe 100644 --- a/src/flowx/sources/airflow/templating.py +++ b/src/flowx/sources/airflow/templating.py @@ -15,13 +15,13 @@ from dataclasses import dataclass from typing import Any -# Airflow date/time macros carrying the run's *logical date* -> a named job parameter (not an inline -# time ref), so a native Databricks backfill can override the parameter per replayed window with -# {{backfill.iso_date}}. Each maps to (parameter_name, time_field); the loader assigns the parameter a -# schedule-aware default (see `date_param_default`). ``ds_nodash``/``ts_nodash`` have no dashless -# dynamic-value form, so they are intentionally NOT mapped -- they're left untouched (surfaced as an -# unresolved reference) rather than emitting an invalid ref. -_DATE_MACRO_PARAM: dict[str, tuple[str, str]] = { +FLOWX_INTERNAL_PARAMETER_PREFIX = "__flowx_" +FLOWX_AIRFLOW_PARAMETER_PREFIX = "__flowx_airflow_" +AIRFLOW_RUN_ID_PARAMETER = f"{FLOWX_AIRFLOW_PARAMETER_PREFIX}run_id" + +# Airflow date/time macros carry the run's logical date through reserved job parameters so native +# Databricks backfills can override them without colliding with user-defined DAG parameters. +_DATE_MACRO_FIELDS: dict[str, tuple[str, str]] = { "ds": ("run_date", "iso_date"), "ts": ("run_timestamp", "iso_datetime"), "data_interval_start": ("data_interval_start", "iso_datetime"), @@ -30,8 +30,10 @@ "logical_date": ("logical_date", "iso_datetime"), } -# job parameter name -> its time field, so the loader can default each to the right granularity. -DATE_PARAM_FIELDS: dict[str, str] = {param: field for param, field in _DATE_MACRO_PARAM.values()} +# Job parameter name -> dynamic-value time field. +DATE_PARAM_FIELDS: dict[str, str] = { + f"{FLOWX_AIRFLOW_PARAMETER_PREFIX}{suffix}": field for suffix, field in _DATE_MACRO_FIELDS.values() +} # Non-date macros with an exact Databricks equivalent, mapped inline (no backfill relevance). _MACRO_TO_DAB_REF: dict[str, str] = { @@ -56,56 +58,90 @@ def date_param_default(field: str, schedule: dict[str, object] | None) -> str: def macro_param_default(name: str, schedule: dict[str, object] | None) -> str | None: """Returns the Databricks-required default for a macro-derived job parameter, or None. - A logical-date parameter (``run_date`` etc.) gets its schedule-aware time ref; ``run_id`` gets the - inline run-id ref (bash/env-var threading forces even ``run_id`` through a job parameter, and its - default must resolve to the run id rather than an empty string). Any other name is not - macro-derived, so this returns None and the caller falls back to its own default. + Reserved logical-date parameters get schedule-aware time refs. The reserved run-id parameter gets + the inline run-id ref because shell and SQL tasks must bind it through a named value. Other names + are not macro-derived, so the caller supplies their default. """ field = DATE_PARAM_FIELDS.get(name) if field is not None: return date_param_default(field, schedule) - if name == "run_id": + if name == AIRFLOW_RUN_ID_PARAMETER: return _MACRO_TO_DAB_REF["run_id"] return None -# {{ params.X }} / {{ var.value.X }} / {{ dag_run.conf['X'] }} -> {{job.parameters.X}} -_PARAM_PATTERNS: list[re.Pattern[str]] = [ - re.compile(r"^params\.([A-Za-z_][A-Za-z0-9_]*)$"), - re.compile(r"^params\[['\"]([^'\"]+)['\"]\]$"), - re.compile(r"^var\.value\.([A-Za-z_][A-Za-z0-9_]*)$"), - re.compile(r"^dag_run\.conf\[['\"]([^'\"]+)['\"]\]$"), +_PARAM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [ + ("parameter", re.compile(r"^params\.([A-Za-z_][A-Za-z0-9_]*)$")), + ("parameter", re.compile(r"^params\[['\"]([^'\"]+)['\"]\]$")), + ("variable", re.compile(r"^var\.value\.([A-Za-z_][A-Za-z0-9_]*)$")), + ("conf", re.compile(r"^dag_run\.conf\[['\"]([^'\"]+)['\"]\]$")), ] _JINJA = re.compile(r"\{\{\s*(.*?)\s*\}\}") +_PARAMETER_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +@dataclass(frozen=True, slots=True) +class _TemplateBinding: + """One recognized Airflow expression and its collision-free Databricks binding.""" + + name: str + value_ref: str + job_parameter: str | None + + +def _job_parameter_ref(name: str) -> str: + return "{{job.parameters." + name + "}}" + + +def _airflow_parameter(namespace: str, name: str) -> str: + return f"{FLOWX_AIRFLOW_PARAMETER_PREFIX}{namespace}_{name}" + + +def _template_binding(expression: str) -> _TemplateBinding | None: + date_macro = _DATE_MACRO_FIELDS.get(expression) + if date_macro is not None: + name = f"{FLOWX_AIRFLOW_PARAMETER_PREFIX}{date_macro[0]}" + return _TemplateBinding(name=name, value_ref=_job_parameter_ref(name), job_parameter=name) + if expression in _MACRO_TO_DAB_REF: + return _TemplateBinding( + name=AIRFLOW_RUN_ID_PARAMETER, + value_ref=_MACRO_TO_DAB_REF[expression], + job_parameter=None, + ) + for namespace, pattern in _PARAM_PATTERNS: + match = pattern.match(expression) + if match is None: + continue + source_name = match.group(1) + if not _PARAMETER_NAME.fullmatch(source_name): + return None + if namespace == "parameter": + if source_name.startswith(FLOWX_INTERNAL_PARAMETER_PREFIX): + return None + name = source_name + else: + name = _airflow_parameter(namespace, source_name) + return _TemplateBinding(name=name, value_ref=_job_parameter_ref(name), job_parameter=name) + return None def convert_template(value: str) -> tuple[str, set[str]]: """Converts Airflow Jinja in *value* to DAB dynamic-value references. - Returns ``(converted_value, referenced_param_names)``. A logical-date macro (``ds``, - ``execution_date``, ...) maps to ``{{job.parameters.run_date}}`` (etc.) so a native backfill can - override it; ``params.X`` / ``var.value.X`` / ``dag_run.conf['X']`` map to ``{{job.parameters.X}}``; - ``run_id`` maps to its inline ref. Referenced parameter names are reported so the pipeline can - declare them. An unrecognised expression is left as-is (so nothing is silently corrupted). + Airflow-owned values use the reserved ``__flowx_airflow_`` namespace, while ``params.X`` retains + the user-visible job parameter name. This keeps logical dates, Variables, run configuration, and + user parameters distinct even when their source names match. Unknown expressions stay unchanged. """ params: set[str] = set() def _sub(match: re.Match[str]) -> str: - expr = match.group(1).strip() - if expr in _DATE_MACRO_PARAM: - name, _ = _DATE_MACRO_PARAM[expr] - params.add(name) - return "{{job.parameters." + name + "}}" - if expr in _MACRO_TO_DAB_REF: - return _MACRO_TO_DAB_REF[expr] - for pattern in _PARAM_PATTERNS: - m = pattern.match(expr) - if m: - name = m.group(1) - params.add(name) - return "{{job.parameters." + name + "}}" - return match.group(0) # unknown expression: leave untouched + binding = _template_binding(match.group(1).strip()) + if binding is None: + return match.group(0) + if binding.job_parameter is not None: + params.add(binding.job_parameter) + return binding.value_ref return _JINJA.sub(_sub, value), params @@ -114,75 +150,189 @@ def _sub(match: re.Match[str]) -> str: r"(?:\bFROM|\bJOIN|\bINTO|\bUPDATE|\bTABLE|\bVIEW|\bSCHEMA|\bCATALOG)\s*$", re.IGNORECASE, ) +_SQL_TYPED_LITERAL_CONTEXT = re.compile(r"\b(?:DATE|INTERVAL|TIME|TIMESTAMP)\s*$", re.IGNORECASE) +_SQL_UNSAFE_MARKER_ADJACENCY = frozenset("._") + + +@dataclass(frozen=True, slots=True) +class _SqlQuotedSpan: + start: int + end: int + delimiter: str + terminated: bool + + +def _sql_quoted_spans(sql: str) -> list[_SqlQuotedSpan]: + """Returns SQL quoted regions while ignoring quotes inside line and block comments.""" + spans: list[_SqlQuotedSpan] = [] + index = 0 + while index < len(sql): + if sql.startswith("--", index): + newline = sql.find("\n", index + 2) + index = len(sql) if newline < 0 else newline + 1 + continue + if sql.startswith("/*", index): + closing = sql.find("*/", index + 2) + index = len(sql) if closing < 0 else closing + 2 + continue + delimiter = sql[index] + if delimiter not in ("'", '"', "`"): + index += 1 + continue + start = index + index += 1 + terminated = False + while index < len(sql): + if sql[index] != delimiter: + index += 1 + continue + if index + 1 < len(sql) and sql[index + 1] == delimiter: + index += 2 + continue + index += 1 + terminated = True + break + spans.append(_SqlQuotedSpan(start=start, end=index, delimiter=delimiter, terminated=terminated)) + return spans + + +def _sql_marker_has_unsafe_adjacency(sql: str, start: int, end: int) -> bool: + """Returns whether replacing this expression would splice a marker into an SQL token.""" + + def unsafe(character: str) -> bool: + return character.isalnum() or character in _SQL_UNSAFE_MARKER_ADJACENCY + + return (start > 0 and unsafe(sql[start - 1])) or (end < len(sql) and unsafe(sql[end])) def convert_sql_template(sql: str) -> tuple[str, dict[str, str]]: """Rewrites Airflow Jinja in *sql* to ``:name`` markers + a ``sql_task.parameters`` map. - Databricks requires dynamic references in a ``sql_task`` to be passed through named parameters, - not interpolated into the SQL text. A logical-date macro ``{{ ds }}`` -> ``:run_date`` with - ``{"run_date": "{{job.parameters.run_date}}"}`` (a job parameter, so a native backfill can override - it); ``{{ params.x }}`` -> ``:x`` with ``{"x": "{{job.parameters.x}}"}``; ``run_id`` binds to its - inline ref. Unknown expressions are left untouched. + Databricks parameter markers are expressions, not text substitution. A macro that occupies an + entire single-quoted literal therefore replaces the quotes as well. A macro embedded inside a + string, quoted identifier, or adjacent SQL token remains unresolved so the loader emits a gap + instead of changing its meaning. Returns ``(sql_with_markers, parameters)``. """ parameters: dict[str, str] = {} + quoted_spans = _sql_quoted_spans(sql) - def _marker(name: str, match: re.Match[str]) -> str: + def _marker(name: str, start: int) -> str: marker = f":{name}" - return f"IDENTIFIER({marker})" if _SQL_IDENTIFIER_CONTEXT.search(sql[: match.start()]) else marker + return f"IDENTIFIER({marker})" if _SQL_IDENTIFIER_CONTEXT.search(sql[:start]) else marker - def _sub(match: re.Match[str]) -> str: - expr = match.group(1).strip() - if expr in _DATE_MACRO_PARAM: - name, _ = _DATE_MACRO_PARAM[expr] - parameters[name] = "{{job.parameters." + name + "}}" - return _marker(name, match) - if expr in _MACRO_TO_DAB_REF: - parameters["run_id"] = _MACRO_TO_DAB_REF[expr] - return _marker("run_id", match) - for pattern in _PARAM_PATTERNS: - m = pattern.match(expr) - if m: - name = m.group(1) - parameters[name] = "{{job.parameters." + name + "}}" - return _marker(name, match) - return match.group(0) - - return _JINJA.sub(_sub, sql), parameters + parts: list[str] = [] + cursor = 0 + for match in _JINJA.finditer(sql): + binding = _template_binding(match.group(1).strip()) + if binding is None: + continue + quoted = next( + (span for span in quoted_spans if span.start < match.start() and match.end() <= span.end), + None, + ) + replacement_start = match.start() + replacement_end = match.end() + if quoted is not None: + whole_single_literal = ( + quoted.delimiter == "'" + and quoted.terminated + and match.start() == quoted.start + 1 + and match.end() == quoted.end - 1 + and not _SQL_IDENTIFIER_CONTEXT.search(sql[: quoted.start]) + and not _SQL_TYPED_LITERAL_CONTEXT.search(sql[: quoted.start]) + and not (quoted.start > 0 and (sql[quoted.start - 1].isalnum() or sql[quoted.start - 1] == "_")) + ) + if not whole_single_literal: + continue + replacement_start = quoted.start + replacement_end = quoted.end + elif _sql_marker_has_unsafe_adjacency(sql, match.start(), match.end()): + continue + parts.append(sql[cursor:replacement_start]) + parts.append(_marker(binding.name, replacement_start)) + cursor = replacement_end + parameters[binding.name] = binding.value_ref + parts.append(sql[cursor:]) + return "".join(parts), parameters def convert_shell_template(command: str) -> tuple[str, dict[str, str]]: """Rewrites Airflow Jinja in a bash command to ``$NAME`` shell variable references. - A DAB dynamic-value ref (``{{job.parameters.X}}``) only resolves in a task *parameter* value, not - inside ``%sh`` notebook source, so a bash macro can't be replaced inline. Instead each recognised - macro becomes a ``$name`` shell variable the runner notebook exports from a widget of the same - name. ``{{ ds }}`` -> ``$run_date``; ``{{ params.x }}`` -> ``$x``; ``run_id`` -> ``$run_id``. - Unknown expressions are left untouched. + A DAB dynamic-value ref resolves in a task parameter, not inside ``%sh`` source. Each recognized + macro therefore becomes a braced shell variable exported from a widget. Braces preserve adjacent + text, and a macro inside single quotes temporarily exits that quote so the variable still expands. Returns ``(command_with_shell_vars, {name: dynamic_value_ref})`` where each ref is what the widget of that name must resolve to (a job parameter, or an inline ref for run_id). """ bindings: dict[str, str] = {} + def _quote_context(position: int) -> tuple[str | None, int | None]: + quote: str | None = None + quote_start: int | None = None + index = 0 + while index < position: + character = command[index] + if ( + quote is None + and character == "#" + and (index == 0 or command[index - 1].isspace() or command[index - 1] in ";|&()") + ): + newline = command.find("\n", index + 1) + index = position if newline < 0 else newline + 1 + continue + if character == "\\" and quote != "'": + index += 2 + continue + if character in ("'", '"'): + if quote is None: + quote = character + quote_start = index + elif quote == character: + quote = None + quote_start = None + index += 1 + return quote, quote_start + + def _inside_quoted_heredoc(position: int) -> bool: + delimiter: str | None = None + strip_tabs = False + for line in command[:position].splitlines(): + candidate = line.lstrip("\t") if strip_tabs else line + if delimiter is not None: + if candidate == delimiter: + delimiter = None + strip_tabs = False + continue + match = re.search(r"<<(-?)\s*(['\"])([A-Za-z_][A-Za-z0-9_]*)\2", line) + if match is not None: + strip_tabs = bool(match.group(1)) + delimiter = match.group(3) + return delimiter is not None + + def _escaped(position: int) -> bool: + backslashes = 0 + index = position - 1 + while index >= 0 and command[index] == "\\": + backslashes += 1 + index -= 1 + return backslashes % 2 == 1 + def _sub(match: re.Match[str]) -> str: - expr = match.group(1).strip() - if expr in _DATE_MACRO_PARAM: - name, _ = _DATE_MACRO_PARAM[expr] - bindings[name] = "{{job.parameters." + name + "}}" - return f"${name}" - if expr in _MACRO_TO_DAB_REF: - bindings["run_id"] = _MACRO_TO_DAB_REF[expr] - return "$run_id" - for pattern in _PARAM_PATTERNS: - m = pattern.match(expr) - if m: - name = m.group(1) - bindings[name] = "{{job.parameters." + name + "}}" - return f"${name}" - return match.group(0) + binding = _template_binding(match.group(1).strip()) + if binding is None: + return match.group(0) + quote, quote_start = _quote_context(match.start()) + if _inside_quoted_heredoc(match.start()) or (quote != "'" and _escaped(match.start())): + return match.group(0) + if quote == "'" and quote_start is not None and quote_start > 0 and command[quote_start - 1] == "$": + return match.group(0) + bindings[binding.name] = binding.value_ref + variable = f"${{{binding.name}}}" + return f"'\"{variable}\"'" if quote == "'" else variable return _JINJA.sub(_sub, command), bindings @@ -354,9 +504,10 @@ class TriggerRuleMapping: "one_failed": "AT_LEAST_ONE_FAILED", "one_success": "AT_LEAST_ONE_SUCCESS", "none_failed": "NONE_FAILED", - "none_failed_or_skipped": "NONE_FAILED", } +_APPROXIMATE_NONE_FAILED_RULES = frozenset({"none_failed_min_one_success", "none_failed_or_skipped"}) + _UNSUPPORTED_TRIGGER_RULES = frozenset({"always", "dummy", "none_skipped", "all_skipped", "one_done"}) @@ -384,7 +535,7 @@ def trigger_rule_mapping(task_kwargs: dict[str, ast.expr]) -> TriggerRuleMapping message="The trigger rule cannot be resolved statically.", ) rule = resolved_rule - if rule == "none_failed_min_one_success": + if rule in _APPROXIMATE_NONE_FAILED_RULES: return TriggerRuleMapping( rule=rule, outcome="NONE_FAILED", @@ -413,8 +564,8 @@ def trigger_rule_outcome(task_kwargs: dict[str, ast.expr]) -> str | None: # Airflow Variable / Connection calls in notebook bodies # -------------------------------------------------------------------------------------- -# Variable.get("x") / Variable.get('x', default) -> dbutils.widgets.get("x") (a job parameter). -_VARIABLE_GET = re.compile(r"""Variable\.get\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*(?:,[^)]*)?\)""") +# Variable.get("x") / Variable.get('x', default) -> a reserved job-parameter widget. +_VARIABLE_GET = re.compile(r"""(? flagged (needs a # secret-scope decision), rewritten to a dbutils.secrets.get with a placeholder scope. _CONNECTION_GET = re.compile( @@ -427,11 +578,11 @@ def airflow_connection_names(source: str) -> set[str]: return set(_CONNECTION_GET.findall(source)) -def rewrite_airflow_calls(source: str) -> tuple[str, set[str], list[str]]: +def rewrite_airflow_calls(source: str, *, rewrite_variable: bool = True) -> tuple[str, set[str], list[str]]: """Rewrites Airflow Variable/Connection calls in notebook-body *source*. - - ``Variable.get("x")`` -> ``dbutils.widgets.get("x")`` (a job parameter; ``x`` is - reported so the pipeline declares it and the notebook reads it as a widget). + - ``Variable.get("x")`` -> ``dbutils.widgets.get("__flowx_airflow_variable_x")``. The reserved + name prevents an Airflow Variable from colliding with a DAG parameter of the same name. - ``BaseHook.get_connection("c")`` -> ``dbutils.secrets.get(scope="_scope", key="...")`` with a note (connections need a manual secret-scope / UC-connection decision). @@ -442,7 +593,7 @@ def rewrite_airflow_calls(source: str) -> tuple[str, set[str], list[str]]: notes: list[str] = [] def _var(match: re.Match[str]) -> str: - name = match.group(1) + name = _airflow_parameter("variable", match.group(1)) params.add(name) return f'dbutils.widgets.get("{name}")' @@ -454,6 +605,6 @@ def _conn(match: re.Match[str]) -> str: ) return f'dbutils.secrets.get(scope="{conn}_scope", key="value") # TODO: set real scope/key' - rewritten = _VARIABLE_GET.sub(_var, source) + rewritten = _VARIABLE_GET.sub(_var, source) if rewrite_variable else source rewritten = _CONNECTION_GET.sub(_conn, rewritten) return rewritten, params, notes diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py index c690dd9..43d02d1 100644 --- a/tests/unit/test_airflow_operators.py +++ b/tests/unit/test_airflow_operators.py @@ -5,6 +5,8 @@ import tempfile from pathlib import Path +import pytest + from flowx.models.ir import ( DbtFactoryActivity, ForEachActivity, @@ -207,21 +209,23 @@ def test_bash_operator_macros_thread_through_shell_env_vars(): task = _by_key(p)["run"] assert isinstance(task, NotebookActivity) # Macros converted to shell variables; the raw {{ ... }} is gone from the %sh cell. - assert "--date $run_date" in task.generated_source - assert "--env $env" in task.generated_source + assert "--date ${__flowx_airflow_run_date}" in task.generated_source + assert "--env ${env}" in task.generated_source assert "{{ ds }}" not in task.generated_source # The widgets are declared and exported to the environment before the %sh cell. - assert "os.environ['run_date'] = dbutils.widgets.get('run_date')" in task.generated_source + assert ( + "os.environ['__flowx_airflow_run_date'] = dbutils.widgets.get('__flowx_airflow_run_date')" + ) in task.generated_source compile("\n".join(task.generated_source.split("# MAGIC %sh")[0].splitlines()), "
", "exec")
     # Each widget must be BOUND to its job parameter: an unbound widget is backfilled with an empty
     # string by the bundler, so the command would silently run with blank values.
     assert task.base_parameters == {
-        "run_date": "{{job.parameters.run_date}}",
+        "__flowx_airflow_run_date": "{{job.parameters.__flowx_airflow_run_date}}",
         "env": "{{job.parameters.env}}",
     }
     # run_date declared as a job parameter with the schedule-aware default (backfill-overridable).
     params = {param["name"]: param["default"] for param in p.parameters}
-    assert params["run_date"] == "{{job.trigger.time.iso_date}}"
+    assert params["__flowx_airflow_run_date"] == "{{job.trigger.time.iso_date}}"
     assert params["env"] == ""
 
 
@@ -249,9 +253,9 @@ def test_bash_operator_run_id_macro_defaults_to_run_id_ref():
         "    t = BashOperator(task_id='run', bash_command='echo {{ run_id }}')\n"
     )
     task = _by_key(p)["run"]
-    assert "echo $run_id" in task.generated_source
+    assert "echo ${__flowx_airflow_run_id}" in task.generated_source
     params = {param["name"]: param["default"] for param in p.parameters}
-    assert params["run_id"] == "{{job.run_id}}"
+    assert params["__flowx_airflow_run_id"] == "{{job.run_id}}"
 
 
 def test_bash_operator_wrapping_spark_submit_becomes_spark_task():
@@ -1047,19 +1051,19 @@ def test_jinja_macros_convert_to_dab_refs_and_collect_params():
     # values is still converted to DAB refs. {{ ds }} routes through a run_date job parameter (so a
     # native backfill can override it), not an inline start_time ref.
     kwargs_json = task.base_parameters["__flowx_op_kwargs"]
-    assert "{{job.parameters.run_date}}" in kwargs_json
+    assert "{{job.parameters.__flowx_airflow_run_date}}" in kwargs_json
     assert "{{job.parameters.env}}" in kwargs_json
-    # Referenced params are declared with a (Databricks-required) default; run_date defaults to the
-    # scheduled trigger time on a cron job. The internal __flowx_ widget is NOT declared.
+    # Referenced params are declared with Databricks-required defaults; the reserved logical-date
+    # parameter defaults to the scheduled trigger time on a cron job.
     assert p.parameters == [
+        {"name": "__flowx_airflow_run_date", "default": "{{job.trigger.time.iso_date}}"},
         {"name": "env", "default": ""},
-        {"name": "run_date", "default": "{{job.trigger.time.iso_date}}"},
     ]
 
 
 def test_execution_date_on_event_triggered_job_defaults_to_start_time():
     # A cron+sensor collapses to a file_arrival trigger -- no scheduled trigger time exists, so the
-    # run_date parameter approximates with the run start time (still overridable by a backfill).
+    # The reserved execution-date parameter approximates with the run start time.
     p = _load(
         "from airflow import DAG\n"
         "from airflow.operators.python import PythonOperator\n"
@@ -1071,7 +1075,7 @@ def test_execution_date_on_event_triggered_job_defaults_to_start_time():
         "    wait >> t\n"
     )
     assert (p.schedule or {}).get("kind") == "file_arrival"
-    assert p.parameters == [{"name": "execution_date", "default": "{{job.start_time.iso_datetime}}"}]
+    assert p.parameters == [{"name": "__flowx_airflow_execution_date", "default": "{{job.start_time.iso_datetime}}"}]
 
 
 def test_catchup_true_tags_pipeline_for_native_backfill():
@@ -1096,8 +1100,7 @@ def test_catchup_false_leaves_no_backfill_tag():
     assert "airflow_catchup" not in p.tags
 
 
-def test_dag_param_named_run_date_keeps_user_default():
-    # An explicit params={'run_date': ...} default wins over the schedule-aware backfill default.
+def test_dag_param_named_run_date_remains_distinct_from_logical_date():
     p = _load(
         "from airflow import DAG\n"
         "from airflow.models.param import Param\n"
@@ -1106,8 +1109,24 @@ def test_dag_param_named_run_date_keeps_user_default():
         "with DAG(dag_id='d', schedule_interval='0 6 * * *', params={'run_date': Param('2024-01-01')}) as dag:\n"
         "    t = PythonOperator(task_id='t', python_callable=w, op_kwargs={'date': '{{ ds }}'})\n"
     )
-    run_date = next(param for param in p.parameters if param["name"] == "run_date")
-    assert run_date["default"] == "2024-01-01"
+    parameters = {param["name"]: param["default"] for param in p.parameters}
+    assert parameters["run_date"] == "2024-01-01"
+    assert parameters["__flowx_airflow_run_date"] == "{{job.trigger.time.iso_date}}"
+    task = _by_key(p)["t"]
+    assert "{{job.parameters.__flowx_airflow_run_date}}" in task.base_parameters["__flowx_op_kwargs"]
+
+
+def test_sql_embedded_string_template_becomes_placeholder():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator\n"
+        "with DAG(dag_id='d') as dag:\n"
+        "    t = SQLExecuteQueryOperator(task_id='report', sql=\"SELECT 'partition_{{ ds }}'\")\n"
+    )
+
+    task = _by_key(p)["report"]
+    assert isinstance(task, PlaceholderActivity)
+    assert any(finding["code"] == "unresolved_airflow_template" for finding in p.not_translatable)
 
 
 def test_unsupported_airflow_macro_becomes_placeholder():
@@ -1450,9 +1469,80 @@ def test_variable_get_rewritten_to_widget_and_declared_as_param():
         "    t = PythonOperator(task_id='ingest', python_callable=ingest)\n"
     )
     task = _by_key(p)["ingest"]
-    assert 'dbutils.widgets.get("target_env")' in task.generated_source
+    assert 'dbutils.widgets.get("__flowx_airflow_variable_target_env")' in task.generated_source
     assert "Variable.get" not in task.generated_source
-    assert {"name": "target_env", "default": ""} in (p.parameters or [])
+    assert {"name": "__flowx_airflow_variable_target_env", "default": ""} in (p.parameters or [])
+
+
+@pytest.mark.parametrize(
+    "expression",
+    [
+        "Variable.get('target_env', 'prod')",
+        "Variable.get('target_env', default_var='prod')",
+        "Variable.get('target_env', deserialize_json=True)",
+        "Variable.get(variable_name)",
+    ],
+)
+def test_variable_get_forms_that_need_airflow_runtime_become_placeholders(expression: str):
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.python import PythonOperator\n"
+        "from airflow.models import Variable\n"
+        "variable_name = 'target_env'\n"
+        "def ingest():\n"
+        f"    print({expression})\n"
+        "with DAG(dag_id='d') as dag:\n"
+        "    t = PythonOperator(task_id='ingest', python_callable=ingest)\n"
+    )
+
+    task = _by_key(p)["ingest"]
+    assert isinstance(task, PlaceholderActivity)
+    assert "Variable.get" in task.comment
+
+
+def test_aliased_airflow_runtime_import_in_callable_becomes_placeholder():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.python import PythonOperator\n"
+        "from airflow.models import Variable as AirflowVariable\n"
+        "def ingest():\n"
+        "    print(AirflowVariable.get('target_env'))\n"
+        "with DAG(dag_id='d') as dag:\n"
+        "    t = PythonOperator(task_id='ingest', python_callable=ingest)\n"
+    )
+
+    task = _by_key(p)["ingest"]
+    assert isinstance(task, PlaceholderActivity)
+    assert "Airflow runtime import" in task.comment
+
+
+def test_function_local_airflow_import_becomes_placeholder():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.python import PythonOperator\n"
+        "def ingest():\n"
+        "    from airflow.models import Variable\n"
+        "    print(Variable.get('target_env'))\n"
+        "with DAG(dag_id='d') as dag:\n"
+        "    t = PythonOperator(task_id='ingest', python_callable=ingest)\n"
+    )
+
+    task = _by_key(p)["ingest"]
+    assert isinstance(task, PlaceholderActivity)
+    assert "Airflow runtime import" in task.comment
+
+
+def test_reserved_flowx_dag_parameter_becomes_an_explicit_gap():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(dag_id='d', params={'__flowx_airflow_run_date': 'spoofed'}) as dag:\n"
+        "    t = BashOperator(task_id='t', bash_command='echo ok')\n"
+    )
+
+    assert p.reconciliation_status == "verified_with_gaps"
+    assert any(item["code"] == "reserved_airflow_parameter_name" for item in p.not_translatable)
+    assert "__flowx_airflow_run_date" not in {param["name"] for param in p.parameters or []}
 
 
 def test_connection_get_becomes_placeholder_for_connection_object_mapping():
diff --git a/tests/unit/test_airflow_production_readiness.py b/tests/unit/test_airflow_production_readiness.py
index dbf14a1..0ad6dfa 100644
--- a/tests/unit/test_airflow_production_readiness.py
+++ b/tests/unit/test_airflow_production_readiness.py
@@ -268,7 +268,8 @@ def test_unresolved_jinja_in_generated_source_becomes_placeholder() -> None:
     assert any(finding["code"] == "unresolved_airflow_template" for finding in pipeline.not_translatable)
 
 
-def test_unsupported_and_approximate_trigger_rules_are_explicit(tmp_path: Path) -> None:
+@pytest.mark.parametrize("rule", ["none_failed_min_one_success", "none_failed_or_skipped"])
+def test_unsupported_and_approximate_trigger_rules_are_explicit(tmp_path: Path, rule: str) -> None:
     unsupported = load_airflow_dag(_REPROS / "t23_tr2.py")
     assert all(isinstance(unsupported.tasks[index], PlaceholderActivity) for index in (1, 2, 3))
 
@@ -279,7 +280,7 @@ def test_unsupported_and_approximate_trigger_rules_are_explicit(tmp_path: Path)
         "with DAG(dag_id='rules') as dag:\n"
         "    up = BashOperator(task_id='up', bash_command='echo up')\n"
         "    down = BashOperator(task_id='down', bash_command='echo down', "
-        "trigger_rule='none_failed_min_one_success')\n"
+        f"trigger_rule={rule!r})\n"
         "    up >> down\n",
         encoding="utf-8",
     )
diff --git a/tests/unit/test_airflow_templating.py b/tests/unit/test_airflow_templating.py
index c05d2c3..25cb9d1 100644
--- a/tests/unit/test_airflow_templating.py
+++ b/tests/unit/test_airflow_templating.py
@@ -15,9 +15,18 @@
 def test_execution_date_macros_route_through_job_parameters():
     # Logical-date macros become an overridable job parameter (not an inline start_time ref) so a
     # native Databricks backfill can override them per replayed window.
-    assert convert_template("{{ ds }}") == ("{{job.parameters.run_date}}", {"run_date"})
-    assert convert_template("{{ execution_date }}") == ("{{job.parameters.execution_date}}", {"execution_date"})
-    assert convert_template("{{ logical_date }}") == ("{{job.parameters.logical_date}}", {"logical_date"})
+    assert convert_template("{{ ds }}") == (
+        "{{job.parameters.__flowx_airflow_run_date}}",
+        {"__flowx_airflow_run_date"},
+    )
+    assert convert_template("{{ execution_date }}") == (
+        "{{job.parameters.__flowx_airflow_execution_date}}",
+        {"__flowx_airflow_execution_date"},
+    )
+    assert convert_template("{{ logical_date }}") == (
+        "{{job.parameters.__flowx_airflow_logical_date}}",
+        {"__flowx_airflow_logical_date"},
+    )
 
 
 def test_run_id_macro_stays_inline():
@@ -33,14 +42,65 @@ def test_dashless_macro_left_untouched():
 
 def test_sql_execution_date_binds_a_job_parameter():
     marked, params = convert_sql_template("SELECT * FROM t WHERE d = {{ ds }}")
-    assert marked == "SELECT * FROM t WHERE d = :run_date"
-    assert params == {"run_date": "{{job.parameters.run_date}}"}
+    assert marked == "SELECT * FROM t WHERE d = :__flowx_airflow_run_date"
+    assert params == {"__flowx_airflow_run_date": "{{job.parameters.__flowx_airflow_run_date}}"}
+
+
+def test_sql_macro_as_entire_string_literal_removes_sql_quotes():
+    marked, params = convert_sql_template("SELECT * FROM sales WHERE order_date = '{{ ds }}'")
+    assert marked == "SELECT * FROM sales WHERE order_date = :__flowx_airflow_run_date"
+    assert params == {"__flowx_airflow_run_date": "{{job.parameters.__flowx_airflow_run_date}}"}
+
+
+def test_sql_macro_embedded_in_string_literal_remains_unresolved():
+    marked, params = convert_sql_template("SELECT 'partition_{{ ds }}'")
+    assert marked == "SELECT 'partition_{{ ds }}'"
+    assert params == {}
+
+
+def test_sql_macros_in_quoted_identifiers_and_adjacent_tokens_remain_unresolved():
+    for sql in (
+        'SELECT * FROM "{{ params.table }}"',
+        "SELECT * FROM `{{ params.table }}`",
+        "SELECT * FROM analytics.{{ params.table }}",
+        "SELECT {{ params.column }}_suffix FROM source",
+    ):
+        assert convert_sql_template(sql) == (sql, {})
+
+
+def test_sql_macros_in_typed_and_prefixed_literals_remain_unresolved():
+    for sql in (
+        "SELECT DATE '{{ ds }}'",
+        "SELECT TIMESTAMP '{{ ts }}'",
+        "SELECT INTERVAL '{{ params.hours }}' HOUR",
+        "SELECT r'{{ params.pattern }}'",
+    ):
+        assert convert_sql_template(sql) == (sql, {})
+
+
+def test_sql_quote_scanning_ignores_quotes_in_comments():
+    sql = "-- owner's date\nSELECT '{{ ds }}'"
+    assert convert_sql_template(sql) == (
+        "-- owner's date\nSELECT :__flowx_airflow_run_date",
+        {"__flowx_airflow_run_date": "{{job.parameters.__flowx_airflow_run_date}}"},
+    )
+
+
+def test_sql_unquoted_identifier_uses_identifier_parameter_marker():
+    sql = "SELECT * FROM {{ params.table }} WHERE id = {{ params.id }}"
+    assert convert_sql_template(sql) == (
+        "SELECT * FROM IDENTIFIER(:table) WHERE id = :id",
+        {
+            "table": "{{job.parameters.table}}",
+            "id": "{{job.parameters.id}}",
+        },
+    )
 
 
 def test_sql_run_id_binds_inline_ref():
     marked, params = convert_sql_template("SELECT '{{ run_id }}'")
-    assert marked == "SELECT ':run_id'"
-    assert params == {"run_id": "{{job.run_id}}"}
+    assert marked == "SELECT :__flowx_airflow_run_id"
+    assert params == {"__flowx_airflow_run_id": "{{job.run_id}}"}
 
 
 def test_date_param_default_is_schedule_aware():
@@ -54,14 +114,64 @@ def test_date_param_default_is_schedule_aware():
 
 def test_shell_template_threads_macros_through_named_vars():
     command, bindings = convert_shell_template("etl.py --date {{ ds }} --run {{ run_id }} --env {{ params.env }}")
-    assert command == "etl.py --date $run_date --run $run_id --env $env"
+    assert command == ("etl.py --date ${__flowx_airflow_run_date} --run ${__flowx_airflow_run_id} --env ${env}")
     assert bindings == {
-        "run_date": "{{job.parameters.run_date}}",
-        "run_id": "{{job.run_id}}",
+        "__flowx_airflow_run_date": "{{job.parameters.__flowx_airflow_run_date}}",
+        "__flowx_airflow_run_id": "{{job.run_id}}",
         "env": "{{job.parameters.env}}",
     }
 
 
+def test_shell_template_braces_adjacent_macros_and_breaks_out_of_single_quotes():
+    command, bindings = convert_shell_template("echo '/data/{{ ds }}_load.csv'")
+    assert command == "echo '/data/'\"${__flowx_airflow_run_date}\"'_load.csv'"
+    assert bindings == {"__flowx_airflow_run_date": "{{job.parameters.__flowx_airflow_run_date}}"}
+
+
+def test_shell_template_leaves_nonexpanding_or_escaped_contexts_unresolved():
+    for command in (
+        "echo $'{{ ds }}'",
+        "printf \\{{ ds }}",
+        "cat <<'EOF'\n{{ ds }}\nEOF",
+    ):
+        assert convert_shell_template(command) == (command, {})
+
+
+def test_shell_quote_scanning_ignores_quotes_in_comments():
+    command = "# owner's note\necho {{ ds }}"
+    assert convert_shell_template(command) == (
+        "# owner's note\necho ${__flowx_airflow_run_date}",
+        {"__flowx_airflow_run_date": "{{job.parameters.__flowx_airflow_run_date}}"},
+    )
+
+
+def test_template_namespaces_do_not_collapse_equal_source_names():
+    converted, params = convert_template(
+        "{{ ds }}|{{ params.run_date }}|{{ var.value.run_date }}|{{ dag_run.conf['run_date'] }}"
+    )
+    assert converted == (
+        "{{job.parameters.__flowx_airflow_run_date}}|{{job.parameters.run_date}}|"
+        "{{job.parameters.__flowx_airflow_variable_run_date}}|"
+        "{{job.parameters.__flowx_airflow_conf_run_date}}"
+    )
+    assert params == {
+        "__flowx_airflow_run_date",
+        "run_date",
+        "__flowx_airflow_variable_run_date",
+        "__flowx_airflow_conf_run_date",
+    }
+
+
+def test_reserved_flowx_parameter_reference_remains_unresolved():
+    value = "{{ params.__flowx_airflow_run_date }}"
+    assert convert_template(value) == (value, set())
+
+
+def test_bracket_parameter_names_must_be_valid_job_parameter_identifiers():
+    value = "{{ params['bad-name'] }}"
+    assert convert_template(value) == (value, set())
+
+
 def test_shell_template_leaves_unknown_expressions():
     command, bindings = convert_shell_template("echo {{ some.unknown }}")
     assert command == "echo {{ some.unknown }}"
@@ -69,8 +179,8 @@ def test_shell_template_leaves_unknown_expressions():
 
 
 def test_macro_param_default_covers_date_and_run_id_and_none():
-    assert macro_param_default("run_date", {"kind": "schedule"}) == "{{job.trigger.time.iso_date}}"
-    assert macro_param_default("run_id", None) == "{{job.run_id}}"
+    assert macro_param_default("__flowx_airflow_run_date", {"kind": "schedule"}) == ("{{job.trigger.time.iso_date}}")
+    assert macro_param_default("__flowx_airflow_run_id", None) == "{{job.run_id}}"
     assert macro_param_default("env", None) is None  # a user param, not macro-derived
 
 

From a5f440c95ed84e1964cef9e42b4dc0dbab73560a Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Tue, 11 Aug 2026 14:11:32 -0700
Subject: [PATCH 66/77] fix(airflow): harden public DAG corpus handling

---
 src/flowx/agentic.py                      |   1 +
 src/flowx/bundler/dab_writer.py           |   5 +
 src/flowx/ir_serde.py                     |   3 +
 src/flowx/models/ir.py                    |   2 +
 src/flowx/preparer/workflow_preparer.py   |   5 +
 src/flowx/sources/airflow/audit.py        |  20 ++-
 src/flowx/sources/airflow/loader.py       | 182 ++++++++++++++++++++--
 src/flowx/validate/bundle_invariants.py   |   9 ++
 tests/unit/test_airflow_operators.py      | 100 ++++++++++++
 tests/unit/test_airflow_reconciliation.py |  52 ++++++-
 tests/unit/test_bundle_invariants.py      |   4 +
 tests/unit/test_bundler.py                |  28 ++++
 12 files changed, 392 insertions(+), 19 deletions(-)

diff --git a/src/flowx/agentic.py b/src/flowx/agentic.py
index ad62d4b..2324b72 100644
--- a/src/flowx/agentic.py
+++ b/src/flowx/agentic.py
@@ -874,6 +874,7 @@ def _build_gap_envelopes(
                     "schedule": pipeline.get("schedule"),
                     "parameters": pipeline.get("parameters"),
                     "tags": pipeline.get("tags"),
+                    "description": pipeline.get("description"),
                 },
                 reason={
                     "code": str(matched_finding.get("code", "operator_placeholder")),
diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py
index 1dac06d..f238571 100644
--- a/src/flowx/bundler/dab_writer.py
+++ b/src/flowx/bundler/dab_writer.py
@@ -1559,6 +1559,10 @@ def _build_job_resource(
         "name": workflow.name,
         "tasks": workflow.tasks,
     }
+    if workflow.description:
+        job_def["description"] = workflow.description
+    if workflow.tags:
+        job_def["tags"] = dict(workflow.tags)
 
     if attach_clusters:
         _bind_cluster_to_notebook_tasks(workflow.tasks)
@@ -1898,6 +1902,7 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d
         parameters.append(entry)
     pipeline = Pipeline(
         name=pipeline_dict.get("name", "unknown"),
+        description=pipeline_dict.get("description"),
         tasks=activities,
         parameters=parameters or None,
         translation_configuration=_reconstruct_configuration(pipeline_dict.get("translation_configuration")),
diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py
index 006e70f..09b25e6 100644
--- a/src/flowx/ir_serde.py
+++ b/src/flowx/ir_serde.py
@@ -68,6 +68,8 @@ def pipeline_to_dict(pipeline: Pipeline) -> dict[str, Any]:
         "migration_status": pipeline.migration_status,
         "audit": dict(pipeline.audit),
     }
+    if pipeline.description is not None:
+        result["description"] = pipeline.description
     if pipeline.translation_configuration is not None:
         result["translation_configuration"] = configuration_to_dict(pipeline.translation_configuration)
     return result
@@ -417,6 +419,7 @@ def pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]:
     return {
         "__class__": "Pipeline",
         "name": pipeline.name,
+        "description": pipeline.description,
         "parameters": pipeline.parameters,
         "schedule": pipeline.schedule,
         "tags": pipeline.tags,
diff --git a/src/flowx/models/ir.py b/src/flowx/models/ir.py
index 12d77f7..373b626 100644
--- a/src/flowx/models/ir.py
+++ b/src/flowx/models/ir.py
@@ -606,6 +606,7 @@ class Pipeline:
 
     Attributes:
         name: Logical pipeline name.
+        description: Human-readable workflow description.
         parameters: Pipeline parameter definitions.
         schedule: Serialized schedule definition, if any.
         tasks: Ordered list of translated activities.
@@ -617,6 +618,7 @@ class Pipeline:
     """
 
     name: str
+    description: str | None = None
     parameters: list[dict[str, Any]] | None = None
     schedule: dict[str, Any] | None = None
     tasks: list[Activity] = field(default_factory=list)
diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py
index ab50ccd..78618ad 100644
--- a/src/flowx/preparer/workflow_preparer.py
+++ b/src/flowx/preparer/workflow_preparer.py
@@ -69,6 +69,8 @@ class PreparedWorkflow:
     # renders as ``schedule:`` / ``trigger:`` on the emitted DAB job.
     schedule: dict[str, Any] | None = None
     source: str | None = None
+    description: str | None = None
+    tags: dict[str, str] = field(default_factory=dict)
 
 
 # The DAB job ``run_if`` vocabulary. Airflow maps ``trigger_rule`` straight to one of these
@@ -410,6 +412,7 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow:
             )
         )
 
+    is_airflow = pipeline.tags.get("source") == "airflow"
     return PreparedWorkflow(
         name=pipeline.name,
         tasks=all_tasks,
@@ -423,6 +426,8 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow:
         parameter_approximations=list(artifacts.parameter_approximations),
         schedule=pipeline.schedule,
         source=str(pipeline.tags.get("source")) if pipeline.tags.get("source") else None,
+        description=pipeline.description if is_airflow else None,
+        tags=dict(pipeline.tags) if is_airflow else {},
     )
 
 
diff --git a/src/flowx/sources/airflow/audit.py b/src/flowx/sources/airflow/audit.py
index 3d906c0..ffe9ac4 100644
--- a/src/flowx/sources/airflow/audit.py
+++ b/src/flowx/sources/airflow/audit.py
@@ -8,6 +8,8 @@
 from pathlib import Path
 from typing import Any
 
+_EDGE_MODIFIER_CONSTRUCTS = frozenset({"Label"})
+
 
 @dataclass(frozen=True, slots=True, kw_only=True)
 class AuditCandidate:
@@ -314,13 +316,23 @@ def _audit_position(self, node: ast.expr) -> list[str]:
         return []
 
     def _audit_shift(self, node: ast.expr) -> list[str]:
+        references, _is_modifier = self._audit_shift_operand(node)
+        return references
+
+    def _audit_shift_operand(self, node: ast.expr) -> tuple[list[str], bool]:
+        """Audits a shift operand while treating Airflow edge metadata as transparent."""
         if not isinstance(node, ast.BinOp) or not isinstance(node.op, (ast.RShift, ast.LShift)):
-            return self._audit_position(node)
-        left = self._audit_shift(node.left)
-        right = self._audit_shift(node.right)
+            is_modifier = isinstance(node, ast.Call) and _leaf(node.func, self.aliases) in _EDGE_MODIFIER_CONSTRUCTS
+            return self._audit_position(node), is_modifier
+        left, left_is_modifier = self._audit_shift_operand(node.left)
+        right, right_is_modifier = self._audit_shift_operand(node.right)
+        if right_is_modifier:
+            return left, left_is_modifier
+        if left_is_modifier:
+            return right, right_is_modifier
         upstreams, downstreams = (left, right) if isinstance(node.op, ast.RShift) else (right, left)
         self._add_edges(node, upstreams, downstreams, "shift")
-        return right
+        return right, False
 
     def _call_reference(self, call: ast.Call) -> str:
         operator, keywords, _mapped = _operator_call(call, self.aliases)
diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py
index 091b83b..afb08c0 100644
--- a/src/flowx/sources/airflow/loader.py
+++ b/src/flowx/sources/airflow/loader.py
@@ -44,6 +44,10 @@
 from flowx.sources.airflow import operators as ops
 from flowx.utils import normalize_task_key
 
+_EDGE_MODIFIER_CONSTRUCTS = frozenset({"Label"})
+_NON_EXECUTION_DAG_SETTINGS = frozenset({"tags", "description", "doc_md", "dag_display_name", "default_args.owner"})
+_DATABRICKS_JOB_TAG_LIMIT = 25
+
 
 @dataclass(slots=True)
 class _TaskFlowTask:
@@ -63,6 +67,7 @@ class _TaskFlowTask:
     task_id: str
     def_name: str
     decorator: str
+    source_reference: str
     is_async: bool = False
     positional_deps: dict[int, str] = field(default_factory=dict)
     keyword_deps: dict[str, str] = field(default_factory=dict)
@@ -723,6 +728,9 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None = None
         # DAG-level params={...} defaults (param name -> literal default), so emitted job parameters
         # carry a Databricks-required default rather than an empty placeholder.
         self.dag_params: dict[str, Any] = {}
+        self.dag_description: str | None = None
+        self.dag_user_tags: list[str] = []
+        self.dag_owner: str | None = None
         # task variable name -> TaskGroup id prefix (for task-key namespacing)
         self.groups: dict[str, str] = {}
         self._group_stack: list[str] = []
@@ -844,7 +852,7 @@ def visit_Assign(self, node: ast.Assign) -> None:
             elif self._register_helper_factory_call(node.value, internal_var, binding=var):
                 self._claimed_statement_ids.add(id(node))
                 pass
-            elif self._register_taskflow_call(node.value, internal_var):
+            elif self._register_taskflow_call(node.value, internal_var, source_reference=var):
                 self._task_bindings[var] = internal_var
                 self._claimed_statement_ids.add(id(node))
                 pass  # a `x = mytask(...)` TaskFlow invocation, captured with var as its key
@@ -1079,7 +1087,7 @@ def _taskflow_def_name(self, call: ast.Call) -> tuple[str | None, bool, str | No
             return func.id, mapped, override_id
         return None, mapped, override_id
 
-    def _register_taskflow_call(self, call: ast.Call, var: str) -> bool:
+    def _register_taskflow_call(self, call: ast.Call, var: str, *, source_reference: str | None = None) -> bool:
         """Records a TaskFlow ``@task`` invocation as a task instance keyed by *var*.
 
         Binds each call argument that references (or nests) another ``@task`` to that upstream task
@@ -1095,6 +1103,7 @@ def _register_taskflow_call(self, call: ast.Call, var: str) -> bool:
             task_id=override_id or var,
             def_name=def_name,
             decorator=decorator,
+            source_reference=source_reference or var,
             is_async=isinstance(function, ast.AsyncFunctionDef),
         )
         self.taskflow_tasks[var] = task
@@ -1215,7 +1224,7 @@ def _resolve_taskflow_arg(self, arg: ast.expr) -> str | None:
             if def_name is not None:
                 self._taskflow_counter += 1
                 synthetic = f"{def_name}__tf{self._taskflow_counter}"
-                self._register_taskflow_call(arg, synthetic)
+                self._register_taskflow_call(arg, synthetic, source_reference=def_name)
                 return synthetic
         return None
 
@@ -1351,7 +1360,8 @@ def _helper_targets_assigned_dag(self, call: ast.Call) -> bool:
 
     def _read_dag_kwargs(self, call: ast.Call) -> None:
         kwargs = {kw.arg: _bind_constants(kw.value, self._constants) for kw in call.keywords if kw.arg}
-        self.dag_id = ops.literal_str(kwargs.get("dag_id"))
+        positional_dag_id = ops.literal_str(call.args[0]) if call.args else None
+        self.dag_id = ops.literal_str(kwargs.get("dag_id")) or positional_dag_id
         self._apply_dag_kwargs(kwargs)
         if self.airflow_generation == "1.10" and not {"schedule", "schedule_interval"} & kwargs.keys():
             self.unresolved_constructs.append(("ambiguous_airflow_1_10_default_schedule", call))
@@ -1364,6 +1374,10 @@ def _apply_dag_kwargs(self, kwargs: dict[str, ast.expr]) -> None:
         )
         self.timezone = _extract_timezone(kwargs.get("start_date")) or _extract_timezone(kwargs.get("timezone"))
         self.catchup = ops.literal_value(kwargs.get("catchup")) is True
+        self.dag_description = ops.literal_str(kwargs.get("description"))
+        tags = ops.literal_value(kwargs.get("tags"))
+        if isinstance(tags, (list, tuple)) and all(isinstance(tag, str) for tag in tags):
+            self.dag_user_tags = list(tags)
         # default_args is a dict literal of DAG-wide task settings (retries, timeouts, email).
         default_args = kwargs.get("default_args")
         if isinstance(default_args, ast.Dict):
@@ -1373,6 +1387,9 @@ def _apply_dag_kwargs(self, kwargs: dict[str, ast.expr]) -> None:
                 if isinstance(key, ast.Constant) and isinstance(key.value, str)
             }
             self.captured_dag_settings.update(f"default_args.{name}" for name in self.default_args)
+            owner = self.default_args.get("owner")
+            if owner is not None:
+                self.dag_owner = ops.literal_str(owner)
         # params={...} supplies DAG parameter defaults; each value is a literal or a Param(default=...).
         params = kwargs.get("params")
         if isinstance(params, ast.Dict):
@@ -1432,7 +1449,7 @@ def visit_Expr(self, node: ast.Expr) -> None:
                 if task_var in self.taskflow_tasks:
                     self._taskflow_counter += 1
                     task_var = f"{def_name}__tf{self._taskflow_counter}"
-                self._register_taskflow_call(value, task_var)
+                self._register_taskflow_call(value, task_var, source_reference=def_name)
                 self._claimed_statement_ids.add(id(node))
             elif self._register_bare_operator_call(value) is not None:
                 self._claimed_statement_ids.add(id(node))
@@ -1500,13 +1517,25 @@ def _collect_shift_chain(self, binop: ast.BinOp) -> None:
 
     def _collect_shift_expression(self, node: ast.expr) -> list[str]:
         """Collects each shift edge recursively and returns the expression's chain result."""
+        tasks, _is_modifier = self._collect_shift_operand(node)
+        return tasks
+
+    def _collect_shift_operand(self, node: ast.expr) -> tuple[list[str], bool]:
+        """Collects a shift operand while treating Airflow edge metadata as transparent."""
         if not isinstance(node, ast.BinOp) or not isinstance(node.op, (ast.RShift, ast.LShift)):
-            return self._shift_position_names(node)
-        left = self._collect_shift_expression(node.left)
-        right = self._collect_shift_expression(node.right)
+            is_modifier = (
+                isinstance(node, ast.Call) and _construct_name(node.func, self._aliases) in _EDGE_MODIFIER_CONSTRUCTS
+            )
+            return self._shift_position_names(node), is_modifier
+        left, left_is_modifier = self._collect_shift_operand(node.left)
+        right, right_is_modifier = self._collect_shift_operand(node.right)
+        if right_is_modifier:
+            return left, left_is_modifier
+        if left_is_modifier:
+            return right, right_is_modifier
         upstream, downstream = (left, right) if isinstance(node.op, ast.RShift) else (right, left)
         self._add_edges(upstream, downstream, node)
-        return right
+        return right, False
 
     def _shift_position_names(self, node: ast.expr) -> list[str]:
         # A shift-chain position resolves to task vars. An inline TaskFlow call (`extract()`) is
@@ -1516,10 +1545,12 @@ def _shift_position_names(self, node: ast.expr) -> list[str]:
         if isinstance(node, ast.Call):
             def_name, _mapped, _override = self._taskflow_def_name(node)
             if def_name is not None:
-                self._taskflow_counter += 1
-                synthetic = f"{def_name}__tf{self._taskflow_counter}"
-                self._register_taskflow_call(node, synthetic)
-                return [synthetic]
+                task_var = def_name
+                if task_var in self.taskflow_tasks:
+                    self._taskflow_counter += 1
+                    task_var = f"{def_name}__tf{self._taskflow_counter}"
+                self._register_taskflow_call(node, task_var, source_reference=def_name)
+                return [task_var]
             # An inline classic operator (`Op(...) >> Op(...)` with no assignments) is still a task.
             bare_var = self._register_bare_operator_call(node)
             if bare_var is not None:
@@ -2621,6 +2652,47 @@ def append_task(activity: Activity, capture_id: str) -> None:
             else:
                 append_task(activity, var)
             continue
+        mapped_output_dependencies = sorted(
+            {
+                dependency
+                for dependency in [*tf.positional_deps.values(), *tf.keyword_deps.values()]
+                if dependency in visitor.mapped
+            }
+        )
+        if mapped_output_dependencies:
+            placeholder = PlaceholderActivity(
+                name=tf.task_id,
+                task_key=task_key,
+                original_type=f"@{tf.decorator}",
+                comment=(
+                    "Airflow aggregates mapped TaskFlow return values for downstream XCom consumers, but "
+                    "Databricks For each tasks do not expose nested task values to downstream tasks. "
+                    "Materialize and aggregate the mapped results explicitly."
+                ),
+                raw_definition={
+                    "operator": f"@{tf.decorator}",
+                    "source": ast.get_source_segment(source, definition) or "",
+                    "invocation": ast.get_source_segment(source, visitor.capture_source_nodes[var]) or "",
+                    "mapped_upstreams": [var_to_task_key[dependency] for dependency in mapped_output_dependencies],
+                },
+            )
+            placeholder.depends_on = depends_on
+            append_task(placeholder, var)
+            semantic_findings.append(
+                _semantic_finding(
+                    source_file or dag_path.name,
+                    visitor.calls.get(var),
+                    code="taskflow_mapped_output_unavailable",
+                    message=(
+                        f"Task {tf.task_id!r} consumes mapped TaskFlow output that Databricks For each "
+                        "tasks cannot expose as an aggregate."
+                    ),
+                    task_key=task_key,
+                    capture_id=var,
+                    upstream_task_keys=[var_to_task_key[dependency] for dependency in mapped_output_dependencies],
+                )
+            )
+            continue
         if var in visitor.mapped and tf.expand_items_json is None:
             # .expand over a non-literal iterable (e.g. an upstream task's output) can't be lowered to
             # a static for_each inputs array -- route to the agentic-gap round instead of silently
@@ -2706,9 +2778,19 @@ def append_task(activity: Activity, capture_id: str) -> None:
         # Airflow catchup=True has no DABs schedule setting; it maps to running a native Databricks
         # backfill, which overrides the reserved logical-date parameter per replayed window.
         tags["airflow_catchup"] = "true"
+    if visitor.dag_owner:
+        tags["airflow_owner"] = visitor.dag_owner
+    available_user_tags = _DATABRICKS_JOB_TAG_LIMIT - len(tags)
+    tags.update(
+        {
+            f"airflow_tag_{index}": value
+            for index, value in enumerate(visitor.dag_user_tags[:available_user_tags], start=1)
+        }
+    )
     expected_ir_edges = {(dependency.task_key, task.task_key) for task in tasks for dependency in task.depends_on or []}
     pipeline = Pipeline(
         name=visitor.dag_id or Path(dag_path).stem,
+        description=visitor.dag_description,
         tasks=tasks,
         parameters=parameters,
         schedule=schedule,
@@ -2744,6 +2826,11 @@ def append_task(activity: Activity, capture_id: str) -> None:
         "default_args.retries",
         "default_args.retry_delay",
         "default_args.execution_timeout",
+        "tags",
+        "description",
+        "doc_md",
+        "dag_display_name",
+        "default_args.owner",
     }
 )
 
@@ -2987,7 +3074,12 @@ def add_capture_claim(code: str, node: ast.AST, discriminator: str, capture_id:
 
     def source_reference(capture_id: str) -> str:
         capture = visitor.task_captures.get(capture_id)
-        return capture.variable if capture is not None else capture_id.split("__L", 1)[0]
+        if capture is not None:
+            return capture.variable
+        taskflow = visitor.taskflow_tasks.get(capture_id)
+        if taskflow is not None:
+            return taskflow.source_reference
+        return capture_id.split("__L", 1)[0]
 
     audited_edge_identities = sorted(
         (str(candidate.details["upstream"]), str(candidate.details["downstream"]))
@@ -3137,6 +3229,47 @@ def source_reference(capture_id: str) -> str:
             )
         )
 
+    for candidate in audit.settings:
+        name = str(candidate.details.get("name"))
+        if name not in _NON_EXECUTION_DAG_SETTINGS:
+            continue
+        emitted_user_tag_count = sum(key.startswith("airflow_tag_") for key in pipeline.tags)
+        partially_mapped = name == "tags" and emitted_user_tag_count < len(visitor.dag_user_tags)
+        mapped = (
+            (name == "tags" and bool(visitor.dag_user_tags))
+            or (name == "description" and visitor.dag_description is not None)
+            or (name == "default_args.owner" and visitor.dag_owner is not None)
+        )
+        transformations.append(
+            {
+                "code": (
+                    "dag_setting_partially_mapped"
+                    if partially_mapped
+                    else "dag_setting_mapped"
+                    if mapped
+                    else "dag_setting_ignored"
+                ),
+                "setting": name,
+                "target": {
+                    "tags": "job.tags",
+                    "description": "job.description",
+                    "default_args.owner": "job.tags.airflow_owner",
+                }.get(name),
+                "rationale": (
+                    "databricks_jobs_support_at_most_25_tags"
+                    if partially_mapped
+                    else "preserved_as_databricks_job_metadata"
+                    if mapped
+                    else "non_execution_metadata_has_no_required_runtime_effect"
+                ),
+                **(
+                    {"source_count": len(visitor.dag_user_tags), "emitted_count": emitted_user_tag_count}
+                    if name == "tags"
+                    else {}
+                ),
+            }
+        )
+
     unresolved_messages = {
         "unresolved_asset_schedule": (
             "An Airflow Asset/Dataset schedule lacks an explicit Databricks table mapping. Add "
@@ -3226,6 +3359,27 @@ def source_task_id(capture_id: str) -> str:
             }
         )
 
+    if not pipeline.tasks and not any(item["severity"] == "failed" for item in findings):
+        pipeline.tasks.append(
+            NotebookActivity(
+                name="Airflow DAG completion",
+                task_key="__flowx_empty_dag",
+                notebook_path="notebooks/__flowx_empty_dag.py",
+                generated_source=(
+                    "# Databricks notebook source\n"
+                    "# This DAG contained no executable tasks after structural operators were rewired.\n"
+                    "print('Airflow DAG completed without executable tasks.')\n"
+                ),
+            )
+        )
+        transformations.append(
+            {
+                "code": "empty_dag_sentinel_emitted",
+                "task_key": "__flowx_empty_dag",
+                "rationale": "preserve_a_runnable_job_for_a_structural_or_empty_airflow_dag",
+            }
+        )
+
     blocking_gaps = [*unsupported_settings, *unresolved]
     placeholder_entries = [
         (
diff --git a/src/flowx/validate/bundle_invariants.py b/src/flowx/validate/bundle_invariants.py
index 01b09dc..2ed275f 100644
--- a/src/flowx/validate/bundle_invariants.py
+++ b/src/flowx/validate/bundle_invariants.py
@@ -99,6 +99,15 @@ def check_job(job_key: str, job: dict[str, Any]) -> list[BundleFinding]:
     findings: list[BundleFinding] = []
     where = f"job '{job_key}'"
 
+    if not job.get("tasks"):
+        findings.append(
+            BundleFinding(
+                code="empty_job",
+                location=where,
+                message="A Lakeflow Job must contain at least one executable task.",
+            )
+        )
+
     # 1. No duplicate job-parameter names.
     param_names = [param.get("name") for param in (job.get("parameters") or []) if isinstance(param, dict)]
     duplicate_params = sorted({name for name in param_names if name is not None and param_names.count(name) > 1})
diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py
index 43d02d1..e54c3a7 100644
--- a/tests/unit/test_airflow_operators.py
+++ b/tests/unit/test_airflow_operators.py
@@ -599,6 +599,25 @@ def test_dummy_rewire_bridges_dependencies():
     assert [d.task_key for d in tasks["b"].depends_on] == ["a"]  # bridged through dropped gate
 
 
+def test_structural_only_dag_emits_completion_sentinel():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.empty import EmptyOperator\n"
+        "with DAG(dag_id='structural_only') as dag:\n"
+        "    start = EmptyOperator(task_id='start')\n"
+        "    end = EmptyOperator(task_id='end')\n"
+        "    start >> end\n"
+    )
+
+    assert p.reconciliation_status == "verified"
+    assert len(p.tasks) == 1
+    sentinel = p.tasks[0]
+    assert isinstance(sentinel, NotebookActivity)
+    assert sentinel.task_key == "__flowx_empty_dag"
+    assert "completed without executable tasks" in (sentinel.generated_source or "")
+    assert any(item["code"] == "empty_dag_sentinel_emitted" for item in p.audit["transformations"])
+
+
 def test_cosmos_dbt_task_group_becomes_dbt_factory():
     p = _load(
         "from airflow import DAG\n"
@@ -1828,6 +1847,87 @@ def test_taskflow_expand_literal_list_becomes_for_each():
     compile(inner.generated_source, "", "exec")
 
 
+def test_taskflow_mapped_output_consumer_becomes_placeholder():
+    p = _load(
+        "from airflow.decorators import dag, task\n"
+        "@task\n"
+        "def add_one(value):\n"
+        "    return value + 1\n"
+        "@task\n"
+        "def total(values):\n"
+        "    return sum(values)\n"
+        "@dag(dag_id='mapped_output')\n"
+        "def pipeline():\n"
+        "    added = add_one.expand(value=[1, 2, 3])\n"
+        "    total(added)\n"
+        "pipeline()\n"
+    )
+
+    tasks = _by_key(p)
+    assert isinstance(tasks["added"], ForEachActivity)
+    assert isinstance(tasks["total"], PlaceholderActivity)
+    assert [dependency.task_key for dependency in tasks["total"].depends_on or []] == ["added"]
+    assert p.reconciliation_status == "verified_with_gaps"
+    assert any(finding["code"] == "taskflow_mapped_output_unavailable" for finding in p.not_translatable)
+
+
+def test_airflow_non_execution_metadata_does_not_create_runtime_gap():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(\n"
+        "    dag_id='metadata',\n"
+        "    tags=['demo', 'daily'],\n"
+        "    description='Customer-facing description',\n"
+        "    doc_md='Long Airflow documentation',\n"
+        "    default_args={'owner': 'data-platform'},\n"
+        ") as dag:\n"
+        "    work = BashOperator(task_id='work', bash_command='echo work')\n"
+    )
+
+    assert p.reconciliation_status == "verified"
+    assert p.description == "Customer-facing description"
+    assert p.tags["airflow_tag_1"] == "demo"
+    assert p.tags["airflow_tag_2"] == "daily"
+    assert p.tags["airflow_owner"] == "data-platform"
+    assert any(
+        item["code"] == "dag_setting_ignored" and item["setting"] == "doc_md" for item in p.audit["transformations"]
+    )
+    assert not any(finding["code"] == "unsupported_dag_setting" for finding in p.not_translatable)
+
+
+def test_positional_dag_id_is_preserved_as_job_identity_metadata():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG('positional_dag') as dag:\n"
+        "    work = BashOperator(task_id='work', bash_command='echo work')\n"
+    )
+
+    assert p.name == "positional_dag"
+    assert p.tags["dag_id"] == "positional_dag"
+
+
+def test_airflow_tags_respect_the_databricks_job_tag_limit():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        f"with DAG(dag_id='many_tags', tags={[f'tag_{index}' for index in range(30)]!r}, "
+        "catchup=True, default_args={'owner': 'data-platform'}) as dag:\n"
+        "    work = BashOperator(task_id='work', bash_command='echo work')\n"
+    )
+
+    assert len(p.tags) == 25
+    assert p.tags["source"] == "airflow"
+    assert p.tags["dag_id"] == "many_tags"
+    assert p.tags["airflow_catchup"] == "true"
+    assert p.tags["airflow_owner"] == "data-platform"
+    assert any(
+        item["code"] == "dag_setting_partially_mapped" and item["setting"] == "tags"
+        for item in p.audit["transformations"]
+    )
+
+
 def test_taskflow_partial_expand_gap_carries_the_mapping_call():
     # .partial() fixed args can't ride on a for_each inner task, so the task becomes a placeholder --
     # but the gap must carry the mapping call, or the fixed argument values are lost and the agentic
diff --git a/tests/unit/test_airflow_reconciliation.py b/tests/unit/test_airflow_reconciliation.py
index e46623d..254aa41 100644
--- a/tests/unit/test_airflow_reconciliation.py
+++ b/tests/unit/test_airflow_reconciliation.py
@@ -220,6 +220,55 @@ def reverse_edge(self, upstreams, downstreams, node):
     assert finding["details"]["captured_edges"] == [["second", "first"]]
 
 
+def test_label_edge_modifier_preserves_every_dependency_segment(tmp_path: Path) -> None:
+    dag_path = tmp_path / "labels.py"
+    dag_path.write_text(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "from airflow.utils.edgemodifier import Label\n"
+        "with DAG(dag_id='labels') as dag:\n"
+        "    upstream = BashOperator(task_id='upstream', bash_command='echo upstream')\n"
+        "    downstream = BashOperator(task_id='downstream', bash_command='echo downstream')\n"
+        "    terminal = BashOperator(task_id='terminal', bash_command='echo terminal')\n"
+        "    upstream >> Label('successful path') >> downstream >> terminal\n",
+        encoding="utf-8",
+    )
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+    tasks = {task.task_key: task for task in pipeline.tasks}
+
+    assert pipeline.reconciliation_status == "verified"
+    assert [dependency.task_key for dependency in tasks["downstream"].depends_on or []] == ["upstream"]
+    assert [dependency.task_key for dependency in tasks["terminal"].depends_on or []] == ["downstream"]
+    assert pipeline.audit["audited_edge_count"] == 2
+    assert pipeline.audit["captured_edge_count"] == 2
+
+
+def test_bare_taskflow_shift_uses_source_identity_in_reconciliation(tmp_path: Path) -> None:
+    dag_path = tmp_path / "taskflow_edges.py"
+    dag_path.write_text(
+        "from airflow.decorators import dag, task\n"
+        "@task\n"
+        "def upstream():\n"
+        "    return 1\n"
+        "@task\n"
+        "def downstream():\n"
+        "    return 2\n"
+        "@dag(dag_id='taskflow_edges')\n"
+        "def build():\n"
+        "    upstream() >> downstream()\n"
+        "build()\n",
+        encoding="utf-8",
+    )
+
+    pipeline = airflow_loader.load_airflow_dag(dag_path)
+    downstream = next(task for task in pipeline.tasks if task.task_key.startswith("downstream"))
+
+    assert pipeline.reconciliation_status == "verified"
+    assert [dependency.task_key for dependency in downstream.depends_on or []] == ["upstream"]
+    assert not any(finding["code"] == "edge_identity_mismatch" for finding in pipeline.not_translatable)
+
+
 def test_captured_edge_removed_from_ir_fails_reconciliation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     dag_path = tmp_path / "missing_ir_edge.py"
     dag_path.write_text(_SIMPLE_DAG, encoding="utf-8")
@@ -331,8 +380,9 @@ def test_uninvoked_module_helper_body_does_not_create_a_task(tmp_path: Path) ->
     pipeline = airflow_loader.load_airflow_dag(dag_path)
 
     assert pipeline.reconciliation_status == "verified"
-    assert pipeline.tasks == []
     assert pipeline.audit["audited_activity_count"] == 0
+    assert [task.task_key for task in pipeline.tasks] == ["__flowx_empty_dag"]
+    assert any(item["code"] == "empty_dag_sentinel_emitted" for item in pipeline.audit["transformations"])
 
 
 def test_unresolved_construct_is_classified_in_coverage(tmp_path: Path) -> None:
diff --git a/tests/unit/test_bundle_invariants.py b/tests/unit/test_bundle_invariants.py
index 92f4606..8b52e55 100644
--- a/tests/unit/test_bundle_invariants.py
+++ b/tests/unit/test_bundle_invariants.py
@@ -24,6 +24,10 @@ def test_clean_job_has_no_findings():
     assert check_job("p", job) == []
 
 
+def test_empty_job_is_flagged():
+    assert "empty_job" in _codes(check_job("p", {"name": "p", "tasks": []}))
+
+
 def test_duplicate_job_parameter_flagged():
     job = {"parameters": [{"name": "region", "default": "us"}, {"name": "region", "default": "us"}], "tasks": []}
     assert "duplicate_job_parameter" in _codes(check_job("p", job))
diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py
index f90cb81..43eceff 100644
--- a/tests/unit/test_bundler.py
+++ b/tests/unit/test_bundler.py
@@ -64,6 +64,34 @@ def _workflow_with_secrets(name: str = "secret_workflow") -> PreparedWorkflow:
     return wf
 
 
+def test_airflow_job_metadata_is_emitted(tmp_path):
+    pipeline = Pipeline(
+        name="airflow_metadata",
+        description="Customer-facing description",
+        tags={
+            "source": "airflow",
+            "dag_id": "airflow_metadata",
+            "airflow_tag_1": "demo",
+            "airflow_owner": "data-platform",
+        },
+        tasks=[
+            NotebookActivity(
+                name="work",
+                task_key="work",
+                notebook_path="work.py",
+                generated_source="# Databricks notebook source\nprint('work')\n",
+            )
+        ],
+    )
+
+    write_bundle(prepare_workflow(pipeline), tmp_path)
+    resource = yaml.safe_load((tmp_path / "resources" / "airflow_metadata.yml").read_text(encoding="utf-8"))
+    job = resource["resources"]["jobs"]["airflow_metadata"]
+
+    assert job["description"] == "Customer-facing description"
+    assert job["tags"] == pipeline.tags
+
+
 # ---------------------------------------------------------------------------
 # Tests
 # ---------------------------------------------------------------------------

From 6d68bd415916daeaf89a1ad563ece59e94349b77 Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Tue, 11 Aug 2026 14:59:07 -0700
Subject: [PATCH 67/77] fix(airflow): preserve runtime policies and pin
 provider v0.2.2

---
 README.md                                     |   2 +-
 scripts/sync_airflow_provider.py              |   4 +-
 .../flowx-convert/sources/airflow-coverage.md |   4 +-
 skills/flowx-resolve-airflow-gaps/SKILL.md    |   4 +-
 .../providers/flowx-gap-resolver/PROFILE.md   |   2 +-
 .../fixtures/gap-deferred.json                |   2 +-
 .../fixtures/gap-needs-input.json             |   2 +-
 .../fixtures/gap-notebook.json                |   2 +-
 .../fixtures/gap-spark-python.json            |   2 +-
 .../flowx-gap-resolver/fixtures/gap-sql.json  |   2 +-
 .../fixtures/resolution-deferred.json         |   2 +-
 .../fixtures/resolution-needs-input.json      |   2 +-
 .../fixtures/resolution-notebook.json         |   2 +-
 .../fixtures/resolution-spark-python.json     |   2 +-
 .../fixtures/resolution-sql.json              |   2 +-
 .../flowx-gap-resolver/provider.json          |   8 +-
 .../references/airflow3-migration.md          |   0
 .../references/dab-schema-reference.md        |   0
 .../references/hadoop-migration-guide.md      |   0
 .../references/lakeflow-connect.md            |   0
 .../references/operator-mapping.md            |   0
 .../references/schedule-trigger-mapping.md    |  17 +-
 .../references/contract-v1.md                 |   4 +-
 src/flowx/agentic.py                          |  10 +-
 src/flowx/bundler/dab_writer.py               |  33 +++
 src/flowx/ir_serde.py                         |   8 +
 src/flowx/models/ir.py                        |   4 +
 src/flowx/motifs/collapser.py                 |   7 +
 src/flowx/preparer/workflow_preparer.py       |   4 +
 src/flowx/sources/airflow/loader.py           | 250 +++++++++++++++++-
 src/flowx/sources/airflow/templating.py       |  84 ++++--
 tests/unit/test_airflow_agentic_resolution.py |  14 +-
 tests/unit/test_airflow_operators.py          | 159 +++++++++++
 tests/unit/test_airflow_provider_sync.py      |   8 +-
 tests/unit/test_bundler.py                    |  17 ++
 tests/unit/test_param_dedup.py                |  36 +++
 tests/unit/test_reporting_results.py          |   2 +-
 37 files changed, 626 insertions(+), 75 deletions(-)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/PROFILE.md (99%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/fixtures/gap-deferred.json (98%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/fixtures/gap-needs-input.json (99%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/fixtures/gap-notebook.json (99%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/fixtures/gap-spark-python.json (98%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/fixtures/gap-sql.json (99%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/fixtures/resolution-deferred.json (98%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json (98%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/fixtures/resolution-notebook.json (99%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json (98%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/fixtures/resolution-sql.json (98%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/providers/flowx-gap-resolver/provider.json (90%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/references/airflow3-migration.md (100%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/references/dab-schema-reference.md (100%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/references/hadoop-migration-guide.md (100%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/references/lakeflow-connect.md (100%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/references/operator-mapping.md (100%)
 rename skills/flowx-resolve-airflow-gaps/references/{airflow-to-dabs-v0.2.1 => airflow-to-dabs}/references/schedule-trigger-mapping.md (91%)

diff --git a/README.md b/README.md
index 5bf2400..a9e23bd 100644
--- a/README.md
+++ b/README.md
@@ -188,7 +188,7 @@ execution) and maps ~35 operator/sensor families to the shared IR. Highlights:
   `params={...}` → job parameters, `>>` / `<<` / `set_upstream` / TaskGroup edges.
 
 Operators without a deterministic mapping become a failing placeholder and are recorded in
-`gaps.json` for review. Eligible leaf gaps can use the fingerprint-bound resolver backed by the pinned [`airflow-to-dabs` v0.2.1](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.1) provider profile; flowx retains ownership of parsing, graph identity, policy, IR, and packaging. Full matrix:
+`gaps.json` for review. Eligible leaf gaps can use the fingerprint-bound resolver backed by the pinned [`airflow-to-dabs` v0.2.2](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.2) provider profile; flowx retains ownership of parsing, graph identity, policy, IR, and packaging. Full matrix:
 [`skills/flowx-convert/sources/airflow-coverage.md`](skills/flowx-convert/sources/airflow-coverage.md).
 
 Airflow discovery independently audits DAG declarations, task candidates, dependency declarations,
diff --git a/scripts/sync_airflow_provider.py b/scripts/sync_airflow_provider.py
index e38bae2..12d5aa3 100644
--- a/scripts/sync_airflow_provider.py
+++ b/scripts/sync_airflow_provider.py
@@ -14,7 +14,7 @@
 from typing import Any
 
 REPOSITORY = "https://github.com/park-peter/airflow-to-dabs"
-DEFAULT_TAG = "v0.2.1"
+DEFAULT_TAG = "v0.2.2"
 PROVIDER_PATH = PurePosixPath("providers/flowx-gap-resolver/provider.json")
 PIN_FIELD = "flowx_pin"
 
@@ -186,7 +186,7 @@ def main() -> int:
         / "skills"
         / "flowx-resolve-airflow-gaps"
         / "references"
-        / f"airflow-to-dabs-{DEFAULT_TAG}",
+        / "airflow-to-dabs",
     )
     parser.add_argument(
         "--check", action="store_true", help="Verify the committed provider pin without network access."
diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md
index 760eaf5..958877a 100644
--- a/skills/flowx-convert/sources/airflow-coverage.md
+++ b/skills/flowx-convert/sources/airflow-coverage.md
@@ -37,6 +37,7 @@ Airflow, and never executes a DAG. Anything the static walk can't see, it can't
 | Schedule | Cron `schedule_interval` → Quartz (Unix DOW 0–6 → Quartz 1–7); exact sub-hour `timedelta` → Quartz, longer intervals → periodic, `@continuous` → continuous mode. Airflow 3 Asset/Dataset lists and uniform `&` / `|` expressions map to `ALL_UPDATED` / `ANY_UPDATED` table triggers when each asset declares `extra={"databricks_table": "catalog.schema.table"}` or an `x-databricks-table:` URI. |
 | `trigger_rule` | Exact supported rules map to `run_if`; `none_failed_min_one_success` and its legacy `none_failed_or_skipped` spelling map to `NONE_FAILED` with the all-skipped delta recorded. Rules without an equivalent become linked placeholders. |
 | Job parameters | `params={...}` / `Param(default=...)` → job parameters with defaults. User `params.x` keeps the name `x`; logical-date macros, `var.value.x`, `dag_run.conf['x']`, and `run_id` use collision-free `__flowx_airflow_*` bindings. User parameter names beginning with `__flowx_` become explicit gaps. |
+| Job policy | Static positive `dagrun_timeout` → Job `timeout_seconds`; static failure recipients → Job `email_notifications.on_failure`. Explicitly disabled `depends_on_past`, retry/failure email, SLA callback, auto-pause, and empty environment settings are recorded as intentional no-ops. |
 | `Variable.get` in a callable | `Variable.get('literal_name')` is rewritten to a collision-free `__flowx_airflow_variable_*` widget. Dynamic keys, Airflow defaults/deserialization options, other Airflow runtime imports, and Airflow `Connection` objects route to placeholders rather than emitting notebooks that require Airflow. |
 | Multiple DAGs | Every DAG, including multiple declarations and repeated static `@dag` factory invocations in one Python file, becomes a sibling job in one shared Airflow bundle so `TriggerDagRunOperator` resource references resolve. Narrow classic factories shaped as one DAG declaration followed by `return dag` are expanded with statically bindable arguments. |
 
@@ -46,7 +47,7 @@ safe fallback is a flagged, failing task rather than a silent omission. Callable
 (`**context` / `ti`) or XCom, and runtime-branching decorators, take the same route rather than
 emitting code that fails at runtime.
 
-The resolver consumes the pinned `airflow-to-dabs` v0.2.1 Flowx provider profile. It receives one flowx-produced gap envelope and cannot express graph or task-policy changes. Accepted `resolved` candidates contribute to mechanically validated code-attached coverage, but remain agentic and do not increase deterministic coverage. `needs_input`, `deferred`, and unreviewed candidates remain linked failing placeholders.
+The resolver consumes the pinned `airflow-to-dabs` v0.2.2 Flowx provider profile. It receives one flowx-produced gap envelope and cannot express graph or task-policy changes. Accepted `resolved` candidates contribute to mechanically validated code-attached coverage, but remain agentic and do not increase deterministic coverage. `needs_input`, `deferred`, and unreviewed candidates remain linked failing placeholders.
 
 ## Not yet supported
 
@@ -66,6 +67,7 @@ decisions.
   package output rather than emitting a filename-derived empty Job.
 - **Unresolved Airflow 3 schedules** — `AssetOrTimeSchedule`, mixed Asset boolean expressions, custom timetables, and Assets without explicit Databricks table metadata become `AirflowSourceSemantics` gaps. Job-level trigger and schedule changes are outside the leaf-only agentic contract.
 - **Ambiguous Airflow 1.10 schedule defaults** — assigned DAGs and legacy imports are supported, but a DAG using strong 1.10 syntax that omits `schedule_interval` becomes an `AirflowSourceSemantics` gap because historical default schedule and catchup behavior cannot be inferred safely from source alone.
+- **Cross-run and operational policy** — active `depends_on_past`, `max_consecutive_failed_dag_runs`, `sla_miss_callback`, retry-email events, dynamic `dagrun_timeout`, and non-empty `default_args.env` have no exact leaf-only Jobs mapping. Each becomes a source-semantics placeholder with a setting-specific remediation message; failure recipients and any independently representable Job timeout remain preserved.
 - **Unsafe inline template contexts** — SQL Jinja embedded in a string, quoted identifier, typed literal, or adjacent identifier fragment and shell Jinja in a non-expanding quoted heredoc, ANSI-C quote, or escaped position route to a placeholder instead of emitting a value with changed lexical semantics.
 - **Sensors beyond the mapped families** (`S3PrefixSensor`, custom sensors, etc.) → placeholder + gap.
   A file sensor with a non-literal path, or a table/SQL sensor with no literal `sql` / `table_name`,
diff --git a/skills/flowx-resolve-airflow-gaps/SKILL.md b/skills/flowx-resolve-airflow-gaps/SKILL.md
index 538db74..60bb28d 100644
--- a/skills/flowx-resolve-airflow-gaps/SKILL.md
+++ b/skills/flowx-resolve-airflow-gaps/SKILL.md
@@ -5,9 +5,9 @@ description: Resolve source-reconciled Airflow leaf gaps through the fingerprint
 
 # Resolve Airflow Leaf Gaps
 
-Use this workflow only for Airflow reports whose deterministic conversion succeeded with gaps. Flowx owns source parsing, task identity, dependencies, task policy, IR, and packaging. This skill reasons about one prepared gap at a time using the migration knowledge from [`park-peter/airflow-to-dabs` v0.2.1](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.1). It must not parse the DAG independently or generate a second bundle.
+Use this workflow only for Airflow reports whose deterministic conversion succeeded with gaps. Flowx owns source parsing, task identity, dependencies, task policy, IR, and packaging. This skill reasons about one prepared gap at a time using the migration knowledge from [`park-peter/airflow-to-dabs` v0.2.2](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.2). It must not parse the DAG independently or generate a second bundle.
 
-Read [`references/contract-v1.md`](references/contract-v1.md) and the pinned [`airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md`](references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md) before authoring a resolution. The profile and every referenced knowledge file are vendored from the exact upstream tag and commit under `references/airflow-to-dabs-v0.2.1/`. If required knowledge is unavailable, return `needs_input` or `deferred`; never infer missing operator semantics.
+Read [`references/contract-v1.md`](references/contract-v1.md) and the pinned [`airflow-to-dabs/providers/flowx-gap-resolver/PROFILE.md`](references/airflow-to-dabs/providers/flowx-gap-resolver/PROFILE.md) before authoring a resolution. The profile and every referenced knowledge file are vendored from the exact upstream tag and commit under `references/airflow-to-dabs/`. If required knowledge is unavailable, return `needs_input` or `deferred`; never infer missing operator semantics.
 
 ## 1. Prepare immutable gap envelopes
 
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/PROFILE.md
similarity index 99%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/PROFILE.md
index 0942dbf..7c9560c 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/PROFILE.md
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/PROFILE.md
@@ -9,7 +9,7 @@ This profile implements flowx Airflow agentic gap contract `1` with the pinned p
 ```json
 {
   "name": "airflow-to-dabs",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "repository": "https://github.com/park-peter/airflow-to-dabs"
 }
 ```
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-deferred.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-deferred.json
similarity index 98%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-deferred.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-deferred.json
index bd4cc76..f823482 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-deferred.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-deferred.json
@@ -40,7 +40,7 @@
   "knowledge_provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.1"
+    "version": "0.2.2"
   },
   "operator": "BranchPythonOperator",
   "operator_fqn": "airflow.operators.python.BranchPythonOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-needs-input.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-needs-input.json
similarity index 99%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-needs-input.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-needs-input.json
index 2b4e564..46a892f 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-needs-input.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-needs-input.json
@@ -49,7 +49,7 @@
   "knowledge_provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.1"
+    "version": "0.2.2"
   },
   "operator": "KubernetesPodOperator",
   "operator_fqn": "airflow.providers.cncf.kubernetes.operators.pod.KubernetesPodOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-notebook.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-notebook.json
similarity index 99%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-notebook.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-notebook.json
index d29ade1..d812c4e 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-notebook.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-notebook.json
@@ -47,7 +47,7 @@
   "knowledge_provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.1"
+    "version": "0.2.2"
   },
   "operator": "SimpleHttpOperator",
   "operator_fqn": "airflow.providers.http.operators.http.SimpleHttpOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-spark-python.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-spark-python.json
similarity index 98%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-spark-python.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-spark-python.json
index 92a1241..d9154bf 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-spark-python.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-spark-python.json
@@ -41,7 +41,7 @@
   "knowledge_provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.1"
+    "version": "0.2.2"
   },
   "operator": "CustomPythonOperator",
   "operator_fqn": "company.airflow.operators.CustomPythonOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-sql.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-sql.json
similarity index 99%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-sql.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-sql.json
index 91ee4ff..887498f 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/gap-sql.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-sql.json
@@ -44,7 +44,7 @@
   "knowledge_provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.1"
+    "version": "0.2.2"
   },
   "operator": "SQLExecuteQueryOperator",
   "operator_fqn": "airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-deferred.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-deferred.json
similarity index 98%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-deferred.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-deferred.json
index ecf4913..ab85c15 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-deferred.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-deferred.json
@@ -27,7 +27,7 @@
   "provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.1"
+    "version": "0.2.2"
   },
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
   "reason": "A faithful branch migration requires condition tasks and downstream dependency rewrites, which are outside the leaf-only provider contract.",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json
similarity index 98%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json
index 94a0f81..fc95779 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json
@@ -37,7 +37,7 @@
   "provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.1"
+    "version": "0.2.2"
   },
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
   "reason": "Provide the container source or packaged application, required Python/system dependencies, registry access requirements, and the Databricks secret or Unity Catalog mappings for orders_secret.",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-notebook.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-notebook.json
similarity index 99%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-notebook.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-notebook.json
index 17032ed..520810d 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-notebook.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-notebook.json
@@ -47,7 +47,7 @@
   "provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.1"
+    "version": "0.2.2"
   },
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
   "replacement": {
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json
similarity index 98%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json
index b8f7948..96d532a 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json
@@ -31,7 +31,7 @@
   "provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.1"
+    "version": "0.2.2"
   },
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
   "replacement": {
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-sql.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-sql.json
similarity index 98%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-sql.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-sql.json
index 53b33ba..bac1dcd 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/fixtures/resolution-sql.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-sql.json
@@ -42,7 +42,7 @@
   "provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.1"
+    "version": "0.2.2"
   },
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
   "replacement": {
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/provider.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/provider.json
similarity index 90%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/provider.json
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/provider.json
index 993fb7f..8881651 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/providers/flowx-gap-resolver/provider.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/provider.json
@@ -12,11 +12,11 @@
     "fixtures/resolution-deferred.json"
   ],
   "flowx_pin": {
-    "commit": "75196aef85ebb2736b926f2d4db13ec7d5c2551c",
-    "content_sha256": "e1e7395204b3f2759722b9320cea08c6b58284a48fd8062686b36bf676b657fd",
+    "commit": "6a940cbb11ac2edd8e028853865f386002a003f0",
+    "content_sha256": "46215e950028f31a157f68fd7a9dade78e3cd7486b59217f17d9811e3b73618e",
     "contract_version": "1",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "tag": "v0.2.1"
+    "tag": "v0.2.2"
   },
   "interface": {
     "contract_versions": [
@@ -66,6 +66,6 @@
   "provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.1"
+    "version": "0.2.2"
   }
 }
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/airflow3-migration.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/airflow3-migration.md
similarity index 100%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/airflow3-migration.md
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/airflow3-migration.md
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/dab-schema-reference.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/dab-schema-reference.md
similarity index 100%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/dab-schema-reference.md
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/dab-schema-reference.md
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/hadoop-migration-guide.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/hadoop-migration-guide.md
similarity index 100%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/hadoop-migration-guide.md
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/hadoop-migration-guide.md
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/lakeflow-connect.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/lakeflow-connect.md
similarity index 100%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/lakeflow-connect.md
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/lakeflow-connect.md
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/operator-mapping.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/operator-mapping.md
similarity index 100%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/operator-mapping.md
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/operator-mapping.md
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/schedule-trigger-mapping.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/schedule-trigger-mapping.md
similarity index 91%
rename from skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/schedule-trigger-mapping.md
rename to skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/schedule-trigger-mapping.md
index 620b1fb..cb0d608 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs-v0.2.1/references/schedule-trigger-mapping.md
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/schedule-trigger-mapping.md
@@ -234,16 +234,25 @@ Common `default_args` fields and their DABs equivalents:
 | `owner` | *(no direct mapping -- do not auto-map identity; document intended run identity in MIGRATION_NOTES.md)* |
 | `retries` | `max_retries` on task |
 | `retry_delay` | `min_retry_interval_millis` on task |
-| `email` | `email_notifications.on_failure` |
-| `email_on_failure` | `email_notifications.on_failure` |
-| `email_on_retry` | *(no direct equivalent, note in migration notes)* |
-| `depends_on_past` | *(no direct equivalent, note in migration notes)* |
+| `email` | Job `email_notifications.on_failure`; record a delta when Airflow used task-level or SLA delivery because a Databricks run can finish as succeeded-with-failures without sending `on_failure`. |
+| `email_on_failure` | Job `email_notifications.on_failure`, subject to the task-vs-job delivery delta above. |
+| `email_on_retry` | `False` is a no-op; active retry notifications have no Jobs event equivalent and must be noted in migration notes. Preserve any independently enabled failure notification. |
+| `depends_on_past` | `False` is a no-op; `True` has no cross-run task dependency equivalent and must be noted in migration notes. `max_concurrent_runs: 1` prevents overlap but does not preserve prior-run success semantics. |
+| `env` | An empty mapping is a no-op. Otherwise bind each value explicitly into the migrated task and move credentials or connection-derived values to Databricks secrets; never inline them. |
 | `start_date` | *(not needed -- DABs jobs start when deployed)* |
 | `end_date` | *(no direct equivalent -- pause the schedule manually)* |
 | `execution_timeout` | `timeout_seconds` on task |
 | `sla` | *(no direct equivalent -- use monitoring/alerts)* |
 | `catchup` | `catchup=True` → use native [Databricks backfill](https://docs.databricks.com/aws/en/jobs/backfill-jobs) to replay history (requires `{{ ds }}` mapped to a job parameter — see the execution-date section); `catchup=False` (the Airflow 3 default) → no backfill. Note the expectation in MIGRATION_NOTES.md. |
 
+Related DAG-level settings:
+
+| Airflow DAG setting | DABs Equivalent |
+|---|---|
+| `dagrun_timeout` | A static positive `timedelta` maps to Job `timeout_seconds`. Dynamic values require manual resolution. |
+| `sla_miss_callback` | `None` is a no-op. An active arbitrary callback has no direct mapping; configure a Job `health.rules` `RUN_DURATION_SECONDS` threshold plus email/webhook notification when that preserves the intent, otherwise migrate the callback explicitly. |
+| `max_consecutive_failed_dag_runs` | `0` is a no-op. A positive automatic-pause threshold has no Jobs equivalent and requires external monitoring/control; do not substitute `max_concurrent_runs`, which governs overlap rather than failure history. |
+
 ---
 
 ## `trigger_rule` → `run_if` Mapping
diff --git a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
index 9cc9b06..1591c76 100644
--- a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
+++ b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
@@ -19,7 +19,7 @@ The provider receives a `GapEnvelope` produced by flowx. It does not receive aut
   "request_sha256": "copied from the envelope",
   "provider": {
     "name": "airflow-to-dabs",
-    "version": "0.2.1",
+    "version": "0.2.2",
     "repository": "https://github.com/park-peter/airflow-to-dabs"
   },
   "model": {"name": "model identifier"},
@@ -61,4 +61,4 @@ Spark Python uses `{"kind": "spark_python", "file": "task.py", "parameters": ["-
 - Notebook `base_parameters` keys must use letters, digits, underscores, dots, and hyphens, starting with a letter or underscore. Flowx-owned names beginning with `__flowx` and Databricks task identity, graph, policy, and task-type field names are reserved case-insensitively.
 - Airflow Jinja is rejected. Databricks dynamic references such as `{{job.parameters.x}}`, `{{tasks.upstream.values.x}}`, and `{{input}}` are valid in replacement parameter values, not in uploaded notebook or SQL source files; source files must read widgets or SQL named parameters.
 - Every source argument in the envelope has exactly one disposition and a non-empty rationale.
-- The provider identity must match the pinned `airflow-to-dabs` v0.2.1 knowledge release.
+- The provider identity must match the pinned `airflow-to-dabs` v0.2.2 knowledge release.
diff --git a/src/flowx/agentic.py b/src/flowx/agentic.py
index 2324b72..ff60317 100644
--- a/src/flowx/agentic.py
+++ b/src/flowx/agentic.py
@@ -24,7 +24,7 @@
 
 CONTRACT_VERSION = "1"
 PROVIDER_NAME = "airflow-to-dabs"
-PROVIDER_VERSION = "0.2.1"
+PROVIDER_VERSION = "0.2.2"
 PROVIDER_REPOSITORY = "https://github.com/park-peter/airflow-to-dabs"
 
 _ALLOWED_REPLACEMENT_KINDS = ("notebook", "sql", "spark_python")
@@ -872,6 +872,8 @@ def _build_gap_envelopes(
                 downstream_task_keys=downstream.get(str(task["task_key"]), []),
                 dag_settings={
                     "schedule": pipeline.get("schedule"),
+                    "timeout_seconds": pipeline.get("timeout_seconds"),
+                    "email_notifications": pipeline.get("email_notifications"),
                     "parameters": pipeline.get("parameters"),
                     "tags": pipeline.get("tags"),
                     "description": pipeline.get("description"),
@@ -1524,11 +1526,7 @@ def _copy_provider_context(destination: Path) -> None:
 
 def _provider_context_path() -> Path:
     source = (
-        Path(__file__).resolve().parents[2]
-        / "skills"
-        / "flowx-resolve-airflow-gaps"
-        / "references"
-        / f"airflow-to-dabs-v{PROVIDER_VERSION}"
+        Path(__file__).resolve().parents[2] / "skills" / "flowx-resolve-airflow-gaps" / "references" / "airflow-to-dabs"
     )
     if not source.is_dir():
         raise AgenticContractError(
diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py
index f238571..08e300a 100644
--- a/src/flowx/bundler/dab_writer.py
+++ b/src/flowx/bundler/dab_writer.py
@@ -1563,6 +1563,12 @@ def _build_job_resource(
         job_def["description"] = workflow.description
     if workflow.tags:
         job_def["tags"] = dict(workflow.tags)
+    if workflow.timeout_seconds is not None:
+        job_def["timeout_seconds"] = workflow.timeout_seconds
+    if workflow.email_notifications:
+        job_def["email_notifications"] = {
+            event: list(recipients) for event, recipients in workflow.email_notifications.items()
+        }
 
     if attach_clusters:
         _bind_cluster_to_notebook_tasks(workflow.tasks)
@@ -1900,6 +1906,31 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d
             else:
                 entry["default"] = normalize_value(str(default_value))
         parameters.append(entry)
+    timeout_seconds = pipeline_dict.get("timeout_seconds")
+    if timeout_seconds is not None and (
+        isinstance(timeout_seconds, bool) or not isinstance(timeout_seconds, int) or timeout_seconds <= 0
+    ):
+        raise ValueError("Pipeline timeout_seconds must be a positive integer")
+    raw_email_notifications = pipeline_dict.get("email_notifications") or {}
+    if not isinstance(raw_email_notifications, dict):
+        raise ValueError("Pipeline email_notifications must be an object")
+    email_notifications: dict[str, list[str]] = {}
+    allowed_email_events = {
+        "on_start",
+        "on_success",
+        "on_failure",
+        "on_duration_warning_threshold_exceeded",
+        "on_streaming_backlog_exceeded",
+    }
+    for event, recipients in raw_email_notifications.items():
+        if (
+            event not in allowed_email_events
+            or not isinstance(recipients, list)
+            or not all(isinstance(recipient, str) and recipient for recipient in recipients)
+        ):
+            raise ValueError(f"Invalid Pipeline email notification entry: {event!r}")
+        email_notifications[str(event)] = list(recipients)
+
     pipeline = Pipeline(
         name=pipeline_dict.get("name", "unknown"),
         description=pipeline_dict.get("description"),
@@ -1907,6 +1938,8 @@ def pipeline_dict_to_ir(pipeline_dict: dict[str, Any]) -> tuple[Pipeline, list[d
         parameters=parameters or None,
         translation_configuration=_reconstruct_configuration(pipeline_dict.get("translation_configuration")),
         schedule=pipeline_dict.get("schedule"),
+        timeout_seconds=timeout_seconds,
+        email_notifications=email_notifications,
         tags=dict(pipeline_dict.get("tags") or {}),
         not_translatable=list(pipeline_dict.get("not_translatable") or []),
         reconciliation_status=pipeline_dict.get("reconciliation_status"),
diff --git a/src/flowx/ir_serde.py b/src/flowx/ir_serde.py
index 09b25e6..aac0896 100644
--- a/src/flowx/ir_serde.py
+++ b/src/flowx/ir_serde.py
@@ -70,6 +70,12 @@ def pipeline_to_dict(pipeline: Pipeline) -> dict[str, Any]:
     }
     if pipeline.description is not None:
         result["description"] = pipeline.description
+    if pipeline.timeout_seconds is not None:
+        result["timeout_seconds"] = pipeline.timeout_seconds
+    if pipeline.email_notifications:
+        result["email_notifications"] = {
+            event: list(recipients) for event, recipients in pipeline.email_notifications.items()
+        }
     if pipeline.translation_configuration is not None:
         result["translation_configuration"] = configuration_to_dict(pipeline.translation_configuration)
     return result
@@ -422,6 +428,8 @@ def pipeline_to_debug_dict(pipeline: Pipeline) -> dict[str, Any]:
         "description": pipeline.description,
         "parameters": pipeline.parameters,
         "schedule": pipeline.schedule,
+        "timeout_seconds": pipeline.timeout_seconds,
+        "email_notifications": pipeline.email_notifications,
         "tags": pipeline.tags,
         "tasks": [activity_to_debug_dict(task) for task in pipeline.tasks],
         "not_translatable": list(pipeline.not_translatable),
diff --git a/src/flowx/models/ir.py b/src/flowx/models/ir.py
index 373b626..d6f45fc 100644
--- a/src/flowx/models/ir.py
+++ b/src/flowx/models/ir.py
@@ -609,6 +609,8 @@ class Pipeline:
         description: Human-readable workflow description.
         parameters: Pipeline parameter definitions.
         schedule: Serialized schedule definition, if any.
+        timeout_seconds: Maximum execution time for one workflow run.
+        email_notifications: Job-level email recipients grouped by notification event.
         tasks: Ordered list of translated activities.
         tags: System and user-defined tags.
         not_translatable: Entries describing properties that could not be translated.
@@ -621,6 +623,8 @@ class Pipeline:
     description: str | None = None
     parameters: list[dict[str, Any]] | None = None
     schedule: dict[str, Any] | None = None
+    timeout_seconds: int | None = None
+    email_notifications: dict[str, list[str]] = field(default_factory=dict)
     tasks: list[Activity] = field(default_factory=list)
     tags: dict[str, str] = field(default_factory=dict)
     not_translatable: list[dict[str, Any]] = field(default_factory=list)
diff --git a/src/flowx/motifs/collapser.py b/src/flowx/motifs/collapser.py
index 8dede0a..4923000 100644
--- a/src/flowx/motifs/collapser.py
+++ b/src/flowx/motifs/collapser.py
@@ -65,11 +65,18 @@ def collapse_motifs(
 
     return Pipeline(
         name=pipeline.name,
+        description=pipeline.description,
         parameters=pipeline.parameters,
         schedule=pipeline.schedule,
+        timeout_seconds=pipeline.timeout_seconds,
+        email_notifications=dict(pipeline.email_notifications),
         tasks=new_tasks,
         tags=pipeline.tags,
         not_translatable=pipeline.not_translatable,
+        reconciliation_status=pipeline.reconciliation_status,
+        migration_status=pipeline.migration_status,
+        audit=dict(pipeline.audit),
+        translation_configuration=pipeline.translation_configuration,
     )
 
 
diff --git a/src/flowx/preparer/workflow_preparer.py b/src/flowx/preparer/workflow_preparer.py
index 78618ad..27a9673 100644
--- a/src/flowx/preparer/workflow_preparer.py
+++ b/src/flowx/preparer/workflow_preparer.py
@@ -68,6 +68,8 @@ class PreparedWorkflow:
     # C-10 (SCHED-001): serialised schedule / trigger spec the bundler
     # renders as ``schedule:`` / ``trigger:`` on the emitted DAB job.
     schedule: dict[str, Any] | None = None
+    timeout_seconds: int | None = None
+    email_notifications: dict[str, list[str]] = field(default_factory=dict)
     source: str | None = None
     description: str | None = None
     tags: dict[str, str] = field(default_factory=dict)
@@ -425,6 +427,8 @@ def prepare_workflow(pipeline: Pipeline) -> PreparedWorkflow:
         pipeline_resources=list(artifacts.pipeline_resources),
         parameter_approximations=list(artifacts.parameter_approximations),
         schedule=pipeline.schedule,
+        timeout_seconds=pipeline.timeout_seconds if is_airflow else None,
+        email_notifications=dict(pipeline.email_notifications) if is_airflow else {},
         source=str(pipeline.tags.get("source")) if pipeline.tags.get("source") else None,
         description=pipeline.description if is_airflow else None,
         tags=dict(pipeline.tags) if is_airflow else {},
diff --git a/src/flowx/sources/airflow/loader.py b/src/flowx/sources/airflow/loader.py
index afb08c0..add62f9 100644
--- a/src/flowx/sources/airflow/loader.py
+++ b/src/flowx/sources/airflow/loader.py
@@ -712,6 +712,7 @@ def __init__(self, module: ast.Module, *, target_dag_variable: str | None = None
         self._claimed_statement_ids: set[int] = set()
         self._dag_scope_depth = 0
         self.captured_dag_settings: set[str] = set()
+        self.dag_kwargs: dict[str, ast.expr] = {}
         # task variable name -> (task_id, operator, kwargs)
         self.operators: dict[str, tuple[str, str, dict[str, ast.expr]]] = {}
         # task variable name -> the operator's ast.Call node (for source-slicing placeholders)
@@ -1367,6 +1368,7 @@ def _read_dag_kwargs(self, call: ast.Call) -> None:
             self.unresolved_constructs.append(("ambiguous_airflow_1_10_default_schedule", call))
 
     def _apply_dag_kwargs(self, kwargs: dict[str, ast.expr]) -> None:
+        self.dag_kwargs.update(kwargs)
         self.captured_dag_settings.update(kwargs)
         self.schedule_node = kwargs.get("schedule_interval") or kwargs.get("schedule")
         self.schedule_interval = ops.literal_str(kwargs.get("schedule_interval")) or ops.literal_str(
@@ -2794,6 +2796,8 @@ def append_task(activity: Activity, capture_id: str) -> None:
         tasks=tasks,
         parameters=parameters,
         schedule=schedule,
+        timeout_seconds=_job_timeout_seconds(visitor),
+        email_notifications=_job_email_notifications(visitor),
         tags=tags,
     )
     return _reconcile_pipeline(
@@ -2813,7 +2817,7 @@ def append_task(activity: Activity, capture_id: str) -> None:
     )
 
 
-_SUPPORTED_DAG_SETTINGS = frozenset(
+_RECOGNIZED_DAG_SETTINGS = frozenset(
     {
         "dag_id",
         "schedule",
@@ -2826,6 +2830,14 @@ def append_task(activity: Activity, capture_id: str) -> None:
         "default_args.retries",
         "default_args.retry_delay",
         "default_args.execution_timeout",
+        "dagrun_timeout",
+        "max_consecutive_failed_dag_runs",
+        "sla_miss_callback",
+        "default_args.depends_on_past",
+        "default_args.email",
+        "default_args.email_on_failure",
+        "default_args.email_on_retry",
+        "default_args.env",
         "tags",
         "description",
         "doc_md",
@@ -2835,6 +2847,206 @@ def append_task(activity: Activity, capture_id: str) -> None:
 )
 
 
+def _job_timeout_seconds(visitor: _DagVisitor) -> int | None:
+    return templating.timedelta_seconds(visitor.dag_kwargs.get("dagrun_timeout"))
+
+
+def _job_email_notifications(visitor: _DagVisitor) -> dict[str, list[str]]:
+    recipients = templating.literal_email_recipients(visitor.default_args.get("email"))
+    on_failure = visitor.default_args.get("email_on_failure")
+    failure_enabled = on_failure is None or (isinstance(on_failure, ast.Constant) and on_failure.value is True)
+    if recipients and failure_enabled:
+        return {"on_failure": recipients}
+    return {}
+
+
+def _retry_email_is_active(visitor: _DagVisitor) -> bool:
+    """Returns whether any captured task can emit an Airflow retry email."""
+    for _, _, kwargs in visitor.operators.values():
+        retry_node = kwargs.get("retries", visitor.default_args.get("retries"))
+        if retry_node is None:
+            continue
+        retry_count = ops.literal_value(retry_node)
+        if isinstance(retry_count, int) and not isinstance(retry_count, bool) and retry_count <= 0:
+            continue
+        on_retry = kwargs.get("email_on_retry", visitor.default_args.get("email_on_retry"))
+        if isinstance(on_retry, ast.Constant) and on_retry.value is False:
+            continue
+        return True
+    return False
+
+
+def _dag_setting_disposition(name: str, visitor: _DagVisitor) -> dict[str, str] | None:
+    """Classifies recognized DAG settings as mapped, intentional no-ops, or runtime gaps."""
+    if name not in _RECOGNIZED_DAG_SETTINGS:
+        return {
+            "status": "gap",
+            "message": f"Airflow DAG setting {name!r} has no deterministic Databricks Jobs mapping.",
+            "rationale": "no_deterministic_databricks_jobs_mapping",
+        }
+    if name == "dagrun_timeout":
+        if _job_timeout_seconds(visitor) is not None:
+            return {
+                "status": "mapped",
+                "target": "job.timeout_seconds",
+                "rationale": "preserved_as_databricks_job_run_timeout",
+            }
+        return {
+            "status": "gap",
+            "message": (
+                "Airflow dagrun_timeout must be a static positive timedelta before it can map to Job timeout_seconds."
+            ),
+            "rationale": "dag_run_timeout_not_statically_resolvable",
+        }
+    if name == "default_args.depends_on_past":
+        value = visitor.default_args.get("depends_on_past")
+        if isinstance(value, ast.Constant) and value.value in {False, None}:
+            return {
+                "status": "ignored",
+                "rationale": "disabled_cross_run_dependency_has_no_runtime_effect",
+            }
+        return {
+            "status": "gap",
+            "message": (
+                "Airflow depends_on_past requires each task instance to depend on the prior DAG run; "
+                "Databricks Jobs has no equivalent cross-run task dependency."
+            ),
+            "rationale": "cross_run_task_state_not_representable",
+        }
+    if name == "max_consecutive_failed_dag_runs":
+        value = visitor.dag_kwargs.get("max_consecutive_failed_dag_runs")
+        if isinstance(value, ast.Constant) and value.value in {0, None}:
+            return {
+                "status": "ignored",
+                "rationale": "automatic_pause_after_failures_is_disabled",
+            }
+        return {
+            "status": "gap",
+            "message": (
+                "Airflow max_consecutive_failed_dag_runs can automatically pause a DAG after repeated failures; "
+                "Databricks Jobs has no equivalent automatic-pause policy."
+            ),
+            "rationale": "automatic_pause_after_consecutive_failures_not_representable",
+        }
+    if name == "sla_miss_callback":
+        value = visitor.dag_kwargs.get("sla_miss_callback")
+        if isinstance(value, ast.Constant) and value.value is None:
+            return {"status": "ignored", "rationale": "sla_callback_is_disabled"}
+        return {
+            "status": "gap",
+            "message": (
+                "Airflow sla_miss_callback executes an arbitrary SLA callback; configure a Databricks duration "
+                "health rule and notification destination or migrate the callback explicitly."
+            ),
+            "rationale": "arbitrary_sla_callback_not_representable",
+        }
+    if name == "default_args.env":
+        value = visitor.default_args.get("env")
+        if (isinstance(value, ast.Constant) and value.value is None) or (
+            isinstance(value, ast.Dict) and not value.keys
+        ):
+            return {"status": "ignored", "rationale": "empty_default_task_environment_has_no_runtime_effect"}
+        return {
+            "status": "gap",
+            "message": (
+                "Airflow default_args.env changes each task environment and may contain connection-derived values; "
+                "map every value to Databricks task parameters or secrets before migration."
+            ),
+            "rationale": "default_task_environment_requires_runtime_secret_mapping",
+        }
+    if name in {"default_args.email", "default_args.email_on_failure", "default_args.email_on_retry"}:
+        recipients = templating.literal_email_recipients(visitor.default_args.get("email"))
+        on_failure = visitor.default_args.get("email_on_failure")
+        on_retry = visitor.default_args.get("email_on_retry")
+        sla_callback = visitor.dag_kwargs.get("sla_miss_callback")
+        sla_callback_active = sla_callback is not None and not (
+            isinstance(sla_callback, ast.Constant) and sla_callback.value is None
+        )
+        failure_disabled = isinstance(on_failure, ast.Constant) and on_failure.value is False
+        retry_disabled = isinstance(on_retry, ast.Constant) and on_retry.value is False
+        retry_active = _retry_email_is_active(visitor)
+        if name == "default_args.email_on_failure":
+            if failure_disabled:
+                return {"status": "ignored", "rationale": "failure_email_notification_is_disabled"}
+            if recipients is None:
+                return {
+                    "status": "gap",
+                    "message": (
+                        "Airflow failure email settings must be static before they can map to Job email notifications."
+                    ),
+                    "rationale": "failure_email_notification_not_statically_resolvable",
+                }
+            if not recipients:
+                return {"status": "ignored", "rationale": "failure_email_has_no_recipients"}
+            if not (isinstance(on_failure, ast.Constant) and on_failure.value is True):
+                return {
+                    "status": "gap",
+                    "message": (
+                        "Airflow failure email settings must be static before they can map to Job email notifications."
+                    ),
+                    "rationale": "failure_email_notification_not_statically_resolvable",
+                }
+            return {
+                "status": "mapped",
+                "target": "job.email_notifications.on_failure",
+                "rationale": "preserved_as_databricks_job_failure_notification",
+            }
+        if name == "default_args.email_on_retry":
+            if retry_disabled or not retry_active:
+                return {"status": "ignored", "rationale": "retry_email_notification_has_no_runtime_effect"}
+            if recipients is None:
+                return {
+                    "status": "gap",
+                    "message": "Airflow retry email settings must be static before they can be migrated.",
+                    "rationale": "retry_email_notification_not_statically_resolvable",
+                }
+            if not recipients:
+                return {"status": "ignored", "rationale": "retry_email_has_no_recipients"}
+            return {
+                "status": "gap",
+                "message": (
+                    "Airflow email_on_retry sends a retry notification, but Databricks Jobs exposes start, "
+                    "success, failure, and duration notifications rather than a retry notification event."
+                ),
+                "rationale": "retry_notification_event_not_available",
+            }
+        if failure_disabled and not retry_active:
+            return {"status": "ignored", "rationale": "all_email_notification_events_are_disabled"}
+        if recipients is None:
+            return {
+                "status": "gap",
+                "message": "Airflow email recipients must be static strings before they can map to Job notifications.",
+                "rationale": "email_recipients_not_statically_resolvable",
+            }
+        if retry_active:
+            return {
+                "status": "gap",
+                "message": (
+                    "Airflow retry email notifications have no Databricks retry event; failure recipients were "
+                    "preserved, but the retry notification still requires an explicit replacement."
+                ),
+                "rationale": "email_recipients_include_unrepresented_retry_event",
+            }
+        if failure_disabled or not recipients:
+            return {"status": "ignored", "rationale": "email_recipients_have_no_enabled_notification_event"}
+        if sla_callback_active:
+            return {
+                "status": "gap",
+                "target": "job.email_notifications.on_failure",
+                "message": (
+                    "Airflow email recipients were preserved for Job failure notifications, but SLA email and "
+                    "callback delivery are not attached to a Databricks duration health rule."
+                ),
+                "rationale": "failure_email_preserved_but_sla_notification_requires_explicit_mapping",
+            }
+        return {
+            "status": "mapped",
+            "target": "job.email_notifications.on_failure",
+            "rationale": "preserved_as_databricks_job_failure_notification",
+        }
+    return None
+
+
 def _semantic_finding(
     source_file: str,
     node: ast.AST | None,
@@ -3197,13 +3409,19 @@ def source_reference(capture_id: str) -> str:
             )
         )
 
+    setting_dispositions = [
+        (candidate, _dag_setting_disposition(str(candidate.details.get("name")), visitor))
+        for candidate in audit.settings
+    ]
     unsupported_settings = [
-        candidate for candidate in audit.settings if candidate.details.get("name") not in _SUPPORTED_DAG_SETTINGS
+        candidate
+        for candidate, disposition in setting_dispositions
+        if disposition is not None and disposition["status"] == "gap"
     ]
     missing_supported_settings = [
         candidate
         for candidate in audit.settings
-        if candidate.details.get("name") in _SUPPORTED_DAG_SETTINGS
+        if candidate.details.get("name") in _RECOGNIZED_DAG_SETTINGS
         and candidate.details.get("name") not in visitor.captured_dag_settings
     ]
     for candidate in missing_supported_settings:
@@ -3217,15 +3435,39 @@ def source_reference(capture_id: str) -> str:
                 candidate=candidate,
             )
         )
+    for candidate, disposition in setting_dispositions:
+        if disposition is None:
+            continue
+        if disposition["status"] == "gap" and not disposition.get("target"):
+            continue
+        transformations.append(
+            {
+                "code": (
+                    "dag_setting_mapped"
+                    if disposition["status"] == "mapped"
+                    else "dag_setting_partially_mapped"
+                    if disposition["status"] == "gap"
+                    else "dag_setting_ignored"
+                ),
+                "setting": str(candidate.details.get("name")),
+                **({"target": disposition["target"]} if disposition.get("target") else {}),
+                "rationale": disposition["rationale"],
+            }
+        )
+    disposition_by_candidate_id = {
+        id(candidate): disposition for candidate, disposition in setting_dispositions if disposition is not None
+    }
     for candidate in unsupported_settings:
         name = str(candidate.details.get("name"))
+        disposition = disposition_by_candidate_id[id(candidate)]
         findings.append(
             source_audit.finding(
                 source_file=source_file,
                 code="unsupported_dag_setting",
                 severity="gap",
-                message=f"Airflow DAG setting {name!r} has no deterministic Databricks Jobs mapping.",
+                message=disposition["message"],
                 candidate=candidate,
+                details={"name": name, "rationale": disposition["rationale"]},
             )
         )
 
diff --git a/src/flowx/sources/airflow/templating.py b/src/flowx/sources/airflow/templating.py
index a4d99fe..68fb8fa 100644
--- a/src/flowx/sources/airflow/templating.py
+++ b/src/flowx/sources/airflow/templating.py
@@ -383,24 +383,73 @@ def unresolved_jinja_expressions(value: Any) -> set[str]:
 # --------------------------------------------------------------------------------------
 
 
-def _timedelta_seconds(node: ast.expr | None) -> int | None:
-    """Parses a ``timedelta(...)`` AST call into total seconds (keyword args only)."""
+def timedelta_seconds(node: ast.expr | None) -> int | None:
+    """Parses a statically numeric ``timedelta(...)`` call into positive whole seconds."""
     if not isinstance(node, ast.Call):
         return None
     func = node.func
     name = func.attr if isinstance(func, ast.Attribute) else (func.id if isinstance(func, ast.Name) else "")
     if name != "timedelta":
         return None
-    units = {"weeks": 604800, "days": 86400, "hours": 3600, "minutes": 60, "seconds": 1, "milliseconds": 0.001}
+
+    units = {
+        "weeks": 604800,
+        "days": 86400,
+        "hours": 3600,
+        "minutes": 60,
+        "seconds": 1,
+        "milliseconds": 0.001,
+        "microseconds": 0.000001,
+    }
+    positional_names = ("days", "seconds", "microseconds", "milliseconds", "minutes", "hours", "weeks")
+    if len(node.args) > len(positional_names) or any(keyword.arg is None for keyword in node.keywords):
+        return None
+
+    values: dict[str, float] = {}
+    for unit, argument in zip(positional_names, node.args):
+        try:
+            value = ast.literal_eval(argument)
+        except (ValueError, SyntaxError):
+            return None
+        if isinstance(value, bool) or not isinstance(value, (int, float)):
+            return None
+        values[unit] = float(value)
+    for keyword in node.keywords:
+        if keyword.arg not in units or keyword.arg in values:
+            return None
+        try:
+            value = ast.literal_eval(keyword.value)
+        except (ValueError, SyntaxError):
+            return None
+        if isinstance(value, bool) or not isinstance(value, (int, float)):
+            return None
+        values[keyword.arg] = float(value)
+
     total = 0.0
-    for kw in node.keywords:
-        if kw.arg in units and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, (int, float)):
-            total += kw.value.value * units[kw.arg]
+    for unit, value in values.items():
+        total += value * units[unit]
     # Round a sub-second total UP to 1s rather than truncating to 0 -- a sub-second timeout/retry_delay
     # is better preserved as 1s than silently dropped (int(0.5) == 0 would read as "unset").
     return math.ceil(total) if total > 0 else None
 
 
+def literal_email_recipients(node: ast.expr | None) -> list[str] | None:
+    """Returns statically declared email recipients, or ``None`` for a dynamic value."""
+    if node is None:
+        return []
+    try:
+        value = ast.literal_eval(node)
+    except (ValueError, SyntaxError):
+        return None
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value] if value else []
+    if isinstance(value, (list, tuple)) and all(isinstance(recipient, str) for recipient in value):
+        return [recipient for recipient in value if recipient]
+    return None
+
+
 def _literal_int(node: ast.expr | None) -> int | None:
     if isinstance(node, ast.Constant) and isinstance(node.value, bool):
         return None
@@ -425,11 +474,11 @@ def pick(key: str) -> ast.expr | None:
     if retries is not None and retries > 0:
         result["max_retries"] = retries
 
-    timeout = _timedelta_seconds(pick("execution_timeout"))
+    timeout = timedelta_seconds(pick("execution_timeout"))
     if timeout is not None:
         result["timeout_seconds"] = timeout
 
-    retry_delay = _timedelta_seconds(pick("retry_delay"))
+    retry_delay = timedelta_seconds(pick("retry_delay"))
     if retry_delay is not None:
         result["min_retry_interval_millis"] = retry_delay * 1000
 
@@ -459,28 +508,11 @@ def supplied(key: str) -> ast.expr | None:
         value = supplied(name)
         if value is None or (isinstance(value, ast.Constant) and value.value is None):
             continue
-        if _timedelta_seconds(value) is None:
+        if timedelta_seconds(value) is None:
             unresolved.append(name)
     return unresolved
 
 
-def email_on_failure(dag_default_args: dict[str, ast.expr], task_kwargs: dict[str, ast.expr]) -> list[str]:
-    """Returns email recipients when email_on_failure is set (for a job-level notification note).
-
-    TODO: not wired up yet -- the shared IR has no email-notification field, so carrying these through
-    to a job's ``email_notifications`` needs an IR addition (tracked separately).
-    """
-    on_failure = task_kwargs.get("email_on_failure", dag_default_args.get("email_on_failure"))
-    if isinstance(on_failure, ast.Constant) and on_failure.value is False:
-        return []
-    email_node = task_kwargs.get("email", dag_default_args.get("email"))
-    if isinstance(email_node, ast.Constant) and isinstance(email_node.value, str):
-        return [email_node.value]
-    if isinstance(email_node, ast.List):
-        return [e.value for e in email_node.elts if isinstance(e, ast.Constant) and isinstance(e.value, str)]
-    return []
-
-
 # --------------------------------------------------------------------------------------
 # trigger_rule -> dependency outcome
 # --------------------------------------------------------------------------------------
diff --git a/tests/unit/test_airflow_agentic_resolution.py b/tests/unit/test_airflow_agentic_resolution.py
index a0e72ec..b6d4a72 100644
--- a/tests/unit/test_airflow_agentic_resolution.py
+++ b/tests/unit/test_airflow_agentic_resolution.py
@@ -116,7 +116,7 @@ def _candidate(gap: dict, *, source: str = "print('Migrated from Airflow')\n", s
         "request_sha256": gap["request_sha256"],
         "provider": {
             "name": "airflow-to-dabs",
-            "version": "0.2.1",
+            "version": "0.2.2",
             "repository": "https://github.com/park-peter/airflow-to-dabs",
         },
         "model": {"name": "test-model"},
@@ -654,7 +654,7 @@ def test_stage_rejects_unresolved_jinja_provider_drift_and_file_hash_mismatch(tm
     wrong_provider = _candidate(gaps[0])
     wrong_provider["provider"]["version"] = "0.1.0"
     assert _stage(output, wrong_provider) == 1
-    assert "pinned airflow-to-dabs v0.2.1" in capsys.readouterr().err
+    assert "pinned airflow-to-dabs v0.2.2" in capsys.readouterr().err
 
     bad_hash = _candidate(gaps[0])
     bad_hash["generated_files"][0]["sha256"] = "0" * 64
@@ -780,19 +780,19 @@ def test_stage_does_not_misclassify_airflow_input_names_as_dynamic_references(tm
     assert "unresolved Airflow Jinja" in capsys.readouterr().err
 
 
-def test_pinned_v021_provider_fixtures_satisfy_the_flowx_contract() -> None:
+def test_pinned_v022_provider_fixtures_satisfy_the_flowx_contract() -> None:
     root = (
         Path(__file__).parents[2]
         / "skills"
         / "flowx-resolve-airflow-gaps"
         / "references"
-        / "airflow-to-dabs-v0.2.1"
+        / "airflow-to-dabs"
         / "providers"
         / "flowx-gap-resolver"
     )
     provider = json.loads((root / "provider.json").read_text(encoding="utf-8"))
 
-    assert provider["provider"]["version"] == "0.2.1"
+    assert provider["provider"]["version"] == "0.2.2"
     for outcome in ("notebook", "sql", "spark-python", "needs-input", "deferred"):
         gap = json.loads((root / "fixtures" / f"gap-{outcome}.json").read_text(encoding="utf-8"))
         candidate = json.loads((root / "fixtures" / f"resolution-{outcome}.json").read_text(encoding="utf-8"))
@@ -1498,7 +1498,7 @@ def test_reviewed_resolution_evidence_drives_honest_code_attached_coverage(tmp_p
     row = build_coverage_rows(metadata)[0]
 
     assert summary == {
-        "provider_version": "0.2.1",
+        "provider_version": "0.2.2",
         "pipelines": {"agentic": {"resolved": 1, "needs_input": 1, "deferred": 0, "declined": 0, "unreviewed": 0}},
     }
     assert row["coverage_pct"] == 100.0
@@ -1506,7 +1506,7 @@ def test_reviewed_resolution_evidence_drives_honest_code_attached_coverage(tmp_p
     assert row["code_attached_coverage_pct"] == 50.0
     assert row["resolved_agentic_count"] == 1
     assert row["unresolved_agentic_count"] == 1
-    assert row["agentic_provider_version"] == "0.2.1"
+    assert row["agentic_provider_version"] == "0.2.2"
     assert row["reconciliation_status"] == "verified_with_reviewed_resolutions"
 
 
diff --git a/tests/unit/test_airflow_operators.py b/tests/unit/test_airflow_operators.py
index e54c3a7..efd395c 100644
--- a/tests/unit/test_airflow_operators.py
+++ b/tests/unit/test_airflow_operators.py
@@ -1191,6 +1191,21 @@ def test_subsecond_timeout_rounds_up_not_dropped():
     assert _by_key(p)["t"].timeout_seconds == 1
 
 
+def test_timedelta_positional_arguments_are_preserved():
+    p = _load(
+        "from datetime import timedelta\n"
+        "from airflow import DAG\n"
+        "from airflow.operators.python import PythonOperator\n"
+        "def w():\n    pass\n"
+        "with DAG(dag_id='d', dagrun_timeout=timedelta(1, 30), "
+        "default_args={'execution_timeout': timedelta(0, 45)}) as dag:\n"
+        "    t = PythonOperator(task_id='t', python_callable=w)\n"
+    )
+
+    assert p.timeout_seconds == 86430
+    assert _by_key(p)["t"].timeout_seconds == 45
+
+
 def test_per_task_retries_override_default_args():
     p = _load(
         "from airflow import DAG\n"
@@ -1896,6 +1911,150 @@ def test_airflow_non_execution_metadata_does_not_create_runtime_gap():
     assert not any(finding["code"] == "unsupported_dag_setting" for finding in p.not_translatable)
 
 
+def test_airflow_dagrun_timeout_and_failure_email_map_to_job_policy():
+    p = _load(
+        "import datetime\n"
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(\n"
+        "    dag_id='job_policy',\n"
+        "    dagrun_timeout=datetime.timedelta(minutes=45),\n"
+        "    default_args={\n"
+        "        'email': ['alerts@example.com'],\n"
+        "        'email_on_failure': True,\n"
+        "        'email_on_retry': False,\n"
+        "    },\n"
+        ") as dag:\n"
+        "    work = BashOperator(task_id='work', bash_command='echo work')\n"
+    )
+
+    assert p.reconciliation_status == "verified"
+    assert p.timeout_seconds == 2700
+    assert p.email_notifications == {"on_failure": ["alerts@example.com"]}
+    assert {
+        (item["setting"], item["code"])
+        for item in p.audit["transformations"]
+        if item.get("setting") in {"dagrun_timeout", "default_args.email", "default_args.email_on_failure"}
+    } == {
+        ("dagrun_timeout", "dag_setting_mapped"),
+        ("default_args.email", "dag_setting_mapped"),
+        ("default_args.email_on_failure", "dag_setting_mapped"),
+    }
+
+
+def test_airflow_disabled_default_args_are_intentional_noops():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(\n"
+        "    dag_id='disabled_defaults',\n"
+        "    max_consecutive_failed_dag_runs=0,\n"
+        "    sla_miss_callback=None,\n"
+        "    default_args={\n"
+        "        'depends_on_past': False,\n"
+        "        'email': ['unused@example.com'],\n"
+        "        'email_on_failure': False,\n"
+        "        'email_on_retry': False,\n"
+        "        'env': {},\n"
+        "    },\n"
+        ") as dag:\n"
+        "    work = BashOperator(task_id='work', bash_command='echo work')\n"
+    )
+
+    assert p.reconciliation_status == "verified"
+    assert p.email_notifications == {}
+    ignored = {item["setting"] for item in p.audit["transformations"] if item.get("code") == "dag_setting_ignored"}
+    assert {
+        "max_consecutive_failed_dag_runs",
+        "sla_miss_callback",
+        "default_args.depends_on_past",
+        "default_args.email",
+        "default_args.email_on_failure",
+        "default_args.email_on_retry",
+        "default_args.env",
+    } <= ignored
+    assert not any(finding["code"] == "unsupported_dag_setting" for finding in p.not_translatable)
+
+
+def test_airflow_sla_email_target_is_preserved_but_remains_an_explicit_gap():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "def notify(*args):\n"
+        "    return None\n"
+        "with DAG(\n"
+        "    dag_id='sla_email',\n"
+        "    sla_miss_callback=notify,\n"
+        "    default_args={'email': 'alerts@example.com'},\n"
+        ") as dag:\n"
+        "    work = BashOperator(task_id='work', bash_command='echo work')\n"
+    )
+
+    assert p.email_notifications == {"on_failure": ["alerts@example.com"]}
+    assert p.reconciliation_status == "verified_with_gaps"
+    finding = next(
+        item
+        for item in p.not_translatable
+        if item["code"] == "unsupported_dag_setting" and item["details"]["name"] == "default_args.email"
+    )
+    assert "SLA email" in finding["message"]
+    assert any(
+        item["code"] == "dag_setting_partially_mapped" and item["setting"] == "default_args.email"
+        for item in p.audit["transformations"]
+    )
+
+
+def test_airflow_retry_email_accounts_for_task_level_retries():
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "with DAG(\n"
+        "    dag_id='task_retry_email',\n"
+        "    default_args={'email': 'ops@example.com', 'email_on_retry': True},\n"
+        ") as dag:\n"
+        "    work = BashOperator(task_id='work', bash_command='echo work', retries=2)\n"
+    )
+
+    assert p.email_notifications == {"on_failure": ["ops@example.com"]}
+    assert p.reconciliation_status == "verified_with_gaps"
+    findings = {
+        item["details"]["name"]: item for item in p.not_translatable if item["code"] == "unsupported_dag_setting"
+    }
+    assert "retry notification" in findings["default_args.email"]["message"]
+    assert "retry notification" in findings["default_args.email_on_retry"]["message"]
+
+
+@pytest.mark.parametrize(
+    ("dag_argument", "expected_reason"),
+    [
+        ("default_args={'depends_on_past': True}", "prior DAG run"),
+        ("max_consecutive_failed_dag_runs=3", "automatically pause"),
+        ("sla_miss_callback=notify", "SLA callback"),
+        ("default_args={'env': {'TOKEN': '{{ conn.api.password }}'}}", "task environment"),
+        (
+            "default_args={'email': 'ops@example.com', 'email_on_retry': True, 'retries': 1}",
+            "retry notification",
+        ),
+        ("dagrun_timeout=runtime_timeout", "static positive timedelta"),
+    ],
+)
+def test_airflow_unrepresentable_dag_runtime_semantics_remain_blocking_gaps(dag_argument, expected_reason):
+    p = _load(
+        "from airflow import DAG\n"
+        "from airflow.operators.bash import BashOperator\n"
+        "runtime_timeout = object()\n"
+        "def notify(*args):\n"
+        "    return None\n"
+        f"with DAG(dag_id='runtime_semantics', {dag_argument}) as dag:\n"
+        "    work = BashOperator(task_id='work', bash_command='echo work')\n"
+    )
+
+    assert p.reconciliation_status == "verified_with_gaps"
+    assert p.tasks[0].task_key == "__flowx_source_gaps"
+    finding = next(item for item in p.not_translatable if item["code"] == "unsupported_dag_setting")
+    assert expected_reason in finding["message"]
+
+
 def test_positional_dag_id_is_preserved_as_job_identity_metadata():
     p = _load(
         "from airflow import DAG\n"
diff --git a/tests/unit/test_airflow_provider_sync.py b/tests/unit/test_airflow_provider_sync.py
index 72880cf..0b240ec 100644
--- a/tests/unit/test_airflow_provider_sync.py
+++ b/tests/unit/test_airflow_provider_sync.py
@@ -10,7 +10,7 @@
 
 ROOT = Path(__file__).parents[2]
 SCRIPT = ROOT / "scripts" / "sync_airflow_provider.py"
-PROVIDER = ROOT / "skills" / "flowx-resolve-airflow-gaps" / "references" / "airflow-to-dabs-v0.2.1"
+PROVIDER = ROOT / "skills" / "flowx-resolve-airflow-gaps" / "references" / "airflow-to-dabs"
 
 
 def _check(destination: Path) -> subprocess.CompletedProcess[str]:
@@ -28,9 +28,9 @@ def test_committed_airflow_provider_pin_is_valid() -> None:
     assert result.returncode == 0, result.stderr
     pin = json.loads(result.stdout)
     assert pin == {
-        "commit": "75196aef85ebb2736b926f2d4db13ec7d5c2551c",
-        "content_sha256": "e1e7395204b3f2759722b9320cea08c6b58284a48fd8062686b36bf676b657fd",
-        "tag": "v0.2.1",
+        "commit": "6a940cbb11ac2edd8e028853865f386002a003f0",
+        "content_sha256": "46215e950028f31a157f68fd7a9dade78e3cd7486b59217f17d9811e3b73618e",
+        "tag": "v0.2.2",
     }
 
 
diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py
index 43eceff..8499a75 100644
--- a/tests/unit/test_bundler.py
+++ b/tests/unit/test_bundler.py
@@ -378,6 +378,23 @@ def test_schedule_block_emitted(self, tmp_path):
         assert job["schedule"]["timezone_id"] == "Europe/Madrid"
         assert job["schedule"]["pause_status"] == "UNPAUSED"
 
+    def test_airflow_job_policy_is_emitted(self, tmp_path):
+        pipeline = Pipeline(
+            name="airflow_policy",
+            tasks=[WaitActivity(name="Pause", task_key="pause", wait_time_seconds=10)],
+            tags={"source": "airflow"},
+            timeout_seconds=1800,
+            email_notifications={"on_failure": ["alerts@example.com"]},
+        )
+
+        workflow = prepare_workflow(pipeline)
+        write_bundle(workflow, tmp_path)
+        resource_file = next((tmp_path / "resources").glob("*.yml"))
+        job = next(iter(yaml.safe_load(resource_file.read_text())["resources"]["jobs"].values()))
+
+        assert job["timeout_seconds"] == 1800
+        assert job["email_notifications"] == {"on_failure": ["alerts@example.com"]}
+
     def test_periodic_trigger_emitted(self, tmp_path):
         """SCHED3-002: periodic schedule spec renders as trigger.periodic."""
         pipeline = Pipeline(
diff --git a/tests/unit/test_param_dedup.py b/tests/unit/test_param_dedup.py
index 2c5680c..6f289d6 100644
--- a/tests/unit/test_param_dedup.py
+++ b/tests/unit/test_param_dedup.py
@@ -2,7 +2,11 @@
 
 from __future__ import annotations
 
+import pytest
+
 from flowx.bundler.dab_writer import _build_job_resource, _pipeline_dict_to_workflow
+from flowx.ir_serde import pipeline_to_dict
+from flowx.models.ir import Pipeline, WaitActivity
 
 
 def _report(default="us"):
@@ -34,3 +38,35 @@ def test_build_job_resource_dedupes_parameters():
     job = _build_job_resource(wf, "pipeline_simple")["resources"]["jobs"]["pipeline_simple"]
     names = [p["name"] for p in job["parameters"]]
     assert names == ["region"]
+
+
+def test_airflow_job_policy_survives_report_round_trip():
+    pipeline = Pipeline(
+        name="airflow_policy",
+        tasks=[WaitActivity(name="Pause", task_key="pause", wait_time_seconds=1)],
+        tags={"source": "airflow"},
+        timeout_seconds=900,
+        email_notifications={"on_failure": ["alerts@example.com"]},
+    )
+
+    workflow = _pipeline_dict_to_workflow(pipeline_to_dict(pipeline))
+    job = _build_job_resource(workflow, "airflow_policy")["resources"]["jobs"]["airflow_policy"]
+
+    assert job["timeout_seconds"] == 900
+    assert job["email_notifications"] == {"on_failure": ["alerts@example.com"]}
+
+
+@pytest.mark.parametrize(
+    ("field", "value"),
+    [
+        ("timeout_seconds", "900"),
+        ("email_notifications", {"on_failure": "alerts@example.com"}),
+        ("email_notifications", {"task_key": ["alerts@example.com"]}),
+    ],
+)
+def test_airflow_job_policy_report_rejects_malformed_values(field, value):
+    report = _report()
+    report[field] = value
+
+    with pytest.raises(ValueError):
+        _pipeline_dict_to_workflow(report)
diff --git a/tests/unit/test_reporting_results.py b/tests/unit/test_reporting_results.py
index 4c45ade..ece0d2c 100644
--- a/tests/unit/test_reporting_results.py
+++ b/tests/unit/test_reporting_results.py
@@ -64,7 +64,7 @@ def test_insert_sql_stamps_run_metadata_and_escapes():
             "resolved_agentic_count": 0,
             "unresolved_agentic_count": 1,
             "agentic_resolution_outcomes": '{"unreviewed":1}',
-            "agentic_provider_version": "0.2.1",
+            "agentic_provider_version": "0.2.2",
             "unsupported_activities": 0,
             "failed_activities": 0,
             "excluded_activities": 0,

From 5a19f3d05106019c01dc44f1fdecff85b3ac5b5b Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Tue, 11 Aug 2026 15:00:00 -0700
Subject: [PATCH 68/77] refactor(airflow): centralize provider version metadata

---
 README.md                                     |  2 +-
 scripts/sync_airflow_provider.py              |  7 ++-
 .../flowx-convert/sources/airflow-coverage.md |  2 +-
 skills/flowx-resolve-airflow-gaps/SKILL.md    |  2 +-
 .../references/contract-v1.md                 |  4 +-
 src/flowx/agentic.py                          | 58 ++++++++++++-------
 tests/unit/test_airflow_agentic_resolution.py | 16 ++---
 tests/unit/test_airflow_provider_sync.py      | 20 ++++++-
 tests/unit/test_reporting_results.py          |  2 +-
 9 files changed, 69 insertions(+), 44 deletions(-)

diff --git a/README.md b/README.md
index a9e23bd..577e064 100644
--- a/README.md
+++ b/README.md
@@ -188,7 +188,7 @@ execution) and maps ~35 operator/sensor families to the shared IR. Highlights:
   `params={...}` → job parameters, `>>` / `<<` / `set_upstream` / TaskGroup edges.
 
 Operators without a deterministic mapping become a failing placeholder and are recorded in
-`gaps.json` for review. Eligible leaf gaps can use the fingerprint-bound resolver backed by the pinned [`airflow-to-dabs` v0.2.2](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.2) provider profile; flowx retains ownership of parsing, graph identity, policy, IR, and packaging. Full matrix:
+`gaps.json` for review. Eligible leaf gaps can use the fingerprint-bound resolver backed by the pinned [`airflow-to-dabs`](https://github.com/park-peter/airflow-to-dabs/tree/main/providers/flowx-gap-resolver) provider profile; flowx retains ownership of parsing, graph identity, policy, IR, and packaging. Full matrix:
 [`skills/flowx-convert/sources/airflow-coverage.md`](skills/flowx-convert/sources/airflow-coverage.md).
 
 Airflow discovery independently audits DAG declarations, task candidates, dependency declarations,
diff --git a/scripts/sync_airflow_provider.py b/scripts/sync_airflow_provider.py
index 12d5aa3..6dda233 100644
--- a/scripts/sync_airflow_provider.py
+++ b/scripts/sync_airflow_provider.py
@@ -14,7 +14,6 @@
 from typing import Any
 
 REPOSITORY = "https://github.com/park-peter/airflow-to-dabs"
-DEFAULT_TAG = "v0.2.2"
 PROVIDER_PATH = PurePosixPath("providers/flowx-gap-resolver/provider.json")
 PIN_FIELD = "flowx_pin"
 
@@ -178,7 +177,7 @@ def verify_provider(destination: Path) -> dict[str, str]:
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--source", type=Path, help="Exact local airflow-to-dabs checkout used for synchronization.")
-    parser.add_argument("--tag", default=DEFAULT_TAG)
+    parser.add_argument("--tag", help="Exact upstream release tag to vendor.")
     parser.add_argument(
         "--destination",
         type=Path,
@@ -194,13 +193,15 @@ def main() -> int:
     args = parser.parse_args()
     if not args.check and args.source is None:
         parser.error("--source is required unless --check is used")
+    if not args.check and args.tag is None:
+        parser.error("--tag is required unless --check is used")
     try:
         result = (
             verify_provider(args.destination)
             if args.check
             else sync_provider(
                 checkout=args.source.resolve() if args.source else Path(),
-                tag=args.tag,
+                tag=str(args.tag),
                 destination=args.destination,
             )
         )
diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md
index 958877a..afae398 100644
--- a/skills/flowx-convert/sources/airflow-coverage.md
+++ b/skills/flowx-convert/sources/airflow-coverage.md
@@ -47,7 +47,7 @@ safe fallback is a flagged, failing task rather than a silent omission. Callable
 (`**context` / `ti`) or XCom, and runtime-branching decorators, take the same route rather than
 emitting code that fails at runtime.
 
-The resolver consumes the pinned `airflow-to-dabs` v0.2.2 Flowx provider profile. It receives one flowx-produced gap envelope and cannot express graph or task-policy changes. Accepted `resolved` candidates contribute to mechanically validated code-attached coverage, but remain agentic and do not increase deterministic coverage. `needs_input`, `deferred`, and unreviewed candidates remain linked failing placeholders.
+The resolver consumes the pinned `airflow-to-dabs` Flowx provider profile. It receives one flowx-produced gap envelope and cannot express graph or task-policy changes. Accepted `resolved` candidates contribute to mechanically validated code-attached coverage, but remain agentic and do not increase deterministic coverage. `needs_input`, `deferred`, and unreviewed candidates remain linked failing placeholders.
 
 ## Not yet supported
 
diff --git a/skills/flowx-resolve-airflow-gaps/SKILL.md b/skills/flowx-resolve-airflow-gaps/SKILL.md
index 60bb28d..64dbb6b 100644
--- a/skills/flowx-resolve-airflow-gaps/SKILL.md
+++ b/skills/flowx-resolve-airflow-gaps/SKILL.md
@@ -5,7 +5,7 @@ description: Resolve source-reconciled Airflow leaf gaps through the fingerprint
 
 # Resolve Airflow Leaf Gaps
 
-Use this workflow only for Airflow reports whose deterministic conversion succeeded with gaps. Flowx owns source parsing, task identity, dependencies, task policy, IR, and packaging. This skill reasons about one prepared gap at a time using the migration knowledge from [`park-peter/airflow-to-dabs` v0.2.2](https://github.com/park-peter/airflow-to-dabs/releases/tag/v0.2.2). It must not parse the DAG independently or generate a second bundle.
+Use this workflow only for Airflow reports whose deterministic conversion succeeded with gaps. Flowx owns source parsing, task identity, dependencies, task policy, IR, and packaging. This skill reasons about one prepared gap at a time using the pinned migration knowledge from [`park-peter/airflow-to-dabs`](https://github.com/park-peter/airflow-to-dabs/tree/main/providers/flowx-gap-resolver). It must not parse the DAG independently or generate a second bundle.
 
 Read [`references/contract-v1.md`](references/contract-v1.md) and the pinned [`airflow-to-dabs/providers/flowx-gap-resolver/PROFILE.md`](references/airflow-to-dabs/providers/flowx-gap-resolver/PROFILE.md) before authoring a resolution. The profile and every referenced knowledge file are vendored from the exact upstream tag and commit under `references/airflow-to-dabs/`. If required knowledge is unavailable, return `needs_input` or `deferred`; never infer missing operator semantics.
 
diff --git a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
index 1591c76..4855d2b 100644
--- a/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
+++ b/skills/flowx-resolve-airflow-gaps/references/contract-v1.md
@@ -19,7 +19,7 @@ The provider receives a `GapEnvelope` produced by flowx. It does not receive aut
   "request_sha256": "copied from the envelope",
   "provider": {
     "name": "airflow-to-dabs",
-    "version": "0.2.2",
+    "version": "copied from GapEnvelope.provider.version",
     "repository": "https://github.com/park-peter/airflow-to-dabs"
   },
   "model": {"name": "model identifier"},
@@ -61,4 +61,4 @@ Spark Python uses `{"kind": "spark_python", "file": "task.py", "parameters": ["-
 - Notebook `base_parameters` keys must use letters, digits, underscores, dots, and hyphens, starting with a letter or underscore. Flowx-owned names beginning with `__flowx` and Databricks task identity, graph, policy, and task-type field names are reserved case-insensitively.
 - Airflow Jinja is rejected. Databricks dynamic references such as `{{job.parameters.x}}`, `{{tasks.upstream.values.x}}`, and `{{input}}` are valid in replacement parameter values, not in uploaded notebook or SQL source files; source files must read widgets or SQL named parameters.
 - Every source argument in the envelope has exactly one disposition and a non-empty rationale.
-- The provider identity must match the pinned `airflow-to-dabs` v0.2.2 knowledge release.
+- The provider identity must match `GapEnvelope.provider` and the prepared workspace's pinned knowledge release.
diff --git a/src/flowx/agentic.py b/src/flowx/agentic.py
index ff60317..663384e 100644
--- a/src/flowx/agentic.py
+++ b/src/flowx/agentic.py
@@ -24,7 +24,6 @@
 
 CONTRACT_VERSION = "1"
 PROVIDER_NAME = "airflow-to-dabs"
-PROVIDER_VERSION = "0.2.2"
 PROVIDER_REPOSITORY = "https://github.com/park-peter/airflow-to-dabs"
 
 _ALLOWED_REPLACEMENT_KINDS = ("notebook", "sql", "spark_python")
@@ -113,11 +112,7 @@ class GapEnvelope:
 
     def as_dict(self) -> dict[str, Any]:
         """Returns the public GapEnvelope v1 representation."""
-        provider = {
-            "name": PROVIDER_NAME,
-            "version": PROVIDER_VERSION,
-            "repository": PROVIDER_REPOSITORY,
-        }
+        provider = _provider_identity()
         payload = {
             "contract_version": CONTRACT_VERSION,
             "gap_id": self.gap_id,
@@ -263,7 +258,7 @@ def prepare_airflow_resolutions(
     return {
         "status": "prepared",
         "contract_version": CONTRACT_VERSION,
-        "provider_version": PROVIDER_VERSION,
+        "provider_version": _provider_identity()["version"],
         "gap_count": len(gaps),
         "requested_gap_id": gap_id,
         "workspace": str(target),
@@ -746,7 +741,7 @@ def _load_persisted_agentic_evidence(evidence_dir: Path) -> PersistedResolutionE
     except AgenticContractError as error:
         raise AgenticContractError(f"agentic resolution evidence failed validation: {error}") from error
     return PersistedResolutionEvidence(
-        provider_version=PROVIDER_VERSION,
+        provider_version=_provider_identity()["version"],
         gaps=gaps,
         resolutions=resolutions,
         reviewed_resolutions=reviewed_resolutions,
@@ -1001,13 +996,11 @@ def _validate_candidate(
         if candidate.get(field) != gap.get(field):
             raise AgenticContractError(f"Candidate {field} does not match its GapEnvelope")
     provider = candidate.get("provider")
-    expected_provider = {
-        "name": PROVIDER_NAME,
-        "version": PROVIDER_VERSION,
-        "repository": PROVIDER_REPOSITORY,
-    }
+    expected_provider = _provider_identity()
     if provider != expected_provider:
-        raise AgenticContractError(f"Candidate provider must match pinned {PROVIDER_NAME} v{PROVIDER_VERSION}")
+        raise AgenticContractError(
+            f"Candidate provider must match pinned {PROVIDER_NAME} v{expected_provider['version']}"
+        )
     model = candidate.get("model")
     if not isinstance(model, dict) or not isinstance(model.get("name"), str) or not model["name"].strip():
         raise AgenticContractError("Candidate model provenance requires a non-empty model.name")
@@ -1263,7 +1256,7 @@ def _apply_to_baseline(baseline: dict[str, Any], resolutions: list[StagedResolut
             )
         pipeline.setdefault("audit", {})["agentic_resolution"] = {
             "contract_version": CONTRACT_VERSION,
-            "provider_version": PROVIDER_VERSION,
+            "provider_version": _provider_identity()["version"],
             "validation_status": "verified",
             "baseline_graph_sha256": baseline_graph,
             "merged_graph_sha256": merged_graph,
@@ -1529,22 +1522,43 @@ def _provider_context_path() -> Path:
         Path(__file__).resolve().parents[2] / "skills" / "flowx-resolve-airflow-gaps" / "references" / "airflow-to-dabs"
     )
     if not source.is_dir():
-        raise AgenticContractError(
-            f"provider_unavailable: pinned {PROVIDER_NAME} v{PROVIDER_VERSION} context is missing"
-        )
+        raise AgenticContractError(f"provider_unavailable: pinned {PROVIDER_NAME} context is missing")
     return source
 
 
 def _provider_identity() -> dict[str, str]:
-    return {"name": PROVIDER_NAME, "version": PROVIDER_VERSION, "repository": PROVIDER_REPOSITORY}
+    manifest_path = _provider_context_path() / "providers" / "flowx-gap-resolver" / "provider.json"
+    try:
+        manifest = _read_json_object(manifest_path)
+    except (OSError, json.JSONDecodeError, AgenticContractError) as error:
+        raise AgenticContractError(f"provider_unavailable: pinned {PROVIDER_NAME} manifest is invalid") from error
+    provider = manifest.get("provider")
+    pin = manifest.get("flowx_pin")
+    if not isinstance(provider, dict) or not isinstance(pin, dict):
+        raise AgenticContractError(f"provider_unavailable: pinned {PROVIDER_NAME} identity is invalid")
+    version = provider.get("version")
+    if (
+        not isinstance(version, str)
+        or not version
+        or provider.get("name") != PROVIDER_NAME
+        or provider.get("repository") != PROVIDER_REPOSITORY
+        or pin.get("repository") != PROVIDER_REPOSITORY
+        or pin.get("tag") != f"v{version}"
+        or pin.get("contract_version") != CONTRACT_VERSION
+    ):
+        raise AgenticContractError(f"provider_unavailable: pinned {PROVIDER_NAME} identity is invalid")
+    return {"name": PROVIDER_NAME, "version": version, "repository": PROVIDER_REPOSITORY}
 
 
 def _validate_manifest_provider(value: Any) -> str:
-    if not isinstance(value, dict) or {key: value.get(key) for key in _provider_identity()} != _provider_identity():
-        raise AgenticContractError(f"Agentic workspace provider must be pinned to {PROVIDER_NAME} v{PROVIDER_VERSION}")
+    expected_provider = _provider_identity()
+    if not isinstance(value, dict) or {key: value.get(key) for key in expected_provider} != expected_provider:
+        raise AgenticContractError(
+            f"Agentic workspace provider must be pinned to {PROVIDER_NAME} v{expected_provider['version']}"
+        )
     sha256 = value.get("sha256")
     if (
-        set(value) != {*_provider_identity(), "sha256"}
+        set(value) != {*expected_provider, "sha256"}
         or not isinstance(sha256, str)
         or not re.fullmatch(r"[0-9a-f]{64}", sha256)
     ):
diff --git a/tests/unit/test_airflow_agentic_resolution.py b/tests/unit/test_airflow_agentic_resolution.py
index b6d4a72..c6d3f52 100644
--- a/tests/unit/test_airflow_agentic_resolution.py
+++ b/tests/unit/test_airflow_agentic_resolution.py
@@ -114,11 +114,7 @@ def _candidate(gap: dict, *, source: str = "print('Migrated from Airflow')\n", s
         "graph_sha256": gap["graph_sha256"],
         "provider_sha256": gap["provider_sha256"],
         "request_sha256": gap["request_sha256"],
-        "provider": {
-            "name": "airflow-to-dabs",
-            "version": "0.2.2",
-            "repository": "https://github.com/park-peter/airflow-to-dabs",
-        },
+        "provider": agentic_contract._provider_identity(),
         "model": {"name": "test-model"},
         "argument_disposition": dispositions,
         "prerequisites": [],
@@ -654,7 +650,7 @@ def test_stage_rejects_unresolved_jinja_provider_drift_and_file_hash_mismatch(tm
     wrong_provider = _candidate(gaps[0])
     wrong_provider["provider"]["version"] = "0.1.0"
     assert _stage(output, wrong_provider) == 1
-    assert "pinned airflow-to-dabs v0.2.2" in capsys.readouterr().err
+    assert "Candidate provider must match pinned airflow-to-dabs" in capsys.readouterr().err
 
     bad_hash = _candidate(gaps[0])
     bad_hash["generated_files"][0]["sha256"] = "0" * 64
@@ -780,7 +776,7 @@ def test_stage_does_not_misclassify_airflow_input_names_as_dynamic_references(tm
     assert "unresolved Airflow Jinja" in capsys.readouterr().err
 
 
-def test_pinned_v022_provider_fixtures_satisfy_the_flowx_contract() -> None:
+def test_pinned_provider_fixtures_satisfy_the_flowx_contract() -> None:
     root = (
         Path(__file__).parents[2]
         / "skills"
@@ -792,7 +788,7 @@ def test_pinned_v022_provider_fixtures_satisfy_the_flowx_contract() -> None:
     )
     provider = json.loads((root / "provider.json").read_text(encoding="utf-8"))
 
-    assert provider["provider"]["version"] == "0.2.2"
+    assert provider["provider"] == agentic_contract._provider_identity()
     for outcome in ("notebook", "sql", "spark-python", "needs-input", "deferred"):
         gap = json.loads((root / "fixtures" / f"gap-{outcome}.json").read_text(encoding="utf-8"))
         candidate = json.loads((root / "fixtures" / f"resolution-{outcome}.json").read_text(encoding="utf-8"))
@@ -1498,7 +1494,7 @@ def test_reviewed_resolution_evidence_drives_honest_code_attached_coverage(tmp_p
     row = build_coverage_rows(metadata)[0]
 
     assert summary == {
-        "provider_version": "0.2.2",
+        "provider_version": agentic_contract._provider_identity()["version"],
         "pipelines": {"agentic": {"resolved": 1, "needs_input": 1, "deferred": 0, "declined": 0, "unreviewed": 0}},
     }
     assert row["coverage_pct"] == 100.0
@@ -1506,7 +1502,7 @@ def test_reviewed_resolution_evidence_drives_honest_code_attached_coverage(tmp_p
     assert row["code_attached_coverage_pct"] == 50.0
     assert row["resolved_agentic_count"] == 1
     assert row["unresolved_agentic_count"] == 1
-    assert row["agentic_provider_version"] == "0.2.2"
+    assert row["agentic_provider_version"] == agentic_contract._provider_identity()["version"]
     assert row["reconciliation_status"] == "verified_with_reviewed_resolutions"
 
 
diff --git a/tests/unit/test_airflow_provider_sync.py b/tests/unit/test_airflow_provider_sync.py
index 0b240ec..45cef3e 100644
--- a/tests/unit/test_airflow_provider_sync.py
+++ b/tests/unit/test_airflow_provider_sync.py
@@ -27,11 +27,25 @@ def test_committed_airflow_provider_pin_is_valid() -> None:
 
     assert result.returncode == 0, result.stderr
     pin = json.loads(result.stdout)
+    manifest = json.loads((PROVIDER / "providers" / "flowx-gap-resolver" / "provider.json").read_text())
     assert pin == {
-        "commit": "6a940cbb11ac2edd8e028853865f386002a003f0",
-        "content_sha256": "46215e950028f31a157f68fd7a9dade78e3cd7486b59217f17d9811e3b73618e",
-        "tag": "v0.2.2",
+        "commit": manifest["flowx_pin"]["commit"],
+        "content_sha256": manifest["flowx_pin"]["content_sha256"],
+        "tag": manifest["flowx_pin"]["tag"],
     }
+    assert pin["tag"] == f"v{manifest['provider']['version']}"
+
+
+def test_airflow_provider_sync_requires_an_explicit_tag() -> None:
+    result = subprocess.run(
+        [sys.executable, str(SCRIPT), "--source", str(ROOT)],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 2
+    assert "--tag is required unless --check is used" in result.stderr
 
 
 def test_airflow_provider_pin_rejects_modified_content(tmp_path: Path) -> None:
diff --git a/tests/unit/test_reporting_results.py b/tests/unit/test_reporting_results.py
index ece0d2c..f7232cc 100644
--- a/tests/unit/test_reporting_results.py
+++ b/tests/unit/test_reporting_results.py
@@ -64,7 +64,7 @@ def test_insert_sql_stamps_run_metadata_and_escapes():
             "resolved_agentic_count": 0,
             "unresolved_agentic_count": 1,
             "agentic_resolution_outcomes": '{"unreviewed":1}',
-            "agentic_provider_version": "0.2.2",
+            "agentic_provider_version": "test-provider-version",
             "unsupported_activities": 0,
             "failed_activities": 0,
             "excluded_activities": 0,

From c7c11ddf472be3568419a04f290fee39b5cd8fdd Mon Sep 17 00:00:00 2001
From: peter-park_data 
Date: Wed, 12 Aug 2026 10:51:04 -0700
Subject: [PATCH 69/77] refactor(airflow): pin provider by tag and digest

---
 scripts/sync_airflow_provider.py              |  37 ++++-
 .../providers/flowx-gap-resolver/PROFILE.md   |   5 +-
 .../fixtures/gap-deferred.json                |   2 +-
 .../fixtures/gap-needs-input.json             |   2 +-
 .../fixtures/gap-notebook.json                |   2 +-
 .../fixtures/gap-spark-python.json            |   2 +-
 .../flowx-gap-resolver/fixtures/gap-sql.json  |   2 +-
 .../fixtures/resolution-deferred.json         |   2 +-
 .../fixtures/resolution-needs-input.json      |   2 +-
 .../fixtures/resolution-notebook.json         |   2 +-
 .../fixtures/resolution-spark-python.json     |   2 +-
 .../fixtures/resolution-sql.json              |   2 +-
 .../flowx-gap-resolver/provider.json          |   9 +-
 .../references/airflow3-migration.md          | 102 +++---------
 .../references/dab-schema-reference.md        |   8 +-
 .../references/hadoop-migration-guide.md      |   2 +-
 .../references/operator-mapping.md            | 152 +++++++++++++++++-
 .../references/schedule-trigger-mapping.md    |  24 ++-
 src/flowx/agentic.py                          | 144 ++++++++++++++---
 tests/unit/test_airflow_agentic_resolution.py |   8 +-
 tests/unit/test_airflow_provider_sync.py      | 105 +++++++++++-
 21 files changed, 474 insertions(+), 142 deletions(-)

diff --git a/scripts/sync_airflow_provider.py b/scripts/sync_airflow_provider.py
index 6dda233..e08f849 100644
--- a/scripts/sync_airflow_provider.py
+++ b/scripts/sync_airflow_provider.py
@@ -7,6 +7,7 @@
 import hashlib
 import json
 import posixpath
+import re
 import shutil
 import subprocess
 import tempfile
@@ -22,6 +23,25 @@ class ProviderSyncError(ValueError):
     """Raised when provider source or vendored content violates the pin contract."""
 
 
+def _release_version(tag: Any) -> str:
+    if not isinstance(tag, str) or re.fullmatch(r"v[0-9A-Za-z][0-9A-Za-z.+-]*", tag) is None:
+        raise ProviderSyncError(f"Provider release tag is invalid: {tag!r}")
+    return tag[1:]
+
+
+def _validate_provider_identity(provider: dict[str, Any], *, tag: str) -> None:
+    identity = provider.get("provider")
+    if (
+        not isinstance(identity, dict)
+        or identity.get("name") != "airflow-to-dabs"
+        or identity.get("repository") != REPOSITORY
+    ):
+        raise ProviderSyncError("Provider manifest has an unsupported identity")
+    version = identity.get("version")
+    if version is not None and version != _release_version(tag):
+        raise ProviderSyncError(f"Provider version {version!r} does not match tag {tag!r}")
+
+
 def _json_object(data: bytes, *, label: str) -> dict[str, Any]:
     try:
         value = json.loads(data)
@@ -92,12 +112,13 @@ def _git_output(checkout: Path, *args: str) -> bytes:
 
 
 def sync_provider(*, checkout: Path, tag: str, destination: Path) -> dict[str, str]:
+    _release_version(tag)
     commit = _git_output(checkout, "rev-parse", f"{tag}^{{commit}}").decode().strip()
+    if re.fullmatch(r"[0-9a-f]{40}", commit) is None:
+        raise ProviderSyncError(f"Provider tag {tag!r} did not resolve to a commit")
     provider_data = _git_output(checkout, "show", f"{commit}:{PROVIDER_PATH.as_posix()}")
     provider = _json_object(provider_data, label=PROVIDER_PATH.as_posix())
-    version = str((provider.get("provider") or {}).get("version", ""))
-    if version != tag.removeprefix("v"):
-        raise ProviderSyncError(f"Provider version {version!r} does not match tag {tag!r}")
+    _validate_provider_identity(provider, tag=tag)
 
     source_files: dict[PurePosixPath, bytes] = {}
     for path in _allowlisted_paths(provider):
@@ -137,9 +158,9 @@ def verify_provider(destination: Path) -> dict[str, str]:
         raise ProviderSyncError("Vendored provider.json is missing flowx_pin metadata")
     if pin.get("repository") != REPOSITORY or pin.get("contract_version") != "1":
         raise ProviderSyncError("Vendored provider pin has an unsupported repository or contract")
-    version = str((provider.get("provider") or {}).get("version", ""))
-    if pin.get("tag") != f"v{version}":
-        raise ProviderSyncError("Vendored provider version does not match its pinned tag")
+    tag = pin.get("tag")
+    _release_version(tag)
+    _validate_provider_identity(provider, tag=tag)
 
     allowlisted_paths = set(_allowlisted_paths(provider))
     actual_paths: set[PurePosixPath] = set()
@@ -169,9 +190,9 @@ def verify_provider(destination: Path) -> dict[str, str]:
     if pin.get("content_sha256") != content_digest:
         raise ProviderSyncError("Vendored provider content digest does not match flowx_pin metadata")
     commit = pin.get("commit")
-    if not isinstance(commit, str) or len(commit) != 40:
+    if not isinstance(commit, str) or re.fullmatch(r"[0-9a-f]{40}", commit) is None:
         raise ProviderSyncError("Vendored provider pin has an invalid commit")
-    return {"tag": str(pin["tag"]), "commit": commit, "content_sha256": content_digest}
+    return {"tag": tag, "commit": commit, "content_sha256": content_digest}
 
 
 def main() -> int:
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/PROFILE.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/PROFILE.md
index 7c9560c..c33b15d 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/PROFILE.md
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/PROFILE.md
@@ -4,16 +4,17 @@ Resolve exactly one source-reconciled Airflow leaf gap supplied by flowx. flowx
 capture identity, task keys, dependencies, task policy, control flow, IR, and bundle packaging. Do
 not reopen or parse the original DAG, construct another task graph, or generate a bundle.
 
-This profile implements flowx Airflow agentic gap contract `1` with the pinned provider identity:
+This profile implements flowx Airflow agentic gap contract `1`. The provider identity is:
 
 ```json
 {
   "name": "airflow-to-dabs",
-  "version": "0.2.2",
   "repository": "https://github.com/park-peter/airflow-to-dabs"
 }
 ```
 
+The consumer pins a release by tag, commit, and content digest. Report that pinned version in the `provider` block of every resolution.
+
 ## Inputs
 
 Accept one `GapEnvelope` JSON object. Use only the captured source, arguments, surrounding task-key
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-deferred.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-deferred.json
index f823482..796d037 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-deferred.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-deferred.json
@@ -40,7 +40,7 @@
   "knowledge_provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.2"
+    "version": ""
   },
   "operator": "BranchPythonOperator",
   "operator_fqn": "airflow.operators.python.BranchPythonOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-needs-input.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-needs-input.json
index 46a892f..2ca15ea 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-needs-input.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-needs-input.json
@@ -49,7 +49,7 @@
   "knowledge_provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.2"
+    "version": ""
   },
   "operator": "KubernetesPodOperator",
   "operator_fqn": "airflow.providers.cncf.kubernetes.operators.pod.KubernetesPodOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-notebook.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-notebook.json
index d812c4e..3298a9e 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-notebook.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-notebook.json
@@ -47,7 +47,7 @@
   "knowledge_provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.2"
+    "version": ""
   },
   "operator": "SimpleHttpOperator",
   "operator_fqn": "airflow.providers.http.operators.http.SimpleHttpOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-spark-python.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-spark-python.json
index d9154bf..5a919da 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-spark-python.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-spark-python.json
@@ -41,7 +41,7 @@
   "knowledge_provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.2"
+    "version": ""
   },
   "operator": "CustomPythonOperator",
   "operator_fqn": "company.airflow.operators.CustomPythonOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-sql.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-sql.json
index 887498f..ee38675 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-sql.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/gap-sql.json
@@ -44,7 +44,7 @@
   "knowledge_provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.2"
+    "version": ""
   },
   "operator": "SQLExecuteQueryOperator",
   "operator_fqn": "airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-deferred.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-deferred.json
index ab85c15..d35a39f 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-deferred.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-deferred.json
@@ -27,7 +27,7 @@
   "provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.2"
+    "version": ""
   },
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
   "reason": "A faithful branch migration requires condition tasks and downstream dependency rewrites, which are outside the leaf-only provider contract.",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json
index fc95779..838285e 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-needs-input.json
@@ -37,7 +37,7 @@
   "provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.2"
+    "version": ""
   },
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
   "reason": "Provide the container source or packaged application, required Python/system dependencies, registry access requirements, and the Databricks secret or Unity Catalog mappings for orders_secret.",
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-notebook.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-notebook.json
index 520810d..21055c2 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-notebook.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-notebook.json
@@ -47,7 +47,7 @@
   "provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.2"
+    "version": ""
   },
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
   "replacement": {
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json
index 96d532a..5a90abc 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-spark-python.json
@@ -31,7 +31,7 @@
   "provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.2"
+    "version": ""
   },
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
   "replacement": {
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-sql.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-sql.json
index bac1dcd..d2f7c35 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-sql.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/fixtures/resolution-sql.json
@@ -42,7 +42,7 @@
   "provider": {
     "name": "airflow-to-dabs",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.2"
+    "version": ""
   },
   "provider_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
   "replacement": {
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/provider.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/provider.json
index 8881651..ef4a96e 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/provider.json
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/provider.json
@@ -12,11 +12,11 @@
     "fixtures/resolution-deferred.json"
   ],
   "flowx_pin": {
-    "commit": "6a940cbb11ac2edd8e028853865f386002a003f0",
-    "content_sha256": "46215e950028f31a157f68fd7a9dade78e3cd7486b59217f17d9811e3b73618e",
+    "commit": "867abee838a0734ff7f7878b04402ee08ff7f3d3",
+    "content_sha256": "3cc9cd3c8fa9253f7485f73216d31b0847f6fbeab4df15042bc7ab2ef6f4751e",
     "contract_version": "1",
     "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "tag": "v0.2.2"
+    "tag": "v0.2.3"
   },
   "interface": {
     "contract_versions": [
@@ -65,7 +65,6 @@
   "profile_schema_version": "1",
   "provider": {
     "name": "airflow-to-dabs",
-    "repository": "https://github.com/park-peter/airflow-to-dabs",
-    "version": "0.2.2"
+    "repository": "https://github.com/park-peter/airflow-to-dabs"
   }
 }
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/airflow3-migration.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/airflow3-migration.md
index 4b4d771..55818ae 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/airflow3-migration.md
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/airflow3-migration.md
@@ -1,19 +1,11 @@
 # Airflow 3 Recognition and Migration Guide
 
-Reference for converting DAGs authored against **Apache Airflow 3.x**. Airflow 3 keeps the same
-operator/sensor *semantics* as Airflow 2 — the DABs mappings in `references/operator-mapping.md`
-are unchanged — but the **import paths and scheduling APIs moved**. The risk in a naïve conversion
-is not a wrong mapping; it is a DAG whose tasks are **silently missed** because the parser only
-recognized Airflow 2 import paths. Recognize the Airflow 3 authoring surface, map the clean
-equivalents, and flag the rest.
+Reference for converting DAGs authored against **Apache Airflow 3.x**. Airflow 3 keeps the same operator/sensor *semantics* as Airflow 2 — the DABs mappings in `references/operator-mapping.md` are unchanged — but the **import paths and scheduling APIs moved**. The risk in a naïve conversion is not a wrong mapping; it is a DAG whose tasks are **silently missed** because the parser only recognized Airflow 2 import paths. Recognize the Airflow 3 authoring surface, map the clean equivalents, and flag the rest.
 
 This skill's approach for Airflow 3 is **recognize → safe-map → flag**:
-- **Recognize** the `airflow.sdk` and `apache-airflow-providers-standard` import paths so no task
-  is dropped.
-- **Safe-map** the constructs with clean Lakeflow equivalents (operators via the existing tiers;
-  `Asset`-based scheduling per the resolution rule).
-- **Flag** constructs with no clean equivalent (`@asset` pipelines, `AssetWatcher`, asset aliases,
-  DAG versioning, deadline alerts) in `MIGRATION_NOTES.md` — do not invent a mapping.
+- **Recognize** the `airflow.sdk` and `apache-airflow-providers-standard` import paths so no task is dropped.
+- **Safe-map** the constructs with clean Lakeflow equivalents (operators via the existing tiers; `Asset`-based scheduling per the resolution rule).
+- **Flag** constructs with no clean equivalent (`@asset` pipelines, `AssetWatcher`, asset aliases, DAG versioning, deadline alerts) in `MIGRATION_NOTES.md` — do not invent a mapping.
 
 ---
 
@@ -24,12 +16,9 @@ Any of these signals Airflow 3 authoring; parse accordingly:
 - Imports from `airflow.sdk` (e.g. `from airflow.sdk import dag, task, task_group, Asset`).
 - Imports from `airflow.providers.standard.*` for common operators/sensors.
 - `Asset(...)` (the Airflow 3 name for `Dataset`) in `schedule=`.
-- `schedule=` used with a **list** of assets, a **boolean** asset expression (`|`, `&`), or an
-  `AssetOrTimeSchedule`.
+- `schedule=` used with a **list** of assets, a **boolean** asset expression (`|`, `&`), or an `AssetOrTimeSchedule`.
 
-`schedule_interval=` is **removed** in Airflow 3 (use `schedule=`), and `SubDagOperator` is
-**removed** (see below). `@dag` / `@task` / `@task_group` behave the same as in Airflow 2 once
-their import path is recognized.
+`schedule_interval=` is **removed** in Airflow 3 (use `schedule=`), and `SubDagOperator` is **removed** (see below). `@dag` / `@task` / `@task_group` behave the same as in Airflow 2 once their import path is recognized.
 
 ---
 
@@ -37,53 +26,32 @@ their import path is recognized.
 
 Reading the DAG's schedule/backfill intent depends on these Airflow 3 defaults and behaviors:
 
-- **`schedule` defaults to `None`** — a DAG with no `schedule=` runs on manual trigger only. Emit no
-  DABs `schedule`/`trigger` for it (manual/`run_job_task`-driven).
-- **`catchup` defaults to `False`** — an unset `catchup` means the DAG does **not** backfill missed
-  intervals. Only treat backfill as intended when `catchup=True` is explicit; note the backfill
-  expectation (and that DABs jobs have no catchup) in `MIGRATION_NOTES.md`.
-- **A raw-cron `schedule` uses `CronTriggerTimetable`** — the run's `logical_date` is the fire time
-  (run-after), not the start of a data interval. When a cron/timetable DAG is date-sensitive (its
-  tasks read `logical_date`/`{{ ds }}` to pick the processing window), confirm the intended window and
-  record it before mapping `{{ ds }}` → `{{job.parameters.run_date}}`; flag any timetable that can't be
-  mapped deterministically.
+- **`schedule` defaults to `None`** — a DAG with no `schedule=` runs on manual trigger only. Emit no DABs `schedule`/`trigger` for it (manual/`run_job_task`-driven).
+- **`catchup` defaults to `False`** — an unset `catchup` means the DAG does **not** backfill missed intervals. Only treat backfill as intended when `catchup=True` is explicit; note the backfill expectation (and that DABs jobs have no catchup) in `MIGRATION_NOTES.md`.
+- **A raw-cron `schedule` uses `CronTriggerTimetable`** — the run's `logical_date` is the fire time (run-after), not the start of a data interval. When a cron/timetable DAG is date-sensitive (its tasks read `logical_date`/`{{ ds }}` to pick the processing window), confirm the intended window and record it before mapping `{{ ds }}` → `{{job.parameters.run_date}}`; flag any timetable that can't be mapped deterministically.
 
 ---
 
 ## Airflow 3 execution-model additions: native async and resumable
 
-Two execution-model constructs are new in Airflow 3 and affect what you parse. Neither has a DABs
-"mode" switch; migrate the underlying operation. (**Deferrable operators are NOT Airflow-3-specific** —
-they date from Airflow 2.2 — so their migration rule lives with the operator mappings in
-`references/operator-mapping.md`, not here.)
+Two execution-model constructs are new in Airflow 3 and affect what you parse. Neither has a DABs "mode" switch; migrate the underlying operation. (**Deferrable operators are NOT Airflow-3-specific** — they date from Airflow 2.2 — so their migration rule lives with the operator mappings in `references/operator-mapping.md`, not here.)
 
 ### Native async TaskFlow (`@task` on `async def`) — Airflow 3.2.0
 
-Airflow **3.2.0** added native async TaskFlow tasks: `@task` decorating an `async def`, using `await`,
-`asyncio.gather`, and async hooks (`HttpAsyncHook`, `SFTPHookAsync`). This is **distinct from
-deferrable** — async tasks do many concurrent I/O ops within **one** worker slot on a shared event
-loop; deferrable frees the slot during a wait. Migration:
+Airflow **3.2.0** added native async TaskFlow tasks: `@task` decorating an `async def`, using `await`, `asyncio.gather`, and async hooks (`HttpAsyncHook`, `SFTPHookAsync`). This is **distinct from deferrable** — async tasks do many concurrent I/O ops within **one** worker slot on a shared event loop; deferrable frees the slot during a wait. Migration:
 
 - Map to a `notebook_task` / wheel task; keep the concurrent I/O **inside one task** by default.
-- The coroutine is **not runnable as-is** — rewrite Airflow async hooks and Connections to native async
-  clients (e.g. `aiohttp`, `asyncssh`) with auth from `dbutils.secrets`; the notebook drives the event
-  loop itself.
-- Optionally split independent `asyncio.gather()` items into a `for_each_task` — flag the changed retry
-  and UI granularity. There is no DABs "async" setting.
+- The coroutine is **not runnable as-is** — rewrite Airflow async hooks and Connections to native async clients (e.g. `aiohttp`, `asyncssh`) with auth from `dbutils.secrets`; the notebook drives the event loop itself.
+- Optionally split independent `asyncio.gather()` items into a `for_each_task` — flag the changed retry and UI granularity. There is no DABs "async" setting.
 
 Reference: https://airflow.apache.org/docs/task-sdk/stable/deferred-vs-async-operators.html
 
 ### Resumable external jobs (`ResumableJobMixin`) — Airflow 3.3.0
 
-Airflow **3.3.0** added `ResumableJobMixin`: an operator persists the external job id before polling and,
-on retry, **reattaches** to the running external job instead of resubmitting (implementers provide
-`submit_job`, `get_job_status`, `is_job_active`, `is_job_succeeded`, `poll_until_complete`,
-`get_job_result`). Migration:
+Airflow **3.3.0** added `ResumableJobMixin`: an operator persists the external job id before polling and, on retry, **reattaches** to the running external job instead of resubmitting (implementers provide `submit_job`, `get_job_status`, `is_job_active`, `is_job_succeeded`, `poll_until_complete`, `get_job_result`). Migration:
 
 - If the operation becomes a **native Databricks task**, drop the resumption mechanics.
-- If the **external job is retained**, preserve the external job id / idempotency / reattachment or
-  **flag** for review — never silently turn a resumable submission into a notebook that resubmits the
-  external job on every retry.
+- If the **external job is retained**, preserve the external job id / idempotency / reattachment or **flag** for review — never silently turn a resumable submission into a notebook that resubmits the external job on every retry.
 
 Reference: https://airflow.apache.org/docs/task-sdk/stable/resumable-job-mixin.html
 
@@ -91,8 +59,7 @@ Reference: https://airflow.apache.org/docs/task-sdk/stable/resumable-job-mixin.h
 
 ## Task SDK import equivalence (`airflow.sdk`)
 
-Airflow 3 exposes the stable authoring interface under `airflow.sdk`. Map these to the same
-handling as their Airflow 2 equivalents:
+Airflow 3 exposes the stable authoring interface under `airflow.sdk`. Map these to the same handling as their Airflow 2 equivalents:
 
 | Airflow 3 (`airflow.sdk`) | Airflow 2 equivalent | Handling |
 |---|---|---|
@@ -109,9 +76,7 @@ handling as their Airflow 2 equivalents:
 
 ## Standard-provider import paths (`apache-airflow-providers-standard`)
 
-In Airflow 3, common operators and sensors moved out of `airflow-core` into the
-`apache-airflow-providers-standard` provider. The **classes and their DABs mappings are unchanged**
-— only the import path differs. Recognize both the new and legacy paths.
+In Airflow 3, common operators and sensors moved out of `airflow-core` into the `apache-airflow-providers-standard` provider. The **classes and their DABs mappings are unchanged** — only the import path differs. Recognize both the new and legacy paths.
 
 | Class | Airflow 3 import path | DABs mapping (unchanged) |
 |---|---|---|
@@ -133,34 +98,24 @@ In Airflow 3, common operators and sensors moved out of `airflow-core` into the
 > There is no `DateTimeSensor` in the standard provider; use `TimeSensor` / `TimeDeltaSensor` /
 > `DayOfWeekSensor`.
 
-**Legacy paths:** In Airflow 3.0–3.1 the old `airflow.operators.*` / `airflow.sensors.*` import
-paths still work with deprecation warnings and are slated for removal in a later release. Recognize
-**both** the legacy and standard-provider paths so a DAG on either side converts identically.
+**Legacy paths:** In Airflow 3.0–3.1 the old `airflow.operators.*` / `airflow.sensors.*` import paths still work with deprecation warnings and are slated for removal in a later release. Recognize **both** the legacy and standard-provider paths so a DAG on either side converts identically.
 
 ---
 
 ## Assets vs Datasets, and asset scheduling
 
-"Datasets" (Airflow 2) are renamed **Assets** (Airflow 3): `airflow.sdk.Asset` replaces
-`airflow.datasets.Dataset`. Asset-based **scheduling** maps to Lakeflow `trigger.table_update`;
-the boolean/list/time-combined forms and the **Asset → UC-table resolution rule** are documented
-in `references/schedule-trigger-mapping.md` (§ Timetable, Dataset, and Asset Scheduling). Summary:
+"Datasets" (Airflow 2) are renamed **Assets** (Airflow 3): `airflow.sdk.Asset` replaces `airflow.datasets.Dataset`. Asset-based **scheduling** maps to Lakeflow `trigger.table_update`; the boolean/list/time-combined forms and the **Asset → UC-table resolution rule** are documented in `references/schedule-trigger-mapping.md` (§ Timetable, Dataset, and Asset Scheduling). Summary:
 
 - `schedule=[asset]` → `trigger.table_update` on the resolved table (single).
 - `schedule=[a, b]` (list = ALL) → `condition: ALL_UPDATED`; `a | b` → `ANY_UPDATED`; `a & b` → `ALL_UPDATED`.
-- `AssetOrTimeSchedule(...)` (time **and** asset) → **flag**; a single Lakeflow job takes a schedule
-  **or** a trigger, not both as a clean 1:1.
-- An `Asset` URI is an arbitrary string, so map to a table **only** via explicit
-  `extra={"databricks_table": "catalog.schema.table"}`, a user-supplied mapping, or the skill-local
-  `x-databricks-table:` scheme — otherwise **flag**. Never infer a table from an arbitrary URI.
+- `AssetOrTimeSchedule(...)` — and its Airflow 2.4–2.10 spelling `DatasetOrTimeSchedule(...)` — carries time **and** asset conditions → **flag and generate a manual job with neither arm**; a single Lakeflow job takes a schedule **or** a trigger, not both as a clean 1:1. Require the user to select the time arm, asset arm, or split jobs before adding automation.
+- An `Asset` URI is an arbitrary string, so map to a table **only** via explicit `extra={"databricks_table": "catalog.schema.table"}`, a user-supplied mapping, or the skill-local `x-databricks-table:` scheme — otherwise **flag**. Never infer a table from an arbitrary URI.
 
 ### `@asset` and related — flag, do not auto-map
 
-These Airflow 3 asset features have no clean Lakeflow equivalent; **flag** them in
-`MIGRATION_NOTES.md` rather than inventing a mapping:
+These Airflow 3 asset features have no clean Lakeflow equivalent; **flag** them in `MIGRATION_NOTES.md` rather than inventing a mapping:
 
-- The **`@asset` decorator** (defining asset-producing workflows) — distinct from using `Asset`
-  objects in `schedule=`.
+- The **`@asset` decorator** (defining asset-producing workflows) — distinct from using `Asset` objects in `schedule=`.
 - **`AssetWatcher`** and event-driven asset watchers.
 - **Asset aliases**.
 - **DAG versioning / DAG bundles** (a deployment concept, not a task-graph one).
@@ -181,12 +136,7 @@ These Airflow 3 asset features have no clean Lakeflow equivalent; **flag** them
 
 ## Recognize → safe-map → flag checklist
 
-1. **Recognize imports.** Accept `airflow.sdk.*` and `airflow.providers.standard.{operators,sensors}.*`
-   in addition to the Airflow 2 `airflow.operators.*` / `airflow.sensors.*` paths. A task whose
-   import path is unrecognized must be surfaced, never dropped.
-2. **Map operators/sensors** through the existing Tier tables in `operator-mapping.md` — the mapping
-   is import-path-independent.
+1. **Recognize imports.** Accept `airflow.sdk.*` and `airflow.providers.standard.{operators,sensors}.*` in addition to the Airflow 2 `airflow.operators.*` / `airflow.sensors.*` paths. A task whose import path is unrecognized must be surfaced, never dropped.
+2. **Map operators/sensors** through the existing Tier tables in `operator-mapping.md` — the mapping is import-path-independent.
 3. **Map asset scheduling** per the resolution rule (above / `schedule-trigger-mapping.md`).
-4. **Flag** `@asset`, `AssetWatcher`, asset aliases, DAG versioning, deadline alerts,
-   `AssetOrTimeSchedule`, and any asset whose URI does not resolve to a UC table — in
-   `MIGRATION_NOTES.md`, with the reason.
+4. **Flag** `@asset`, `AssetWatcher`, asset aliases, DAG versioning, deadline alerts, `AssetOrTimeSchedule`, and any asset whose URI does not resolve to a UC table — in `MIGRATION_NOTES.md`, with the reason.
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/dab-schema-reference.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/dab-schema-reference.md
index 16f5b2a..1d91d17 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/dab-schema-reference.md
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/dab-schema-reference.md
@@ -1,4 +1,4 @@
-# Databricks Asset Bundles YAML Schema Reference
+# Databricks Declarative Automation Bundles YAML Schema Reference
 
 Condensed reference for generating DABs configuration files. Covers all task types, triggers, clusters, and job-level configuration supported as of Jan 2026.
 
@@ -85,6 +85,8 @@ resources:
         team: data-engineering
         source: airflow-migration
       max_concurrent_runs: 1
+      queue:
+        enabled: true        # Required by this skill for file-arrival jobs.
       timeout_seconds: 3600
 
       # Schedule (see Schedule section below)
@@ -561,6 +563,8 @@ Event-driven triggers (mutually exclusive with `schedule`).
 ### File Arrival
 
 ```yaml
+queue:
+  enabled: true                                          # Prevent triggered runs from being skipped at concurrency limits.
 trigger:
   file_arrival:
     url: "s3://bucket/path/"                             # Required. UC external location or volume URL.
@@ -568,6 +572,8 @@ trigger:
     wait_after_last_change_seconds: 60                   # Optional. Minimum allowed is 60.
 ```
 
+File-arrival triggers recurse through subdirectories and only new arrivals start runs. Ingestion must discover the same root recursively, and deployment needs a manual bootstrap run for existing files. Preserve the source sensor's filename/glob predicate in Auto Loader or custom discovery because the trigger URL is a prefix.
+
 ### Table Update
 
 ```yaml
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/hadoop-migration-guide.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/hadoop-migration-guide.md
index ebab686..f236681 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/hadoop-migration-guide.md
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/hadoop-migration-guide.md
@@ -1,6 +1,6 @@
 # Hadoop/HDFS to Databricks Migration Guide
 
-Reference for converting on-prem Airflow DAGs that orchestrate Spark jobs on Hadoop/YARN clusters to Databricks Asset Bundles. Covers HDFS path conversion, YARN Spark config cleanup, Hive metastore migration, data ingestion alternatives, and detection of `spark-submit` commands embedded in BashOperator/SSHOperator tasks.
+Reference for converting on-prem Airflow DAGs that orchestrate Spark jobs on Hadoop/YARN clusters to Databricks Declarative Automation Bundles (formerly Databricks Asset Bundles; DABs). Covers HDFS path conversion, YARN Spark config cleanup, Hive metastore migration, data ingestion alternatives, and detection of `spark-submit` commands embedded in BashOperator/SSHOperator tasks.
 
 ---
 
diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/operator-mapping.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/operator-mapping.md
index 690e3da..938920a 100644
--- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/operator-mapping.md
+++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/operator-mapping.md
@@ -1,6 +1,6 @@
 # Airflow Operator to DABs Task Type Mapping
 
-Authoritative reference for converting Apache Airflow operators to Databricks Asset Bundles (DABs) job task types. All task types confirmed supported in DABs YAML as of Jan 2026.
+Authoritative reference for converting Apache Airflow operators to Databricks Declarative Automation Bundles job task types (formerly Databricks Asset Bundles; DABs). Task and trigger fields are checked against current Databricks documentation and the Databricks CLI bundle schema during validation.
 
 ---
 
@@ -39,6 +39,22 @@ mapping**. A `conn_id` name or host string is a **hint only** — surface it as
 review**. **Never export or inline credentials** — auth becomes a UC connection (federation/Connect)
 or `dbutils.secrets` (connector notebook), created out-of-band.
 
+**Unresolved executable identifiers fail closed.** Never turn a suggestive `conn_id`, host, filename, bare table, or surrounding example into a guessed catalog/schema/table FQN, external-location URL, connection name, warehouse ID, job ID, or other executable value. Emit a **required bundle variable with no default** and use its substitution at the execution site, or use a deliberately invalid `` placeholder when that resource cannot accept a variable. List the missing value and its source of truth in `MIGRATION_NOTES.md`. Do not emit plausible defaults such as `main.default.
` or a catalog inferred from `snowflake_conn_id`; a generated project must stop before deployment rather than run against the wrong object. + +```yaml +variables: + source_table_fqn: + description: Required three-level Unity Catalog source table + +resources: + jobs: + consumer_job: + trigger: + table_update: + table_names: + - "${var.source_table_fqn}" +``` + **Lakeflow Connect eligibility** (all must hold, else flag): recurring ingestion/replication; a connector exists for the source; **Connect can create and own the destination streaming table** (it fails if the destination already exists — an existing target needs a new landing table + downstream @@ -56,9 +72,9 @@ Applies across all tiers and to **any Airflow version** — deferrability has ex worker-efficiency mechanism (release a worker slot while waiting) and does **not** change what the task does, so **ignore the deferrability and map the underlying operation normally**: -- Drop `deferrable=True` / the `*DeferrableOperator` suffix, triggerer configuration, and `poke_interval`. +- Drop `deferrable=True` / the `*DeferrableOperator` suffix and triggerer configuration. - Keep task **timeout** and **retry** settings where they apply. -- Generate **no polling** — Lakeflow owns waiting/queueing/triggers natively (sensors → job triggers). +- **A sensor that converts to a job-level trigger generates no polling** and drops `poke_interval` — Lakeflow owns waiting/queueing/triggers natively. A sensor that stays a task (mid-graph, an arbitrary predicate, or a return value consumed downstream — see the Tier-3 sensor sections) keeps its polling loop and its `poke_interval` at every `mode`/`deferrable` setting. - Preserve **wait-for-completion** behavior **only** when Databricks submits to an external system via a notebook/wheel and the original operator waited; `wait_for_completion=False` → submit and return. - The `[operators] default_deferrable` config only affects operators that support switching modes — it @@ -219,9 +235,35 @@ plain `notebook_task`: | `@task.virtualenv` / `@task.external_python` | `notebook_task`; **flag** the environment/deps — recreate via a serverless environment or `%pip install`, record in `MIGRATION_NOTES.md`. | | `@task.sensor` | Tier-3 job-level trigger **only** when it is a *root* sensor whose `PokeReturnValue` output is unused; otherwise **flag** or keep the polling logic in a notebook. | | `@task.run_if` / `@task.skip_if` | The predicate is arbitrary runtime context, but Lakeflow `run_if` evaluates only **upstream task states**. Map a status-equivalent predicate to `run_if`; map other predicates through a `condition_task`; **flag** anything not reducible to either. | -| `@setup` / `@teardown` | **Flag** — no native Lakeflow setup/teardown lifecycle. Emit as ordinary first/last tasks and note the semantic loss. | +| `@setup` / `@teardown` | **Flag** — no native Lakeflow setup/teardown lifecycle. Emit as ordinary first/last tasks only with explicit `depends_on`/`run_if`, and document both semantic losses: Airflow schedules teardown only after its setup succeeds; an ordinary teardown failure affects the Lakeflow job result unless explicitly redesigned, while Airflow excludes teardown failure from DAG-run status by default unless configured otherwise. | | `@task.kubernetes` / `@task.docker` / other provider `@task.*` | **Flag** — route through the matching Tier-2/Tier-4 operator rule (`KubernetesPodOperator`, `DockerOperator`, …). | +**Retained file-sensor discovery.** When an `@task.sensor` or `PythonSensor` returns a file collection consumed downstream, preserve the source callable's listing semantics instead of replacing it with a shallow directory read. **Recursive prefix listings must remain recursive**; for example, an object-store hook's `list_keys(prefix=...)` includes nested keys, while one `dbutils.fs.ls()` call returns only direct children. Generate a recursive walk or an equivalent paginated object-store/Auto Loader listing, retain the original glob/suffix filters, sorting, timeout, and size guard, and document any intentional scope change. + +```python +MAX_FILES = 10_000 # walk bound; raising past this is a failure +LISTING_TIMEOUT_SECONDS = 300 + +def list_files_recursive(root: str) -> list[str]: + deadline = time.monotonic() + LISTING_TIMEOUT_SECONDS + pending, files = [root], [] + while pending: + if time.monotonic() > deadline: + raise TimeoutError(f"Listing {root} exceeded {LISTING_TIMEOUT_SECONDS}s") + for entry in dbutils.fs.ls(pending.pop()): + # A directory entry's path ends in "/" on every dbutils implementation; + # entry.isDir() is absent from the SDK/Connect FileInfo. + if entry.path.endswith("/"): + pending.append(entry.path) + else: + files.append(entry.path) + if len(files) > MAX_FILES: + raise RuntimeError(f"{root} exceeds {MAX_FILES} files; narrow the prefix") + return sorted(files) +``` + +Call `list_files_recursive(source_path)` inside every polling attempt, then apply the source predicate (for example, `path.endswith(".json")`) and enforce the 48 KiB task-value limit before publishing the collection. Keep the walk in a `notebook_task` so `dbutils.fs` is available. Do not replace the helper with a one-level list comprehension over `dbutils.fs.ls(source_path)`. If direct object-store pagination is required to preserve metadata or scale, use the cloud SDK with secrets/identity instead of `dbutils.fs.ls()`. + --- ### BashOperator @@ -544,9 +586,9 @@ COPY_OPTIONS ('force' = 'true') ### SQLExecuteQueryOperator -`SQLExecuteQueryOperator` is connection-agnostic. Its **DABs task type is `sql_task` only when the -resolved connection targets Databricks SQL**; otherwise apply the source-aware classification step -above. +**DABs task type:** `sql_task` when the resolved connection targets Databricks SQL; otherwise routed by the source-aware classification step above. + +`SQLExecuteQueryOperator` is connection-agnostic, so the resolved connection decides the mapping. - **Databricks SQL connection** → `sql_task` (the mapping shown below). If SQL is inline, extract it to a `.sql` file and reference via `sql_task.file.path`; if it references an existing Databricks SQL query, @@ -746,7 +788,7 @@ Fall back to a single `dbt_task` when: **dbt Cloud (`DbtCloudRunJobOperator`) is NOT a `dbt_task` fallback** — `dbt_task` runs dbt Core and cannot trigger a dbt Cloud job. Route it to Tier 4 (notebook calling the dbt Cloud API, or migrate the project to Databricks). -For factory mode, generate the artifacts described in **dbt factory mode — generated artifacts** under the cosmos section in Tier 2 (the mechanics are identical for CLI operators; extract `project_dir`, `profiles_dir`, `target`, `vars`, and selectors from the operator arguments instead of cosmos configs). Multiple dbt operator tasks over the same project (e.g. `dbt_seed >> dbt_run >> dbt_test`) collapse into ONE factory job with ONE `run_job_task` hop — the manifest explosion already covers seeds, models, snapshots, and tests, with ordering derived from the dbt DAG instead of the coarse seed→run→test chain. Note the semantic shift in `MIGRATION_NOTES.md`: tests run after each model and gate downstream nodes, instead of one test phase at the end. +For factory mode, generate the artifacts described in **dbt factory mode — generated artifacts** under the cosmos section in Tier 2 (the mechanics are identical for CLI operators; extract `project_dir`, `profiles_dir`, `target`, `vars`, and selectors from the operator arguments instead of cosmos configs). Multiple dbt operator tasks over the same project (e.g. `dbt_seed >> dbt_run >> dbt_test`) collapse into ONE factory job with ONE `run_job_task` hop — the manifest explosion already covers seeds, models, snapshots, and tests, with ordering derived from the dbt DAG instead of the coarse seed→run→test chain. Note both semantic shifts in `MIGRATION_NOTES.md`: tests run after each model and gate downstream nodes instead of one test phase at the end, and the coarse Airflow stages' separate retry envelopes are replaced by per-node Lakeflow repair while the `run_job_task` hop itself must not retry the entire generated job. #### Fallback mapping: single `dbt_task` @@ -874,6 +916,8 @@ ssh_spark = SSHOperator( These operators require reasoning about intent to determine the best DABs equivalent. +**Retry-boundary rule.** Whenever a mapping consolidates multiple Airflow tasks, mapped stages, or lifecycle steps into one Lakeflow task or one `run_job_task` hop, compare the retry boundaries before and after conversion. Add a **collapsed retry envelope** entry to `MIGRATION_NOTES.md` naming the original per-task retries, the new larger rerun unit, and any side effects that could repeat. Do not silently copy one task's retry count onto a consolidated hop. + --- ### BranchPythonOperator / ShortCircuitOperator @@ -945,6 +989,54 @@ branch = BranchPythonOperator( --- +### BranchDateTimeOperator + +**DABs task type:** evaluator `notebook_task` + `condition_task` + `depends_on.outcome` + +Preserve `target_lower`, `target_upper`, `follow_task_ids_if_true`, `follow_task_ids_if_false`, and `use_task_logical_date`. Generate a small evaluator notebook that computes an inclusive, timezone-aware range test and writes a string task value such as `in_range=true|false`; route both branch lists through one `condition_task`. Preserve Airflow's time-only rollover rule: when `target_lower` is later than `target_upper`, treat the upper bound as the following day. When `use_task_logical_date=True`, define a `logical_datetime` job parameter (default `{{job.trigger.time.iso_datetime}}` for a scheduled job and overridable with `{{backfill.iso_datetime}}`) rather than using a date-only value, so scheduled runs and native backfills preserve time-of-day semantics. When it is false, use `{{job.start_time.iso_datetime}}` as an evaluator parameter. A missing or dynamic range bound is manual review; never substitute today's date. + +`logical_datetime` is the full-precision form of the same logical instant the DAG-wide `run_date` parameter carries. When a DAG needs both, declare `logical_datetime` and derive `run_date` from its date part rather than defaulting the two independently, and give both the same backfill override so a replayed window moves them together. A `logical_datetime` left at the scheduled default while `run_date` is overridden evaluates the branch against the wrong window for the entire replay. + +```yaml +- task_key: evaluate_datetime_branch + notebook_task: + notebook_path: ../src/evaluate_datetime_branch.py + base_parameters: + logical_datetime: "{{job.parameters.logical_datetime}}" + +- task_key: choose_datetime_branch + depends_on: + - task_key: evaluate_datetime_branch + condition_task: + left: "{{tasks.evaluate_datetime_branch.values.in_range}}" + op: EQUAL_TO + right: "true" +``` + +Record any loss when Airflow branch IDs select more than two independent downstream sets or when downstream trigger rules depend on Airflow skip propagation. + +--- + +### BranchDayOfWeekOperator + +**DABs task type:** evaluator `notebook_task` + `condition_task` + `depends_on.outcome` + +Preserve `week_day`, `use_task_logical_date`, DAG timezone, and both branch lists. The evaluator parses the same logical parameter the DAG already declares — `logical_datetime` when the DAG has one, otherwise `run_date` — when `use_task_logical_date=True`, or the job start timestamp otherwise, converts it to the DAG timezone, computes the weekday, and writes `matches_day=true|false`. Do not rewrite this branch as a cron schedule unless it is a root task and changing the entire DAG's run cadence is explicitly acceptable; a mid-DAG branch controls only part of the graph. + +```yaml +- task_key: choose_weekday_branch + depends_on: + - task_key: evaluate_weekday_branch + condition_task: + left: "{{tasks.evaluate_weekday_branch.values.matches_day}}" + op: EQUAL_TO + right: "true" +``` + +Normalize Airflow weekday enum/string values in the evaluator and flag dynamic `week_day` expressions or branch fan-out that cannot be represented by a single boolean outcome. + +--- + ### PythonVirtualenvOperator / ExternalPythonOperator **DABs task type:** `python_wheel_task` or `notebook_task` @@ -1581,6 +1673,44 @@ Airflow sensors that wait for external conditions map to DABs job-level triggers --- +### BashSensor + +**DABs equivalent:** supported job-level trigger when the command is a provably equivalent root condition; otherwise a polling `notebook_task` + +`BashSensor` is an arbitrary shell predicate: exit code `0` succeeds and any other code is retried until timeout. Do not convert it to `file_arrival` merely because the command contains a path. Use a job-level file/table trigger only when the command is a root sensor, its output is unused, and its complete predicate is exactly a supported external arrival/update condition with a resolved location or table. Otherwise extract the command into a notebook loop using `subprocess.run`, preserving environment parameters, `poke_interval`, timeout, failure output, and the original success-code contract. A constant `exit 0` succeeds on the first probe; a constant nonzero command still waits to timeout. Preserve `soft_fail=False` by raising on timeout. `soft_fail=True` raises `AirflowSkipException`, and under the default `all_success` trigger rule that skip propagates to the whole downstream subgraph, so **always** gate on the result: complete the poller with `sensor_satisfied=false` and emit a `condition_task` on that value that gates every downstream task the sensor fed. Record the state change in `MIGRATION_NOTES.md` (Lakeflow shows a success plus a false condition where Airflow showed a skip). Reject destructive or side-effecting predicates for automatic polling and flag them for redesign. + +```yaml +- task_key: wait_for_shell_condition + timeout_seconds: 3600 + notebook_task: + notebook_path: ../src/wait_for_shell_condition.py + base_parameters: + poke_interval_seconds: "60" +``` + +The notebook task remains in the original graph; only a true root-trigger conversion removes the sensor task. Record that notebook polling consumes compute and consider replacing the external contract with a file/table event. + +--- + +### PythonSensor + +**DABs equivalent:** supported job-level trigger when the callable is a provably equivalent root condition; otherwise a polling `notebook_task` + +Extract and inspect the `python_callable`, `op_args`, and `op_kwargs`. Convert to `trigger.file_arrival` or `trigger.table_update` only when the root callable is wholly reducible to that resolved external event and its return/XCom value is unused. An arbitrary Python predicate, a mid-graph sensor, or a `PokeReturnValue` consumed downstream stays as a notebook that polls until truthy, preserving `poke_interval`, timeout, parameters, exception behavior, and any returned value through `dbutils.jobs.taskValues`. A callable that always returns true succeeds on its first probe; one that always returns false still waits to timeout. Preserve `soft_fail=False` by raising on timeout. `soft_fail=True` raises `AirflowSkipException`, and under the default `all_success` trigger rule that skip propagates to the whole downstream subgraph, so **always** gate on the result: complete the poller with `sensor_satisfied=false` and emit a `condition_task` on that value that gates every downstream task the sensor fed. Record the state change in `MIGRATION_NOTES.md` (Lakeflow shows a success plus a false condition where Airflow showed a skip). Airflow context, Connections, and Variables must become explicit job parameters, required bundle variables, UC connections, or secrets; unresolved dependencies are manual review. + +```yaml +- task_key: wait_for_python_condition + timeout_seconds: 3600 + notebook_task: + notebook_path: ../src/wait_for_python_condition.py + base_parameters: + poke_interval_seconds: "60" +``` + +Do not assume that a constant-looking example callable is safe to run only once: preserve polling unless event equivalence is proven. + +--- + ### DatabricksSqlSensor / DatabricksSQLStatementsSensor **DABs equivalent:** `depends_on`, `trigger.table_update`, or polling task (intent-dependent) @@ -1683,6 +1813,8 @@ wait_for_data = HdfsSensor( **DABs YAML (job-level trigger):** ```yaml +queue: + enabled: true trigger: file_arrival: url: s3://datalake-bucket/data/landing/ @@ -1719,6 +1851,8 @@ resources: jobs: process_upload_job: name: process-upload-job + queue: + enabled: true trigger: file_arrival: url: s3://data-landing/incoming/ @@ -1729,6 +1863,8 @@ resources: notebook_path: ../src/process_upload.py ``` +Apply the complete file-arrival contract from `references/schedule-trigger-mapping.md`: the trigger and ingestion discovery recurse over the same root, the ingestion task preserves the original key/glob filter, and `MIGRATION_NOTES.md` requires an initial manual run for existing files. Keep `queue.enabled: true` on every generated file-arrival job. + --- ### ExternalTaskSensor diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/schedule-trigger-mapping.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/schedule-trigger-mapping.md index cb0d608..3bc7957 100644 --- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/schedule-trigger-mapping.md +++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/schedule-trigger-mapping.md @@ -1,6 +1,6 @@ # Airflow Schedule and Trigger Mapping Reference -Maps Airflow scheduling mechanisms (cron expressions, presets, sensors) to Databricks Asset Bundles schedule and trigger configurations. +Maps Airflow scheduling mechanisms (cron expressions, presets, sensors) to Databricks Declarative Automation Bundles schedule and trigger configurations (formerly Databricks Asset Bundles; DABs). --- @@ -117,19 +117,27 @@ wait_for_data = S3KeySensor( ) ``` -**DABs:** +**Bundle job resource excerpt:** ```yaml -trigger: - file_arrival: - url: s3://landing-zone/data/ - min_time_between_triggers_seconds: 60 - wait_after_last_change_seconds: 60 +process_landing_job: + queue: + enabled: true + trigger: + file_arrival: + url: s3://landing-zone/data/ + min_time_between_triggers_seconds: 60 + wait_after_last_change_seconds: 60 ``` **Key differences:** + - Airflow sensors are task-level (block one task). DABs triggers are job-level (start the whole job). - Move the sensor to the job trigger. Downstream tasks that depended on the sensor now just run as the first task(s) in the job. +- Always set `queue.enabled: true` on a file-arrival job so an arrival detected while the job is at its concurrency limit waits instead of producing a skipped run. +- A file-arrival trigger is recursive: it watches new files in every subdirectory below `url`. Configure Auto Loader or any custom discovery code to scan the same root recursively, and preserve the original sensor's filename/glob filtering in the ingestion task. If the ingestion code lists only the top-level directory while the trigger watches descendants, nested arrivals can start runs that never process those files. +- Only new arrivals trigger a run. Files already present when the trigger is created do not bootstrap the job. Add a deployment action to `MIGRATION_NOTES.md`: run the job once manually to process existing files, normally with Auto Loader `cloudFiles.includeExistingFiles=true` and a durable checkpoint, then let the trigger handle later arrivals. +- Point `url` at a Unity Catalog external location or volume and enable managed file events when available. A trigger URL is a prefix rather than the original wildcard, so record any broadened trigger scope and enforce the original suffix/glob in ingestion. --- @@ -202,7 +210,7 @@ both `Dataset(...)` (Airflow 2) and `Asset(...)` (Airflow 3). See `references/ai | `schedule=[asset_a, asset_b]` (list — Airflow: ALL updated) | `trigger.table_update` on both tables with `condition: ALL_UPDATED`. | | `schedule=(asset_a \| asset_b)` (OR) | `trigger.table_update` with `condition: ANY_UPDATED`. | | `schedule=(asset_a & asset_b)` (AND) | `trigger.table_update` with `condition: ALL_UPDATED`. | -| `AssetOrTimeSchedule(timetable=..., assets=...)` (time **and** asset) | **Flag** — a single Lakeflow job takes either a `schedule` **or** a trigger, not both as a clean 1:1. Choose the dominant intent (or split), and record the tradeoff in `MIGRATION_NOTES.md`. | +| `AssetOrTimeSchedule(timetable=..., assets=...)` / `DatasetOrTimeSchedule(...)` (time **and** asset) | **Flag and emit a manual job. Do not emit either arm automatically.** A single Lakeflow job takes either a `schedule` or a trigger, not both as a clean 1:1. Generate neither `schedule` nor `trigger` until the user chooses the time arm, the asset arm, or split jobs; record that required decision in `MIGRATION_NOTES.md`. | | Custom `Timetable` subclass | Flag for manual review and map to `schedule` or `trigger` based on business intent. | | `@continuous` | Use job-level `continuous` (not periodic trigger). | diff --git a/src/flowx/agentic.py b/src/flowx/agentic.py index 663384e..5592240 100644 --- a/src/flowx/agentic.py +++ b/src/flowx/agentic.py @@ -11,12 +11,13 @@ import copy import hashlib import json +import posixpath import re import shutil import tempfile import textwrap from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any from flowx.ir_serde import pipeline_to_dict @@ -25,6 +26,8 @@ CONTRACT_VERSION = "1" PROVIDER_NAME = "airflow-to-dabs" PROVIDER_REPOSITORY = "https://github.com/park-peter/airflow-to-dabs" +_PROVIDER_MANIFEST_PATH = PurePosixPath("providers/flowx-gap-resolver/provider.json") +_PROVIDER_PIN_FIELD = "flowx_pin" _ALLOWED_REPLACEMENT_KINDS = ("notebook", "sql", "spark_python") _RESOLUTION_STATUSES = {"resolved", "needs_input", "deferred"} @@ -110,9 +113,9 @@ class GapEnvelope: dag_settings: dict[str, Any] reason: dict[str, str] - def as_dict(self) -> dict[str, Any]: + def as_dict(self, *, provider: dict[str, str] | None = None) -> dict[str, Any]: """Returns the public GapEnvelope v1 representation.""" - provider = _provider_identity() + provider = _provider_identity() if provider is None else provider payload = { "contract_version": CONTRACT_VERSION, "gap_id": self.gap_id, @@ -789,6 +792,7 @@ def _build_gap_envelopes( provider_sha256: str, ) -> list[dict[str, Any]]: envelopes: list[dict[str, Any]] = [] + provider = _provider_identity() for pipeline in _pipeline_list(baseline): if pipeline.get("migration_status") == "excluded": continue @@ -878,7 +882,7 @@ def _build_gap_envelopes( "message": str(task.get("comment") or matched_finding.get("message", "")), }, ) - envelopes.append(envelope.as_dict()) + envelopes.append(envelope.as_dict(provider=provider)) ordered = sorted(envelopes, key=lambda item: (item["pipeline_name"], item["task_path"])) gap_ids = [item["gap_id"] for item in ordered] if len(gap_ids) != len(set(gap_ids)): @@ -1526,27 +1530,125 @@ def _provider_context_path() -> Path: return source +def _provider_release_version(tag: Any) -> str: + if not isinstance(tag, str) or re.fullmatch(r"v[0-9A-Za-z][0-9A-Za-z.+-]*", tag) is None: + raise AgenticContractError(f"provider release tag is invalid: {tag!r}") + return tag[1:] + + +def _resolve_provider_path(base: PurePosixPath, relative: str) -> PurePosixPath: + if not relative or PurePosixPath(relative).is_absolute(): + raise AgenticContractError(f"provider manifest contains an unsafe path: {relative!r}") + normalized = PurePosixPath(posixpath.normpath((base / relative).as_posix())) + if normalized.as_posix() == ".." or normalized.as_posix().startswith("../"): + raise AgenticContractError(f"provider manifest path escapes its root: {relative!r}") + return normalized + + +def _provider_allowlisted_paths(manifest: dict[str, Any]) -> set[PurePosixPath]: + base = _PROVIDER_MANIFEST_PATH.parent + interface = manifest.get("interface") + if not isinstance(interface, dict) or interface.get("contract_versions") != [CONTRACT_VERSION]: + raise AgenticContractError(f"provider manifest must declare contract version {CONTRACT_VERSION}") + paths = { + _PROVIDER_MANIFEST_PATH, + _resolve_provider_path(base, str(interface.get("entrypoint", ""))), + } + knowledge = manifest.get("knowledge") + fixtures = manifest.get("fixtures") + if not isinstance(knowledge, list) or not isinstance(fixtures, list): + raise AgenticContractError("provider manifest knowledge and fixtures must be lists") + for item in knowledge: + if not isinstance(item, dict) or not isinstance(item.get("path"), str): + raise AgenticContractError("every provider knowledge entry requires a path") + paths.add(_resolve_provider_path(base, item["path"])) + for item in fixtures: + if not isinstance(item, str): + raise AgenticContractError("every provider fixture entry must be a path string") + paths.add(_resolve_provider_path(base, item)) + return paths + + +def _canonical_provider_bytes(path: PurePosixPath, data: bytes, *, strip_pin: bool = False) -> bytes: + if path.suffix != ".json": + return data + try: + value = json.loads(data) + except json.JSONDecodeError as error: + raise AgenticContractError(f"provider file {path} contains invalid JSON: {error}") from error + if not isinstance(value, dict): + raise AgenticContractError(f"provider file {path} must contain a JSON object") + if strip_pin: + value.pop(_PROVIDER_PIN_FIELD, None) + return _json_bytes(value) + + +def _provider_content_digest(root: Path, manifest: dict[str, Any]) -> str: + allowlisted = _provider_allowlisted_paths(manifest) + actual: set[PurePosixPath] = set() + for local_path in root.rglob("*"): + if local_path.is_symlink(): + raise AgenticContractError(f"provider context cannot contain symlinks: {local_path.relative_to(root)}") + if local_path.is_file(): + actual.add(PurePosixPath(local_path.relative_to(root).as_posix())) + unexpected = sorted(actual - allowlisted, key=lambda item: item.as_posix()) + if unexpected: + raise AgenticContractError( + "provider context contains files outside its manifest allowlist: " + + ", ".join(path.as_posix() for path in unexpected) + ) + + files: dict[PurePosixPath, bytes] = {} + for relative_path in sorted(allowlisted, key=lambda item: item.as_posix()): + local = root / relative_path.as_posix() + if not local.is_file(): + raise AgenticContractError(f"provider reference is missing: {relative_path.as_posix()}") + data = local.read_bytes() + canonical = _canonical_provider_bytes(relative_path, data) + if data != canonical: + raise AgenticContractError(f"provider JSON is not canonical: {relative_path.as_posix()}") + files[relative_path] = _canonical_provider_bytes( + relative_path, + data, + strip_pin=relative_path == _PROVIDER_MANIFEST_PATH, + ) + + digest = hashlib.sha256() + for relative_path in sorted(files, key=lambda item: item.as_posix()): + content = files[relative_path] + digest.update(relative_path.as_posix().encode()) + digest.update(b"\0") + digest.update(str(len(content)).encode()) + digest.update(b"\0") + digest.update(content) + return digest.hexdigest() + + def _provider_identity() -> dict[str, str]: - manifest_path = _provider_context_path() / "providers" / "flowx-gap-resolver" / "provider.json" + root = _provider_context_path() + manifest_path = root / _PROVIDER_MANIFEST_PATH.as_posix() try: manifest = _read_json_object(manifest_path) + provider = manifest.get("provider") + pin = manifest.get(_PROVIDER_PIN_FIELD) + if not isinstance(provider, dict) or not isinstance(pin, dict): + raise AgenticContractError("identity is invalid") + version = _provider_release_version(pin.get("tag")) + declared_version = provider.get("version") + if ( + provider.get("name") != PROVIDER_NAME + or provider.get("repository") != PROVIDER_REPOSITORY + or pin.get("repository") != PROVIDER_REPOSITORY + or pin.get("contract_version") != CONTRACT_VERSION + or (declared_version is not None and declared_version != version) + or re.fullmatch(r"[0-9a-f]{40}", str(pin.get("commit", ""))) is None + or re.fullmatch(r"[0-9a-f]{64}", str(pin.get("content_sha256", ""))) is None + ): + raise AgenticContractError("identity is invalid") + if _provider_content_digest(root, manifest) != pin["content_sha256"]: + raise AgenticContractError("content digest does not match the pinned release") except (OSError, json.JSONDecodeError, AgenticContractError) as error: - raise AgenticContractError(f"provider_unavailable: pinned {PROVIDER_NAME} manifest is invalid") from error - provider = manifest.get("provider") - pin = manifest.get("flowx_pin") - if not isinstance(provider, dict) or not isinstance(pin, dict): - raise AgenticContractError(f"provider_unavailable: pinned {PROVIDER_NAME} identity is invalid") - version = provider.get("version") - if ( - not isinstance(version, str) - or not version - or provider.get("name") != PROVIDER_NAME - or provider.get("repository") != PROVIDER_REPOSITORY - or pin.get("repository") != PROVIDER_REPOSITORY - or pin.get("tag") != f"v{version}" - or pin.get("contract_version") != CONTRACT_VERSION - ): - raise AgenticContractError(f"provider_unavailable: pinned {PROVIDER_NAME} identity is invalid") + raise AgenticContractError(f"provider_unavailable: pinned {PROVIDER_NAME} {error}") from error return {"name": PROVIDER_NAME, "version": version, "repository": PROVIDER_REPOSITORY} diff --git a/tests/unit/test_airflow_agentic_resolution.py b/tests/unit/test_airflow_agentic_resolution.py index c6d3f52..05e576f 100644 --- a/tests/unit/test_airflow_agentic_resolution.py +++ b/tests/unit/test_airflow_agentic_resolution.py @@ -787,11 +787,17 @@ def test_pinned_provider_fixtures_satisfy_the_flowx_contract() -> None: / "flowx-gap-resolver" ) provider = json.loads((root / "provider.json").read_text(encoding="utf-8")) + provider_identity = agentic_contract._provider_identity() - assert provider["provider"] == agentic_contract._provider_identity() + assert provider["provider"] == { + "name": provider_identity["name"], + "repository": provider_identity["repository"], + } for outcome in ("notebook", "sql", "spark-python", "needs-input", "deferred"): gap = json.loads((root / "fixtures" / f"gap-{outcome}.json").read_text(encoding="utf-8")) candidate = json.loads((root / "fixtures" / f"resolution-{outcome}.json").read_text(encoding="utf-8")) + gap["knowledge_provider"] = provider_identity + candidate["provider"] = provider_identity manifest = {"baseline_report_sha256": gap["baseline_report_sha256"]} resolution = _validate_candidate(candidate, gap_by_id={gap["gap_id"]: gap}, manifest=manifest) diff --git a/tests/unit/test_airflow_provider_sync.py b/tests/unit/test_airflow_provider_sync.py index 45cef3e..2c17ba7 100644 --- a/tests/unit/test_airflow_provider_sync.py +++ b/tests/unit/test_airflow_provider_sync.py @@ -3,11 +3,17 @@ from __future__ import annotations import json +import re import shutil import subprocess import sys from pathlib import Path +import pytest + +import flowx.agentic as agentic_contract +from flowx.agentic import AgenticContractError + ROOT = Path(__file__).parents[2] SCRIPT = ROOT / "scripts" / "sync_airflow_provider.py" PROVIDER = ROOT / "skills" / "flowx-resolve-airflow-gaps" / "references" / "airflow-to-dabs" @@ -22,6 +28,53 @@ def _check(destination: Path) -> subprocess.CompletedProcess[str]: ) +def _sync(checkout: Path, destination: Path, *, tag: str = "v9.8.7") -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--source", + str(checkout), + "--tag", + tag, + "--destination", + str(destination), + ], + check=False, + capture_output=True, + text=True, + ) + + +def _versionless_checkout(tmp_path: Path, *, tag: str = "v9.8.7") -> Path: + checkout = tmp_path / "upstream" + shutil.copytree(PROVIDER, checkout) + manifest_path = checkout / "providers" / "flowx-gap-resolver" / "provider.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["provider"].pop("version", None) + manifest.pop("flowx_pin") + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + subprocess.run(["git", "init", "-q"], cwd=checkout, check=True) + subprocess.run(["git", "add", "."], cwd=checkout, check=True) + subprocess.run( + [ + "git", + "-c", + "user.name=flowx-test", + "-c", + "user.email=flowx-test@example.com", + "commit", + "-qm", + "provider fixture", + "--no-verify", + ], + cwd=checkout, + check=True, + ) + subprocess.run(["git", "tag", tag], cwd=checkout, check=True) + return checkout + + def test_committed_airflow_provider_pin_is_valid() -> None: result = _check(PROVIDER) @@ -33,7 +86,57 @@ def test_committed_airflow_provider_pin_is_valid() -> None: "content_sha256": manifest["flowx_pin"]["content_sha256"], "tag": manifest["flowx_pin"]["tag"], } - assert pin["tag"] == f"v{manifest['provider']['version']}" + assert agentic_contract._provider_identity()["version"] == pin["tag"].removeprefix("v") + + +def test_airflow_provider_sync_accepts_versionless_manifest(tmp_path: Path) -> None: + checkout = _versionless_checkout(tmp_path) + destination = tmp_path / "vendored" + + result = _sync(checkout, destination) + + assert result.returncode == 0, result.stderr + manifest = json.loads( + (destination / "providers" / "flowx-gap-resolver" / "provider.json").read_text(encoding="utf-8") + ) + pin = json.loads(result.stdout) + resolved_commit = subprocess.check_output(["git", "rev-parse", "v9.8.7^{commit}"], cwd=checkout, text=True).strip() + assert "version" not in manifest["provider"] + assert pin == { + "tag": "v9.8.7", + "commit": resolved_commit, + "content_sha256": manifest["flowx_pin"]["content_sha256"], + } + assert re.fullmatch(r"[0-9a-f]{40}", pin["commit"]) + assert re.fullmatch(r"[0-9a-f]{64}", pin["content_sha256"]) + checked = _check(destination) + assert checked.returncode == 0, checked.stderr + assert json.loads(checked.stdout) == pin + + +def test_airflow_provider_runtime_derives_version_from_pin(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + checkout = _versionless_checkout(tmp_path) + destination = tmp_path / "vendored" + result = _sync(checkout, destination) + assert result.returncode == 0, result.stderr + monkeypatch.setattr(agentic_contract, "_provider_context_path", lambda: destination) + + assert agentic_contract._provider_identity() == { + "name": "airflow-to-dabs", + "version": "9.8.7", + "repository": "https://github.com/park-peter/airflow-to-dabs", + } + + +def test_airflow_provider_runtime_rejects_modified_content(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + destination = tmp_path / PROVIDER.name + shutil.copytree(PROVIDER, destination) + profile = destination / "providers" / "flowx-gap-resolver" / "PROFILE.md" + profile.write_text(profile.read_text(encoding="utf-8") + "\nmodified\n", encoding="utf-8") + monkeypatch.setattr(agentic_contract, "_provider_context_path", lambda: destination) + + with pytest.raises(AgenticContractError, match="content digest"): + agentic_contract._provider_identity() def test_airflow_provider_sync_requires_an_explicit_tag() -> None: From e88d8df323dea28a76d2d430cce6f38f664cbecb Mon Sep 17 00:00:00 2001 From: peter-park_data Date: Wed, 12 Aug 2026 13:15:13 -0700 Subject: [PATCH 70/77] pin/update airflow dbt-factory to 0.3.3 --- .../flowx-convert/sources/airflow-coverage.md | 10 ++- .../flowx-gap-resolver/provider.json | 4 +- .../references/dab-schema-reference.md | 2 +- .../references/operator-mapping.md | 6 +- src/flowx/models/ir.py | 4 +- .../activity_preparers/dbt_factory.py | 84 +++++++++++++------ src/flowx/sources/airflow/convert.py | 3 +- tests/unit/test_dbt_factory_preparer.py | 53 +++++++++++- 8 files changed, 125 insertions(+), 41 deletions(-) diff --git a/skills/flowx-convert/sources/airflow-coverage.md b/skills/flowx-convert/sources/airflow-coverage.md index afae398..e91e78f 100644 --- a/skills/flowx-convert/sources/airflow-coverage.md +++ b/skills/flowx-convert/sources/airflow-coverage.md @@ -73,8 +73,9 @@ decisions. A file sensor with a non-literal path, or a table/SQL sensor with no literal `sql` / `table_name`, also falls back to a placeholder. - **Dynamic dbt configuration.** Project/profile paths, selectors, excludes, vars, and full-refresh - flags must be statically visible. Missing project, profile, or manifest inputs produce a failing - setup-required placeholder rather than a partially deployable dbt job. + flags must be statically visible. Selectors, excludes, and vars are rendered by static explosion + only; in `--dbt-mode pydabs` they force a static fallback. Missing project, profile, or manifest + inputs produce a failing setup-required placeholder rather than a partially deployable dbt job. ## dbt factory mode @@ -90,8 +91,9 @@ mode with `--dbt-mode {static,pydabs}` on the convert phase (default `static`). `resources/__init__.py` package marker) at the bundle root, registers it under `databricks.yml` `python.resources`, generates a pinned uv `pyproject.toml` plus the dbt-factory-compatible runner, and copies the project/profile/manifest inputs. `bundle deploy` runs the hook to build the dbt job. - A source `--select` restriction falls back to static explosion so the generated per-node commands - can preserve dbt selector intersection semantics. + A source selector, exclusion, or `--vars` restriction falls back to static explosion: the factory + owns resource selection and parse context, so it rejects those options in the per-task dbt + commands. Static explosion applies them to the generated per-node commands instead. ## Priority for remaining follow-ups diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/provider.json b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/provider.json index ef4a96e..aefdf0d 100644 --- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/provider.json +++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/providers/flowx-gap-resolver/provider.json @@ -12,8 +12,8 @@ "fixtures/resolution-deferred.json" ], "flowx_pin": { - "commit": "867abee838a0734ff7f7878b04402ee08ff7f3d3", - "content_sha256": "3cc9cd3c8fa9253f7485f73216d31b0847f6fbeab4df15042bc7ab2ef6f4751e", + "commit": "dee6efe51b6264025dd69ee136b81029c15c1dbc", + "content_sha256": "87ee069f050eacc34db7c186770805fa3a9c83744120632b4753f673b809f7ae", "contract_version": "1", "repository": "https://github.com/park-peter/airflow-to-dabs", "tag": "v0.2.3" diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/dab-schema-reference.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/dab-schema-reference.md index 1d91d17..c987aa7 100644 --- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/dab-schema-reference.md +++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/dab-schema-reference.md @@ -689,7 +689,7 @@ dependencies: - dbt-core==1.11.12 # pin dbt-core too, not just the adapter ``` -For dbt factory mode, pin **both** `dbt-databricks` and `dbt-core` to the exact versions in the bundle venv. `dbt-databricks` alone allows a `dbt-core` range, but the factory glue imports the local `dbt-core` for its selector-exactness check — the runtime environment must resolve the identical `dbt-core` for that guarantee to hold. +For dbt factory mode, pin **both** `dbt-databricks` and `dbt-core` to the exact versions in the bundle venv. `dbt-databricks` alone allows a `dbt-core` range, but the runner injects a parse cache produced by the local `dbt-core` — the runtime environment must resolve the identical `dbt-core` version. --- diff --git a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/operator-mapping.md b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/operator-mapping.md index 938920a..665c92d 100644 --- a/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/operator-mapping.md +++ b/skills/flowx-resolve-airflow-gaps/references/airflow-to-dabs/references/operator-mapping.md @@ -769,9 +769,9 @@ Factory mode changes the bundle toolchain: it adds a PyDABs `python:` block, a ` | `deps`/`docs` only | not factory-eligible — use the single-`dbt_task` fallback | | Multiple commands | union of the above | -databricks-dbt-factory 0.3.1 selects every node by its full dot-joined FQN, emits one task per dbt test (including unit tests), and derives readable task keys (`_`, e.g. `orders_model`/`countries_seed`; bundled tests keyed `_test`) that are guaranteed unique and ≤100 chars — collisions are disambiguated by package/hash inside the factory. The glue post-processes that output: it prunes `depends_on` references to omitted node types (the factory emits dangling dependencies when a node type has no factory), and applies deploy-time fail-closed guards as defense-in-depth: generated selectors that do not resolve to exactly their own node — the check imports dbt's own `is_selected_node` matcher at deploy time, so it covers everything dbt's semantics cover (prefix matching, leaf shortcuts, versioned models, wildcard slurp, package-stripped retry) and also rejects a selector resolving to a single wrong node, or one whose FQN — package, any directory component, or name — contains anything outside `[A-Za-z0-9_.-]` (an allowlist checked over the full FQN, not just the leaf; hyphens are allowed since dbt path components use them; other characters would be reinterpreted by dbt's CLI selector grammar or corrupt the runner's `shlex.split`); plus a final check that the emitted task keys are unique (the factory already guarantees this, so this only trips on a factory regression). The runner also rejects any dbt command carrying its own `--vars` (both `--vars ` and `--vars=`) — vars must use the canonical `dbt_vars.json`/`dbt_vars` channel. +databricks-dbt-factory addresses each node with an intersected selector — its `fqn:` plus `package:`, `file:`, `resource_type:` and (for generic tests) `test_name:` terms, comma-joined as dbt's AND — emits one task per dbt test (including unit tests), and derives readable task keys (`_`, e.g. `orders_model`/`countries_seed`; bundled tests keyed `_test`) that are guaranteed unique and ≤100 chars. The factory validates every selector against dbt's own grammar and refuses to emit one it cannot address exactly. The glue post-processes the output: it prunes `depends_on` references to omitted node types (the factory emits dangling dependencies when a node type has no factory), confirms the emitted task keys are unique, and fails closed above the 1,000-task per-job limit. The runner rejects any dbt command carrying its own `--vars` (both `--vars ` and `--vars=`) — vars must use the canonical `dbt_vars.json`/`dbt_vars` channel. -**Task count and the 1,000-task per-job limit.** A single Databricks job holds at most 1,000 tasks; one-task-per-dbt-node can exceed that on large, test-heavy projects. After `make manifest`, run `make task-count` to compare unbundled vs bundled counts. When the unbundled count is over the warn threshold (900), set `BUNDLE_TESTS = True` in the glue: this collapses each resource's single-model tests into one bundled test task (`dbt test --select --indirect-selection cautious`, keyed `_test`) — the single biggest reduction — while cross-model and zero-dep tests still get their own tasks. The tradeoff is coarser retry granularity: a model's tests rerun together, not per individual test. The bundled task targets the resource with `--indirect-selection cautious` so it still sweeps in that resource's tests; selector-exactness skips test nodes in bundled mode since individual tests are not selected on their own. If the count exceeds 1,000 even bundled, do not auto-fall-back: record the options in MIGRATION_NOTES (split the project by dbt tag into multiple factory jobs, await a dbt-factory sub-job-splitting API, or a user-chosen single `dbt_task`). The glue fails closed above 1,000 tasks at deploy time so an over-limit job is caught at `bundle validate` rather than by the Jobs API. +**Task count and the 1,000-task per-job limit.** A single Databricks job holds at most 1,000 tasks; one-task-per-dbt-node can exceed that on large, test-heavy projects. After `make manifest`, run `make task-count` to compare unbundled vs bundled counts. When the unbundled count is over the warn threshold (900), set `BUNDLE_TESTS = True` in the glue: this collapses each resource's single-model tests into one bundled test task (one `dbt test` task repeating `--select` per test at `--indirect-selection empty`, keyed `_test`) — the single biggest reduction — while cross-model and zero-dep tests still get their own tasks. The tradeoff is coarser retry granularity: a model's tests rerun together, not per individual test. The bundled task names each of the resource's tests with its own `--select` term and pins `--indirect-selection empty`, so only those tests run. If the count exceeds 1,000 even bundled, do not auto-fall-back: record the options in MIGRATION_NOTES (split the project by dbt tag into multiple factory jobs, await a dbt-factory sub-job-splitting API, or a user-chosen single `dbt_task`). The glue fails closed above 1,000 tasks at deploy time so an over-limit job is caught at `bundle validate` rather than by the Jobs API. **Vars.** Static `vars` (literal dicts) live in ONE committed file: `dbt_vars.json` at the bundle root (required; `{}` when none). `make manifest` feeds it to `dbt parse --vars` and the runner falls back to it at run time whenever the `dbt_vars` job parameter is an empty object — so parse-time and run-time always agree, and no JSON is ever inlined into shell or Python quoting. A runtime override that differs from the file also bypasses the parse-cache injection (the cache was compiled with static vars — hooks, materializations, and grants would silently keep static values); dbt re-parses in-task instead, at some startup cost. A non-empty runtime `dbt_vars` REPLACES the whole dict (dbt does not merge repeated `--vars`), so overriding callers must pass the complete set. Never smuggle vars through `EXTRA_DBT_COMMAND_OPTIONS` (two `--vars` flags: dbt silently uses the last one). Runtime overrides are safe only when they do not change the dbt graph (enabled nodes, dependencies, schemas, aliases), because the task graph was compiled at deploy time. Disqualifiers: a var that changes the graph, or dbt operator tasks passing conflicting vars dicts (no single canonical value exists) — fall back to `dbt_task`. @@ -1309,7 +1309,7 @@ Factory mode adds these artifacts to the bundle (templates in `assets/templates/ | `pyproject.toml` | `dbt-pyproject.toml.tmpl` | Pins `databricks-bundles`, `databricks-dbt-factory`, and EXACT `dbt-databricks`/`dbt-core` (dbt version/runtime parity, since uv.lock is git-ignored; transitive deps not locked). Shared across DAGs. | | `Makefile` | `dbt-Makefile.tmpl` | `TARGET ?= dev`; `setup` (uv sync) / `manifest` (dbt deps + parse `--target $(TARGET)` `--target-path target/$(TARGET)`) / `validate` / `deploy`. Per-target manifest paths keep dev-parsed artifacts (profile-resolved catalog/schema are baked into the manifest at parse time) out of prod deployments. | | `dbt_profiles/profiles.yml` | `dbt-profiles.yml.tmpl` | dev/prod outputs named after bundle targets; host/token injected by the runner notebook. | -| `src/run_dbt_command.py` | `dbt-run-command.py.tmpl` | Runner notebook owned by the bundle: the 0.3.1 packaged runner extended with `dbt_vars` (appended as `--vars` argv, never string-interpolated; empty/`{}` falls back to `dbt_vars.json`) and per-target parse-cache lookup. Re-diff against the packaged runner when bumping the pin. | +| `src/run_dbt_command.py` | `dbt-run-command.py.tmpl` | Runner notebook owned by the bundle: the packaged runner extended with `dbt_vars` (appended as `--vars` argv, never string-interpolated; empty/`{}` falls back to `dbt_vars.json`) and per-target parse-cache lookup. Re-diff against the packaged runner when bumping the pin. | | `dbt_vars.json` | — (write `{}` or the DAG's static vars) | Single source of static dbt vars, committed at the bundle root; consumed by `make manifest` (parse time) and the runner (run time). REQUIRED — the runner fails if it is missing. | | dbt project at bundle root | — (copied) | `dbt_project.yml`, `models/`, `seeds/`, etc. **v1 constraint: exactly one dbt project per bundle, colocated at the bundle root.** Multiple dbt projects → split bundles. | | `.gitignore` additions | — | `.venv/`, `logs/`, `dbt_packages/`, `uv.lock`, `target/**`, `dbt_serverless_env.yaml`. `target/*/manifest.json` is a local hook input (not synced); `dbt_serverless_env.yaml` and `target/*/partial_parse.msgpack` are uploaded via `sync.include` despite being git-ignored. Exact `dbt-databricks`/`dbt-core` pins in `pyproject.toml` give dbt version/runtime parity (transitive deps unlocked). | diff --git a/src/flowx/models/ir.py b/src/flowx/models/ir.py index d6f45fc..dc284e9 100644 --- a/src/flowx/models/ir.py +++ b/src/flowx/models/ir.py @@ -501,7 +501,9 @@ class DbtFactoryActivity(Activity): - ``pydabs`` (opt-in): emit a PyDABs hook module that calls ``databricks-dbt-factory`` at ``bundle deploy`` time, so the dbt job tracks the project automatically (at the cost of being invisible to - static coverage until deploy). + static coverage until deploy). Workloads carrying selectors, exclusions, + or vars use the static renderer because the factory owns resource + selection and parse context. Attributes: project_dir: Path to the dbt project (relative to the bundle root). diff --git a/src/flowx/preparer/activity_preparers/dbt_factory.py b/src/flowx/preparer/activity_preparers/dbt_factory.py index eedccaf..d4051c9 100644 --- a/src/flowx/preparer/activity_preparers/dbt_factory.py +++ b/src/flowx/preparer/activity_preparers/dbt_factory.py @@ -32,6 +32,7 @@ _RUNNER_RELATIVE_PATH = "notebooks/run_dbt_command.py" _DBT_PROJECT_RELATIVE_PATH = "dbt_project" _DBT_PROFILES_RELATIVE_PATH = "dbt_profiles" +_DBT_FACTORY_VERSION = "0.3.3" _EXCLUDED_DBT_PATH_PARTS = {".git", ".venv", "__pycache__", "logs", "target"} @@ -70,6 +71,14 @@ def _dbt_source_artifacts(activity: DbtFactoryActivity) -> list[DabNotebook]: binary_content=manifest_path.read_bytes(), ) ) + partial_parse_path = manifest_path.parent / "partial_parse.msgpack" + if partial_parse_path.is_file(): + artifacts.append( + DabNotebook( + relative_path=f"{_DBT_PROJECT_RELATIVE_PATH}/target/partial_parse.msgpack", + binary_content=partial_parse_path.read_bytes(), + ) + ) return artifacts @@ -79,10 +88,10 @@ def _pydabs_pyproject_source() -> str: "[project]\n" 'name = "flowx-dbt-bundle"\n' 'version = "0.1.0"\n' - 'requires-python = ">=3.10"\n' + 'requires-python = ">=3.10,<3.13"\n' "dependencies = [\n" ' "databricks-bundles>=1.0.0,<2.0.0",\n' - ' "databricks-dbt-factory==0.3.1",\n' + f' "databricks-dbt-factory=={_DBT_FACTORY_VERSION}",\n' ' "dbt-databricks==1.12.2",\n' ' "dbt-core==1.11.12",\n' "]\n" @@ -90,13 +99,8 @@ def _pydabs_pyproject_source() -> str: def _pydabs_options_by_resource_type(activity: DbtFactoryActivity) -> dict[str, str]: - """Returns shell-safe dbt options for each generated task-factory type.""" + """Returns factory-compatible dbt options for each generated task-factory type.""" common = ["--target", activity.target] - for selector in activity.exclude_selectors: - common.extend(("--exclude", selector)) - if activity.variables is not None: - variables = json.dumps(activity.variables) if isinstance(activity.variables, dict) else activity.variables - common.extend(("--vars", variables)) options: dict[str, str] = {} for resource_type in activity.resource_types or ["model", "seed", "snapshot", "test"]: tokens = [*common] @@ -207,17 +211,25 @@ def _pydabs_runner_notebook_source() -> str: "# Databricks notebook source\n\n" "import json\n" "import os\n" - "import shlex\n\n" + "import shlex\n" + "import shutil\n" + "import tempfile\n" + "from urllib.parse import urlparse\n\n" "from dbt.cli.main import dbtRunner\n\n" "dbutils.widgets.text('dbt_commands', '')\n" "dbutils.widgets.text('project_directory', '')\n" "dbutils.widgets.text('profiles_directory', '')\n\n" - "commands = json.loads(dbutils.widgets.get('dbt_commands'))\n" + "dbt_commands = dbutils.widgets.get('dbt_commands')\n" "project_directory = dbutils.widgets.get('project_directory')\n" - "profiles_directory = dbutils.widgets.get('profiles_directory')\n" + "profiles_directory = dbutils.widgets.get('profiles_directory')\n\n" + "if not dbt_commands:\n" + " raise ValueError('dbt_commands parameter is required')\n" + "commands = json.loads(dbt_commands)\n\n" "context = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n" "os.environ['DBT_ACCESS_TOKEN'] = context.apiToken().get()\n" - "os.environ['DBT_HOST'] = context.apiUrl().get()\n\n" + "api_url = context.apiUrl().get()\n" + "parsed_url = urlparse(api_url)\n" + "os.environ['DBT_HOST'] = parsed_url.netloc or parsed_url.path.strip('/')\n\n" "if project_directory:\n" " notebook_dir = os.path.dirname('/Workspace' + context.notebookPath().get())\n" " project_path = (\n" @@ -226,18 +238,40 @@ def _pydabs_runner_notebook_source() -> str: " else os.path.normpath(os.path.join(notebook_dir, project_directory))\n" " )\n" " os.chdir(project_path)\n\n" - "runner = dbtRunner()\n" - "for command in commands:\n" - " command = command.strip()\n" - " if command.startswith('dbt '):\n" - " command = command[4:]\n" - " arguments = shlex.split(command)\n" - " if profiles_directory:\n" - " arguments.extend(['--profiles-dir', profiles_directory])\n" - " result = runner.invoke(arguments)\n" - " if not result.success:\n" - " detail = result.exception or result.result or '(no further details)'\n" - " raise RuntimeError(f\"dbt command failed: dbt {' '.join(arguments)}\\n{detail}\")\n" + "local_dir = tempfile.mkdtemp(prefix='dbt_local_')\n" + "os.environ['DBT_TARGET_PATH'] = local_dir\n" + "os.environ['DBT_LOG_PATH'] = local_dir\n\n" + "manifest = None\n" + "prebuilt_manifest_path = os.path.join('target', 'partial_parse.msgpack')\n" + "if os.path.exists(prebuilt_manifest_path):\n" + " try:\n" + " from dbt.contracts.graph.manifest import Manifest\n\n" + " with open(prebuilt_manifest_path, 'rb') as manifest_file:\n" + " manifest = Manifest.from_msgpack(manifest_file.read())\n" + " manifest.build_flat_graph()\n" + " print(f'[dbt-factory] using pre-built manifest from {prebuilt_manifest_path}')\n" + " except Exception as error:\n" + " print(f'[dbt-factory] pre-built manifest unavailable; dbt will parse the project: {error}')\n" + " manifest = None\n\n" + "try:\n" + " runner = dbtRunner(manifest=manifest)\n" + " for command in commands:\n" + " command = command.strip()\n" + " if command.startswith('dbt '):\n" + " command = command[4:]\n" + " arguments = shlex.split(command)\n" + " if profiles_directory:\n" + " arguments.extend(['--profiles-dir', profiles_directory])\n" + " result = runner.invoke(arguments)\n" + " if not result.success:\n" + " detail = result.exception or result.result or '(no further details)'\n" + " raise RuntimeError(f\"dbt command failed: dbt {' '.join(arguments)}\\n{detail}\")\n" + "finally:\n" + " os.environ.pop('DBT_ACCESS_TOKEN', None)\n" + " os.environ.pop('DBT_HOST', None)\n" + " os.environ.pop('DBT_TARGET_PATH', None)\n" + " os.environ.pop('DBT_LOG_PATH', None)\n" + " shutil.rmtree(local_dir, ignore_errors=True)\n" ) @@ -438,7 +472,7 @@ def prepare(activity: DbtFactoryActivity, *, scope: str = "") -> PreparedActivit if missing_inputs: return _prepare_missing_inputs(activity, missing_inputs) if activity.render_mode == "pydabs": - if activity.selectors: + if activity.selectors or activity.exclude_selectors or activity.variables is not None: return _prepare_static(activity, _nodes_from_activity(activity)) return _prepare_pydabs(activity) nodes = _nodes_from_activity(activity) diff --git a/src/flowx/sources/airflow/convert.py b/src/flowx/sources/airflow/convert.py index 71455ad..0811d8e 100644 --- a/src/flowx/sources/airflow/convert.py +++ b/src/flowx/sources/airflow/convert.py @@ -39,7 +39,8 @@ def main(argv: list[str] | None = None) -> int: choices=("static", "pydabs"), default="static", help="dbt-factory render mode: 'static' (inner job of per-node tasks, default) or 'pydabs' " - "(a deploy-time PyDABs hook that builds the dbt job from the live manifest).", + "(a deploy-time PyDABs hook that builds the dbt job from the live manifest; source selectors, exclusions, " + "or vars use the static renderer).", ) parser.add_argument( "--merge-agentic", diff --git a/tests/unit/test_dbt_factory_preparer.py b/tests/unit/test_dbt_factory_preparer.py index f10dbaf..5ad32b6 100644 --- a/tests/unit/test_dbt_factory_preparer.py +++ b/tests/unit/test_dbt_factory_preparer.py @@ -4,6 +4,8 @@ import json +import pytest + from flowx.models.ir import DbtFactoryActivity, Dependency, NotebookActivity, Pipeline from flowx.preparer.workflow_preparer import prepare_activity, prepare_workflow @@ -42,14 +44,15 @@ def _pydabs_activity(tmp_path, **overrides): (project / "dbt_project.yml").write_text("name: demo\nprofile: demo\n") (project / "target" / "manifest.json").write_text(json.dumps({"nodes": {}})) (profiles / "profiles.yml").write_text("demo:\n target: dev\n outputs: {}\n") - return _dbt_activity( + kwargs = dict( nodes=[], render_mode="pydabs", project_dir=str(project), profiles_dir=str(profiles), manifest_path=str(project / "target" / "manifest.json"), - **overrides, ) + kwargs.update(overrides) + return _dbt_activity(**kwargs) def test_static_parent_task_is_run_job_hop(): @@ -173,16 +176,55 @@ def test_pydabs_emits_hook_module_and_no_inner_job(tmp_path): assert "load_resources" in hook.content assert "from databricks_dbt_factory.Utils import read_dbt_manifest" in hook.content assert "DbtFactory(task_factories" in hook.content - # 0.3.1 dropped SpecsHandler; the manifest reader is a module-level Utils function. + # The supported factory API exposes the manifest reader as a module-level Utils function. assert "SpecsHandler" not in hook.content assert "read_dbt_manifest(MANIFEST_PATH)" in hook.content runner = next(nb for nb in prepared.notebooks if nb.relative_path == "notebooks/run_dbt_command.py") assert "dbt_commands" in runner.content + assert "dbt_commands parameter is required" in runner.content assert "project_directory" in runner.content assert "profiles_directory" in runner.content + assert "urlparse" in runner.content + assert "DBT_TARGET_PATH" in runner.content + assert "partial_parse.msgpack" in runner.content + assert "shutil.rmtree" in runner.content + compile(runner.content, runner.relative_path, "exec") assert "run_job_task" in prepared.task +@pytest.mark.parametrize( + "reserved_options", + [ + {"selectors": ["tag:daily"]}, + {"exclude_selectors": ["tag:slow"]}, + {"variables": {"region": "west"}}, + ], +) +def test_pydabs_reserved_factory_options_fall_back_to_static(tmp_path, reserved_options): + prepared = prepare_activity(_pydabs_activity(tmp_path, nodes=_NODES, **reserved_options)) + + assert len(prepared.inner_workflows) == 1 + assert {task["task_key"] for task in prepared.inner_workflows[0].tasks} == { + "seed_codes", + "model_stg", + "model_fct", + "test_stg", + } + assert not any(task.type == "pydabs_dbt_factory" for task in prepared.setup_tasks) + + +def test_pydabs_keeps_supported_target_and_full_refresh_options(tmp_path): + prepared = prepare_activity( + _pydabs_activity(tmp_path, target="prod", full_refresh=True, resource_types=["model", "test"]) + ) + + hook = next(notebook for notebook in prepared.notebooks if notebook.relative_path.endswith("_dbt_job.py")) + assert "'model': '--target prod --full-refresh'" in hook.content + assert "'test': '--target prod'" in hook.content + assert "--exclude" not in hook.content + assert "--vars" not in hook.content + + def test_pydabs_records_setup_task(tmp_path): prepared = prepare_activity(_pydabs_activity(tmp_path)) setup_types = {t.type for t in prepared.setup_tasks} @@ -272,7 +314,8 @@ def test_pydabs_bundle_wires_python_resources_and_setup(tmp_path): assert (tmp_path / "resources" / "__init__.py").exists() assert not (tmp_path / "src" / "resources").exists() pyproject = (tmp_path / "pyproject.toml").read_text() - assert "databricks-dbt-factory==0.3.1" in pyproject + assert 'requires-python = ">=3.10,<3.13"' in pyproject + assert "databricks-dbt-factory==0.3.3" in pyproject assert "dbt-databricks==1.12.2" in pyproject setup = (tmp_path / "SETUP.md").read_text() assert "dbt factory (PyDABs mode)" in setup @@ -289,6 +332,7 @@ def test_pydabs_copies_available_dbt_project_into_bundle(tmp_path): (project / "models" / "orders.sql").write_text("select 1\n") (project / "target").mkdir() (project / "target" / "manifest.json").write_text(json.dumps({"nodes": {}})) + (project / "target" / "partial_parse.msgpack").write_bytes(b"prebuilt-dbt-graph") profiles = tmp_path / "profiles" profiles.mkdir() (profiles / "profiles.yml").write_text("demo:\n target: dev\n outputs: {}\n") @@ -309,3 +353,4 @@ def test_pydabs_copies_available_dbt_project_into_bundle(tmp_path): assert (output / "src" / "dbt_project" / "dbt_project.yml").exists() assert (output / "src" / "dbt_project" / "models" / "orders.sql").exists() + assert (output / "src" / "dbt_project" / "target" / "partial_parse.msgpack").read_bytes() == b"prebuilt-dbt-graph" From e3b7e8cad59d693b5c422c65a0f107df37111f63 Mon Sep 17 00:00:00 2001 From: Lorenzo Rubio Date: Thu, 13 Aug 2026 18:39:14 +0200 Subject: [PATCH 71/77] emit a fresh depends_on dict per branch root (no shared YAML anchor) --- .../activity_preparers/if_condition.py | 7 +- tests/unit/test_bundler.py | 82 +++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/src/flowx/preparer/activity_preparers/if_condition.py b/src/flowx/preparer/activity_preparers/if_condition.py index eb96072..c50b444 100644 --- a/src/flowx/preparer/activity_preparers/if_condition.py +++ b/src/flowx/preparer/activity_preparers/if_condition.py @@ -41,7 +41,6 @@ def inject_outcome_dependency(tasks: list[dict[str, Any]], condition_key: str, o outcome: ``"true"`` or ``"false"``. """ branch_keys = {task.get("task_key") for task in tasks} - outcome_dep = {"task_key": condition_key, "outcome": outcome} for task in tasks: deps = list(task.get("depends_on") or []) refers_to_branch_sibling = any(dep.get("task_key") in branch_keys for dep in deps) @@ -49,7 +48,11 @@ def inject_outcome_dependency(tasks: list[dict[str, Any]], condition_key: str, o continue if any(dep.get("task_key") == condition_key and dep.get("outcome") == outcome for dep in deps): continue - task["depends_on"] = [outcome_dep, *deps] + # Build a fresh dict per task -- never share one object across branch roots. A shared dict is + # serialised by PyYAML as a YAML anchor/alias (&id/*id); generated bundles should stay + # anchor-free, since a strict package pre-flight can reject anchors as invariant violations + # (issue #34). + task["depends_on"] = [{"task_key": condition_key, "outcome": outcome}, *deps] def prepare(activity: IfConditionActivity, *, scope: str = "") -> PreparedActivity: diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index 90ecb02..d3a4797 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -8,8 +8,11 @@ from flowx.models.dab import SecretInstruction, SetupTask from flowx.models.ir import ( CopyActivity, + IfConditionActivity, NotebookActivity, Pipeline, + SwitchActivity, + SwitchCase, WaitActivity, ) from flowx.preparer.workflow_preparer import PreparedWorkflow, prepare_workflow @@ -92,6 +95,85 @@ def test_databricks_yml_structure(self, tmp_path): assert "dev" in content["targets"] assert "prod" in content["targets"] + def test_condition_fanout_emits_no_yaml_anchor(self, tmp_path): + """Issue #34: an IfCondition/Switch branch that fans out to >=2 root tasks must not leak a + YAML anchor/alias into the emitted bundle. ``inject_outcome_dependency`` must build a fresh + ``depends_on`` dict per branch root; sharing one object makes PyYAML serialise it as + ``&id001``/``*id001`` -- benign, valid YAML, but PR #13's package pre-flight rejects + ``yaml_anchor`` as a fatal violation and aborts the whole batch (0 bundles).""" + pipeline = Pipeline( + name="condition_fanout", + tasks=[ + IfConditionActivity( + name="If_Condition1", + task_key="if_condition1", + op="EQUAL_TO", + left="@pipeline().x", + right="1", + if_true_activities=[ + WaitActivity(name="TrueWaitA", task_key="true_wait_a", wait_time_seconds=1), + WaitActivity(name="TrueWaitB", task_key="true_wait_b", wait_time_seconds=2), + ], + if_false_activities=[], + ), + ], + ) + write_bundle(prepare_workflow(pipeline), tmp_path) + resource_files = list((tmp_path / "resources").glob("*.yml")) + combined = "\n".join(path.read_text() for path in resource_files) + assert "&id" not in combined and "*id" not in combined, f"YAML anchor leaked:\n{combined}" + # The fix must preserve semantics: both branch roots still gate on the condition's "true" outcome. + gated_on_true = 0 + for path in resource_files: + doc = yaml.safe_load(path.read_text()) or {} + jobs = (doc.get("resources") or {}).get("jobs") or {} + for job in jobs.values(): + for task in job.get("tasks") or []: + for dep in task.get("depends_on") or []: + if dep.get("task_key") == "if_condition1" and dep.get("outcome") == "true": + gated_on_true += 1 + assert gated_on_true == 2, f"both branch roots must gate on the true outcome, got {gated_on_true}" + + def test_switch_fanout_emits_no_yaml_anchor(self, tmp_path): + """Issue #34 (Switch path): Switch routes its branch gating through the same + ``inject_outcome_dependency`` helper as IfCondition, so a Switch case that fans out to >=2 + tasks must likewise emit no YAML anchor/alias into the bundle.""" + pipeline = Pipeline( + name="switch_fanout", + tasks=[ + SwitchActivity( + name="Switch1", + task_key="switch1", + on_expression="@pipeline().sel", + cases=[ + SwitchCase( + value="a", + activities=[ + WaitActivity(name="CaseA1", task_key="case_a1", wait_time_seconds=1), + WaitActivity(name="CaseA2", task_key="case_a2", wait_time_seconds=2), + ], + ) + ], + default_activities=[], + ), + ], + ) + write_bundle(prepare_workflow(pipeline), tmp_path) + resource_files = list((tmp_path / "resources").glob("*.yml")) + combined = "\n".join(path.read_text() for path in resource_files) + assert "&id" not in combined and "*id" not in combined, f"YAML anchor leaked:\n{combined}" + # The fix must preserve semantics: both case roots still gate on the case condition's "true" outcome. + gated_on_true = 0 + for path in resource_files: + doc = yaml.safe_load(path.read_text()) or {} + jobs = (doc.get("resources") or {}).get("jobs") or {} + for job in jobs.values(): + for task in job.get("tasks") or []: + for dep in task.get("depends_on") or []: + if dep.get("task_key") == "switch1_case_a" and dep.get("outcome") == "true": + gated_on_true += 1 + assert gated_on_true == 2, f"both case roots must gate on the case 'true' outcome, got {gated_on_true}" + def test_job_resource_yml_exists(self, tmp_path): """A job resource YAML is created under resources/.""" wf = _simple_workflow("my_job") From 45ed87492ea8a68640221a3be2862afeb7f6e560 Mon Sep 17 00:00:00 2001 From: Lorenzo Rubio Date: Sun, 16 Aug 2026 18:32:01 +0200 Subject: [PATCH 72/77] fix package failing on multi-pipeline translation reports (#6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes Add a `"pipelines"` branch to `_load_report`, mirroring the adapter's `_load_pipelines`: filter each entry with `isinstance(p, dict) and "tasks" in p and "name" in p` and route it through the existing `_pipeline_dict_to_workflow` (the same machinery the single-pipeline branch uses). Strictly additive — the single-pipeline and `translations` branches are untouched, so existing reports behave exactly as before. ### Linked issues Resolves #5 ### Tests Reproduced the abort with a multi-pipeline `{"pipelines":[...]}` report through the real `package` CLI; after the fix, `package` writes one bundle per pipeline (exit 0). Added `test_load_report_handles_pipelines_format` (`tests/unit/test_bundler.py`): a 2-pipeline report yields 2 workflows. - [x] manually tested - [x] added unit tests - [ ] added integration tests --------- Co-authored-by: Greg Hansen --- src/flowx/bundler/dab_writer.py | 67 ++++++- src/flowx/bundler/prereqs_writer.py | 23 +++ tests/unit/test_bundler.py | 282 +++++++++++++++++++++++++--- tests/unit/test_preparers.py | 4 +- tests/unit/test_prereqs_writer.py | 29 +++ 5 files changed, 375 insertions(+), 30 deletions(-) diff --git a/src/flowx/bundler/dab_writer.py b/src/flowx/bundler/dab_writer.py index 362d48a..ab16bf3 100644 --- a/src/flowx/bundler/dab_writer.py +++ b/src/flowx/bundler/dab_writer.py @@ -84,6 +84,7 @@ def write_bundle( catalog: str = "main", schema: str = "default", bundle_name: str | None = None, + skipped_pipelines: list[str] | None = None, ) -> list[Path]: """Writes all DAB files to output_dir. @@ -93,6 +94,8 @@ def write_bundle( catalog: Default target catalog name. schema: Default target schema name. bundle_name: Optional bundle name (defaults to workflow name). + skipped_pipelines: Report-level entries _load_report could not package + (surfaced in SETUP.md so a dropped pipeline is documented, not silent). Returns: List of absolute paths to all created files. @@ -289,6 +292,7 @@ def write_bundle( manual_schedule_time_of_day=manual_schedule_time_of_day_configs, manual_credentials=manual_credential_configs, neutralized_conditions=list(_neutralized_conditions), + skipped_pipelines=list(skipped_pipelines or []), ) setup_path = output_dir / "SETUP.md" setup_path.write_text(render_setup_md(prereqs, bundle_name=effective_name), encoding="utf-8") @@ -328,7 +332,9 @@ def main(argv: list[str] | None = None) -> int: """Package-phase entry point for DAB bundle generation. Returns a process exit code so the adapter can run this phase in-process (instead of spawning a - second interpreter) and still propagate failures. + second interpreter) and still propagate failures: ``0`` on success, ``1`` when the report has no + translated pipelines, ``2`` when workspace-file auth is required but unavailable, and ``3`` when + every entry in the report was malformed (nothing left to package). """ parser = argparse.ArgumentParser( description="Generate a Databricks Declarative Automation Bundle from a translation report.", @@ -413,9 +419,23 @@ def main(argv: list[str] | None = None) -> int: enable_workspace_downloads(True) print(f"Loading translation report: {args.report}") - workflows = _load_report(args.report) + workflows, skipped_pipelines = _load_report(args.report) + if skipped_pipelines: + print( + f"Warning: skipped {len(skipped_pipelines)} malformed pipeline " + f"entr{'y' if len(skipped_pipelines) == 1 else 'ies'} in the report " + f"(see SETUP.md 'Skipped pipelines'): {', '.join(skipped_pipelines)}", + file=sys.stderr, + ) if not workflows: + # Distinguish "every entry was malformed" (something to fix) from a genuinely empty report. + if skipped_pipelines: + print( + "No valid pipelines to package: every entry in the report was malformed.", + file=sys.stderr, + ) + return 3 print("No translated pipelines found in the report.", file=sys.stderr) return 1 @@ -433,6 +453,7 @@ def main(argv: list[str] | None = None) -> int: catalog=args.catalog, schema=args.schema, bundle_name=effective_bundle_name, + skipped_pipelines=skipped_pipelines, ) all_created.extend(created) print(f" [{index + 1}/{len(workflows)}] {workflow.name}: {len(created)} files") @@ -1350,24 +1371,28 @@ def _normalize_base_parameters( return resolved -def _load_report(report_path: Path) -> list[PreparedWorkflow]: +def _load_report(report_path: Path) -> tuple[list[PreparedWorkflow], list[str]]: """Loads a translation report and reconstruct PreparedWorkflow objects. Args: report_path: Path to the translation report JSON file. Returns: - List of PreparedWorkflow objects, one per pipeline. + A ``(workflows, skipped)`` tuple: one PreparedWorkflow per pipeline, plus the + labels of any ``{"pipelines": [...]}`` entries that were skipped (not raised) + because they were malformed. The caller surfaces ``skipped`` in SETUP.md so a + dropped pipeline is documented rather than silently missing. """ with open(report_path, encoding="utf-8") as report_file: report = json.load(report_file) workflows: list[PreparedWorkflow] = [] + skipped: list[str] = [] if "tasks" in report and "name" in report: workflow = _pipeline_dict_to_workflow(report) workflows.append(workflow) - return workflows + return workflows, skipped if "translations" in report: # Aggregated translation_report.json: ``translations`` is a flat list of {pipeline, ir, status}. @@ -1403,10 +1428,38 @@ def _load_report(report_path: Path) -> list[PreparedWorkflow]: pipeline_dict["schedule"] = pipeline_schedules[pipeline_name] workflow = _pipeline_dict_to_workflow(pipeline_dict) workflows.append(workflow) - return workflows + return workflows, skipped + + if "pipelines" in report and isinstance(report["pipelines"], list): + # Aggregated report written by engine.py / modify ({"pipelines": [...]}): one dict per + # pipeline, each already in the single-pipeline {"name", "tasks", ...} IR shape. Route each + # through the same machinery the single-pipeline branch uses. Mirrors the adapter's + # _load_pipelines (adapter/__main__.py) so both report consumers agree on this shape. + for index, pipeline_dict in enumerate(report["pipelines"]): + if isinstance(pipeline_dict, dict) and "tasks" in pipeline_dict and "name" in pipeline_dict: + workflows.append(_pipeline_dict_to_workflow(pipeline_dict)) + else: + # PR #6: a non-conforming entry (corruption / an internal bug) is skipped so the other + # valid pipelines still convert. Record each offender rather than dropping it silently — + # the caller surfaces the returned skip list in every bundle's SETUP.md. + name = pipeline_dict.get("name") if isinstance(pipeline_dict, dict) else None + if name: + # Store the bare name; renderers quote/backtick it for their medium + # (SETUP.md wraps it in backticks). Avoids leaking Python repr quotes. + skipped.append(str(name)) + else: + # Enrich the label with hints about what went wrong so users can debug. + if not isinstance(pipeline_dict, dict): + hint = "not a JSON object" + elif "tasks" in pipeline_dict: + hint = "has tasks, missing name" + else: + hint = "missing name/tasks" + skipped.append(f"index {index} ({hint})") + return workflows, skipped # Empty or unrecognised report shape — nothing to do. - return workflows + return workflows, skipped def _pipeline_dict_to_workflow(pipeline_dict: dict[str, Any]) -> PreparedWorkflow: diff --git a/src/flowx/bundler/prereqs_writer.py b/src/flowx/bundler/prereqs_writer.py index 0149507..87954a9 100644 --- a/src/flowx/bundler/prereqs_writer.py +++ b/src/flowx/bundler/prereqs_writer.py @@ -131,6 +131,10 @@ class Prereqs: # C-43 (CF5-001 / CF5-002): condition_task operands blanked because they referenced a task in another # job ({task_key, field, original_ref}); a blanked operand is always-true, so the user must re-wire it. neutralized_conditions: list[dict[str, str]] = field(default_factory=list) + # PR #6: report entries dropped by _load_report because they were not a dict with 'name' and 'tasks'. + # Each entry is a human-readable identifier (the pipeline name, or ``index N`` when unnamed) so a + # skipped pipeline is surfaced rather than silently missing from the bundle. + skipped_pipelines: list[str] = field(default_factory=list) def is_empty(self) -> bool: """Return ``True`` when nothing needs to happen before ``bundle run``.""" @@ -150,6 +154,7 @@ def is_empty(self) -> bool: and not self.manual_schedule_time_of_day and not self.manual_credentials and not self.neutralized_conditions + and not self.skipped_pipelines ) @@ -361,6 +366,7 @@ def build_prereqs( manual_schedule_time_of_day: list[dict[str, Any]] | None = None, manual_credentials: list[dict[str, Any]] | None = None, neutralized_conditions: list[dict[str, str]] | None = None, + skipped_pipelines: list[str] | None = None, ) -> Prereqs: """Assemble a :class:`Prereqs` from the bundle's generated artifacts. @@ -408,6 +414,7 @@ def build_prereqs( manual_schedule_time_of_day=list(manual_schedule_time_of_day or []), manual_credentials=list(manual_credentials or []), neutralized_conditions=list(neutralized_conditions or []), + skipped_pipelines=list(skipped_pipelines or []), ) @@ -760,4 +767,20 @@ def render_setup_md(prereqs: Prereqs, *, bundle_name: str) -> str: lines.append(f"| {label} | `{endpoint.target}` | {endpoint.notes} |") lines.append("") + if prereqs.skipped_pipelines: + lines.append("## Skipped pipelines") + lines.append("") + lines.append( + "The translation report contained the following entries that flowx could not turn " + "into a bundle (each was not a pipeline object with both `name` and `tasks`). They were " + "skipped so the valid pipelines could still be generated. This usually signals a " + "corrupt or truncated report — re-run `convert` for these pipelines and package again." + ) + lines.append("") + for skipped in prereqs.skipped_pipelines: + # Backtick-wrap so pipeline names read cleanly and stay unambiguous in + # Markdown even when they contain spaces or Markdown-special characters. + lines.append(f"- `{skipped}`") + lines.append("") + return "\n".join(lines) diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index 90ecb02..26e9932 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -2,9 +2,15 @@ from __future__ import annotations +import json + import yaml -from flowx.bundler.dab_writer import write_bundle +from flowx.bundler.dab_writer import ( + _load_report, + write_bundle, +) +from flowx.bundler.dab_writer import main as dab_main from flowx.models.dab import SecretInstruction, SetupTask from flowx.models.ir import ( CopyActivity, @@ -220,10 +226,6 @@ def test_load_report_handles_aggregated_translations_format(self, tmp_path): documented ``translation_report.json`` aggregated format would have hit ``NameError`` the first time a notebook task was emitted. """ - import json - - from flowx.bundler.dab_writer import _load_report - report = { "translations": [ { @@ -256,12 +258,262 @@ def test_load_report_handles_aggregated_translations_format(self, tmp_path): report_path = tmp_path / "translation_report.json" report_path.write_text(json.dumps(report)) - workflows = _load_report(report_path) + workflows, _ = _load_report(report_path) assert len(workflows) == 1 assert workflows[0].name == "agg_pipeline" task_keys = {task["task_key"] for task in workflows[0].tasks} assert task_keys == {"pause", "run_nb"} + def test_load_report_handles_pipelines_format(self, tmp_path): + """``_load_report`` accepts the ``{"pipelines": [...]}`` aggregated report. + + Regression: ``convert``/``modify`` serialize multi-pipeline reports under a + top-level ``"pipelines"`` key (engine.py: ``{"pipelines": all_pipeline_dicts}``), + but ``_load_report`` only understood the single-pipeline and legacy + ``"translations"`` shapes and silently returned ``[]`` for this one -- so + ``package`` aborted with "No translated pipelines found" for any factory with + more than one pipeline. + """ + report = { + "pipelines": [ + { + "name": "pipeline_a", + "tasks": [ + { + "type": "WaitActivity", + "name": "Pause", + "task_key": "pause", + "wait_time_seconds": 5, + }, + ], + }, + { + "name": "pipeline_b", + "tasks": [ + { + "type": "NotebookActivity", + "name": "Run NB", + "task_key": "run_nb", + "notebook_path": "/Shared/etl/run", + }, + ], + }, + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + + workflows, _ = _load_report(report_path) + assert len(workflows) == 2 + assert {wf.name for wf in workflows} == {"pipeline_a", "pipeline_b"} + by_name = {wf.name: wf for wf in workflows} + assert {task["task_key"] for task in by_name["pipeline_a"].tasks} == {"pause"} + assert {task["task_key"] for task in by_name["pipeline_b"].tasks} == {"run_nb"} + + def test_load_report_skips_malformed_pipelines_entry(self, tmp_path): + """A malformed ``"pipelines"`` entry is skipped so valid pipelines still convert. + + PR #6 review: aborting the whole run because one entry is malformed drops + every other valid pipeline in the report. Instead, ``_load_report`` keeps + the well-formed pipelines and records each skipped entry (surfaced in + SETUP.md downstream) rather than raising. + """ + report = { + "pipelines": [ + { + "name": "pipeline_ok", + "tasks": [ + { + "type": "WaitActivity", + "name": "Pause", + "task_key": "pause", + "wait_time_seconds": 5, + }, + ], + }, + {"name": "no_tasks_here"}, # missing "tasks" + {"tasks": []}, # missing "name" + "not-even-a-dict", # not a dict at all + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + + workflows, skipped_pipelines = _load_report(report_path) + + # The one valid pipeline still produces a workflow. + assert [wf.name for wf in workflows] == ["pipeline_ok"] + + # Every offender is recorded so SETUP.md can name it (collect-all, not fail-fast). + skipped = " ".join(skipped_pipelines) + assert "no_tasks_here" in skipped + assert "index 2" in skipped + assert "index 3" in skipped + + # Named entries are stored bare — no Python repr quotes leak into the label. + assert "no_tasks_here" in skipped_pipelines + assert "'no_tasks_here'" not in skipped + + def test_load_report_enriches_skip_labels_with_hints(self, tmp_path): + """Skipped pipeline entries are labeled with hints about what went wrong. + + PR #6 polish: when a pipeline entry is missing a name, recording just + ``"index N"`` tells the user nothing about what the entry contained. + Enrich the label to hint at what field is missing or wrong: + - If it has tasks but no name: ``index N (has tasks, missing name)`` + - If it has neither: ``index N (missing name/tasks)`` + - If it's not a dict: ``index N (not a JSON object)`` + - If it has a name: record the bare name (no Python ``repr`` quotes); + renderers wrap it for their medium (SETUP.md uses backticks). + """ + report = { + "pipelines": [ + { + "name": "pipeline_ok", + "tasks": [ + { + "type": "WaitActivity", + "name": "Pause", + "task_key": "pause", + "wait_time_seconds": 5, + }, + ], + }, + # Case 1: missing name but has tasks + {"tasks": [{"type": "WaitActivity", "name": "P", "task_key": "p", "wait_time_seconds": 1}]}, + # Case 2: missing both name and tasks + {"other_field": "value"}, + # Case 3: not a dict at all + "not-even-a-dict", + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + + workflows, skipped_pipelines = _load_report(report_path) + + # One valid pipeline. + assert [wf.name for wf in workflows] == ["pipeline_ok"] + + # Check enriched skip labels. + assert len(skipped_pipelines) == 3 + # Index 1: has tasks, missing name + assert "index 1" in skipped_pipelines[0] + assert "has tasks" in skipped_pipelines[0] + assert "missing name" in skipped_pipelines[0] + # Index 2: missing both name and tasks + assert "index 2" in skipped_pipelines[1] + assert "missing name/tasks" in skipped_pipelines[1] + # Index 3: not a dict + assert "index 3" in skipped_pipelines[2] + assert "not a JSON object" in skipped_pipelines[2] + + def test_package_main_writes_skipped_section_and_continues(self, tmp_path): + """``package`` skips a malformed entry, still writes valid bundles, and notes the skip.""" + report = { + "pipelines": [ + { + "name": "pipeline_ok", + "tasks": [ + { + "type": "WaitActivity", + "name": "Pause", + "task_key": "pause", + "wait_time_seconds": 5, + }, + ], + }, + {"name": "no_tasks_here"}, + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + out_dir = tmp_path / "out" + + exit_code = dab_main( + [ + "--report", + str(report_path), + "--output-dir", + str(out_dir), + "--no-download-workspace-files", + ] + ) + + # Valid pipeline was written, so package succeeds. + assert exit_code == 0 + setup_md = (out_dir / "SETUP.md").read_text(encoding="utf-8") + assert "Skipped pipelines" in setup_md + assert "no_tasks_here" in setup_md + + def test_package_main_repeats_skipped_section_in_every_bundle(self, tmp_path): + """Each bundle in a multi-pipeline run carries the skip note. + + Bundles are consumed independently (one per pipeline directory), so the + dropped-pipeline warning is intentionally repeated in every bundle's + SETUP.md rather than written to a single shared location. + """ + report = { + "pipelines": [ + { + "name": "pipeline_a", + "tasks": [ + {"type": "WaitActivity", "name": "Pause", "task_key": "pause", "wait_time_seconds": 5}, + ], + }, + { + "name": "pipeline_b", + "tasks": [ + {"type": "WaitActivity", "name": "Hold", "task_key": "hold", "wait_time_seconds": 5}, + ], + }, + {"name": "no_tasks_here"}, # malformed -> skipped + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + out_dir = tmp_path / "out" + + exit_code = dab_main( + [ + "--report", + str(report_path), + "--output-dir", + str(out_dir), + "--no-download-workspace-files", + ] + ) + + assert exit_code == 0 + # Two valid pipelines -> one bundle subdirectory each, both naming the skipped entry. + for pipeline_name in ("pipeline_a", "pipeline_b"): + setup_md = (out_dir / pipeline_name / "SETUP.md").read_text(encoding="utf-8") + assert "Skipped pipelines" in setup_md + assert "no_tasks_here" in setup_md + + def test_package_main_returns_nonzero_when_all_entries_skipped(self, tmp_path): + """``package`` fails when every pipeline entry is malformed (nothing to write).""" + report = { + "pipelines": [ + {"name": "no_tasks_here"}, + "not-even-a-dict", + ] + } + report_path = tmp_path / "translation_report.json" + report_path.write_text(json.dumps(report)) + + exit_code = dab_main( + [ + "--report", + str(report_path), + "--output-dir", + str(tmp_path / "out"), + "--no-download-workspace-files", + ] + ) + + assert exit_code != 0 + class TestScheduleEmission: """C-10 (SCHED-001): schedule spec on PreparedWorkflow lands in job YAML.""" @@ -470,10 +722,6 @@ class TestAggregatedReportPipelineParameters: """Change pipeline-parameters-and-variables-round-trip (P0): VAR-001.""" def test_load_report_carries_pipeline_parameters(self, tmp_path): - import json - - from flowx.bundler.dab_writer import _load_report - report = { "translations": [ { @@ -492,7 +740,7 @@ def test_load_report_carries_pipeline_parameters(self, tmp_path): report_path = tmp_path / "translation_report.json" report_path.write_text(json.dumps(report)) - workflows = _load_report(report_path) + workflows, _ = _load_report(report_path) assert len(workflows) == 1 wf = workflows[0] # Pipeline-level parameters must survive round-trip. @@ -505,10 +753,6 @@ class TestAggregatedReportSchedule: """Change fix-aggregated-report-propagates-schedule (P0): SCHED3-001.""" def test_load_report_carries_pipeline_schedule(self, tmp_path): - import json - - from flowx.bundler.dab_writer import _load_report - schedule_spec = { "kind": "cron", "quartz_cron_expression": "0 0 2 ? * * *", @@ -533,7 +777,7 @@ def test_load_report_carries_pipeline_schedule(self, tmp_path): report_path = tmp_path / "translation_report.json" report_path.write_text(json.dumps(report)) - workflows = _load_report(report_path) + workflows, _ = _load_report(report_path) assert len(workflows) == 1 wf = workflows[0] assert wf.schedule is not None @@ -542,10 +786,6 @@ def test_load_report_carries_pipeline_schedule(self, tmp_path): def test_load_report_carries_pipeline_schedule_from_ir(self, tmp_path): """Older single-pipeline reports nest schedule under ``ir.schedule``.""" - import json - - from flowx.bundler.dab_writer import _load_report - schedule_spec = { "kind": "cron", "quartz_cron_expression": "0 0 4 ? * MON,TUE,WED,THU,FRI *", @@ -570,7 +810,7 @@ def test_load_report_carries_pipeline_schedule_from_ir(self, tmp_path): report_path = tmp_path / "translation_report.json" report_path.write_text(json.dumps(report)) - workflows = _load_report(report_path) + workflows, _ = _load_report(report_path) assert len(workflows) == 1 assert workflows[0].schedule is not None assert workflows[0].schedule["quartz_cron_expression"].startswith("0 0 4") diff --git a/tests/unit/test_preparers.py b/tests/unit/test_preparers.py index d8ea058..00b0250 100644 --- a/tests/unit/test_preparers.py +++ b/tests/unit/test_preparers.py @@ -906,7 +906,7 @@ def test_run_job_round_trips_through_translation_report_json(self, tmp_path): report_path = tmp_path / "rj.json" report_path.write_text(json.dumps(pipeline_dict)) - workflows = _load_report(report_path) + workflows, _ = _load_report(report_path) bundle_dir = tmp_path / "bundle" bundle_dir.mkdir() write_bundle(workflows[0], bundle_dir) @@ -1146,7 +1146,7 @@ def test_reload_path_resolves_unresolved_on_expression(self, tmp_path): } report_path = tmp_path / "switch.json" report_path.write_text(json.dumps(pipeline_dict)) - workflows = _load_report(report_path) + workflows, _ = _load_report(report_path) bundle_dir = tmp_path / "bundle" bundle_dir.mkdir() write_bundle(workflows[0], bundle_dir) diff --git a/tests/unit/test_prereqs_writer.py b/tests/unit/test_prereqs_writer.py index f053618..abe4042 100644 --- a/tests/unit/test_prereqs_writer.py +++ b/tests/unit/test_prereqs_writer.py @@ -74,3 +74,32 @@ def test_setup_md_lists_unioned_secrets(self): assert "key_from_notebook" in md assert "scope_from_workflow" in md assert "key_from_workflow" in md + + +class TestSkippedPipelines: + """PR #6: skipped malformed report entries surface in SETUP.md instead of aborting package.""" + + def test_setup_md_lists_skipped_pipelines(self): + """SETUP.md documents every skipped pipeline so a dropped entry is not silent.""" + prereqs = build_prereqs( + notebooks=[], + tasks=[], + known_bundle_jobs=set(), + skipped_pipelines=["orphaned_pipeline", "index 3 (not a JSON object)"], + ) + md = render_setup_md(prereqs, bundle_name="test_bundle") + assert "Skipped pipelines" in md + # Names are stored bare and backtick-wrapped by the renderer (no repr quotes). + assert "- `orphaned_pipeline`" in md + assert "- `index 3 (not a JSON object)`" in md + assert "'orphaned_pipeline'" not in md + + def test_skipped_pipelines_make_prereqs_non_empty(self): + """A report with only skipped entries must still render a non-empty SETUP.md.""" + prereqs = build_prereqs( + notebooks=[], + tasks=[], + known_bundle_jobs=set(), + skipped_pipelines=["index 0"], + ) + assert not prereqs.is_empty() From 2f692ffd1857153751d18b6cc639627a64623d01 Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:08:44 -0400 Subject: [PATCH 73/77] Update flowx documentation (#16) ## Changes This PR updates the flowx documentation to improve readability and update the installation, user guide, and configuration pages. It adds typed code blocks and modifies the site theme. ### Linked issues N/A ### Tests - [x] manually tested - [ ] added unit tests - [ ] added integration tests --- .build-constraints.txt | 16 +- docs/app/docs/[[...slug]]/page.tsx | 3 +- docs/app/global.css | 2 +- .../docs/{options.mdx => configuration.mdx} | 15 +- docs/content/docs/guide.mdx | 60 +++--- docs/content/docs/index.mdx | 14 +- docs/content/docs/installation.mdx | 171 ++++++++---------- docs/content/docs/meta.json | 2 +- docs/mdx-components.tsx | 24 +++ docs/source.config.ts | 10 +- 10 files changed, 177 insertions(+), 140 deletions(-) rename docs/content/docs/{options.mdx => configuration.mdx} (94%) create mode 100644 docs/mdx-components.tsx diff --git a/.build-constraints.txt b/.build-constraints.txt index 3a6ff79..2ad60de 100644 --- a/.build-constraints.txt +++ b/.build-constraints.txt @@ -1,9 +1,9 @@ -hatchling==1.31.0 \ - --hash=sha256:6b48ad4068a482ed7239b3a8215bc55b47aad3345d58dfc94e553c5d2d46211b \ - --hash=sha256:aac80bec8b6fe35e8480f1c335be8910fa210a0e6f735a139be205dadcacb544 -packaging==26.2 \ - --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ - --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 +hatchling==1.32.0 \ + --hash=sha256:0bdbde4a52b06c37e3eca395f85a762bf0ef06fe374fd8ae429dc6be10230f5f \ + --hash=sha256:0e17c9c3b9aa7c625acc8d0f5b622f107d5049af9ecf5ada4de1aada5be7cdbc +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c # via hatchling pathspec==1.1.1 \ --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ @@ -13,6 +13,10 @@ pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 # via hatchling +tomlkit==0.15.1 \ + --hash=sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304 \ + --hash=sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97 + # via hatchling trove-classifiers==2026.6.1.19 \ --hash=sha256:ab4c4ec93cc4a4e7815fa759906e05e6bb3f2fbd92ea0f897288c6a43efd15b3 \ --hash=sha256:c5132b4b61a829d11cfbd2d72e97f20a45ed6edb95e45c5efdeb5e00836b2745 diff --git a/docs/app/docs/[[...slug]]/page.tsx b/docs/app/docs/[[...slug]]/page.tsx index 40a24cf..e15409b 100644 --- a/docs/app/docs/[[...slug]]/page.tsx +++ b/docs/app/docs/[[...slug]]/page.tsx @@ -1,6 +1,7 @@ import { source } from '@/lib/source'; import { DocsPage, DocsBody, DocsTitle, DocsDescription } from 'fumadocs-ui/page'; import { notFound } from 'next/navigation'; +import { getMDXComponents } from '@/mdx-components'; export default async function Page(props: { params: Promise<{ slug?: string[] }>; @@ -16,7 +17,7 @@ export default async function Page(props: { {page.data.title} {page.data.description} - + ); diff --git a/docs/app/global.css b/docs/app/global.css index 7408c0a..44885b1 100644 --- a/docs/app/global.css +++ b/docs/app/global.css @@ -1,5 +1,5 @@ @import 'tailwindcss'; -@import 'fumadocs-ui/css/neutral.css'; +@import 'fumadocs-ui/css/solar.css'; @import 'fumadocs-ui/css/preset.css'; @source '../node_modules/fumadocs-ui/dist/**/*.js'; diff --git a/docs/content/docs/options.mdx b/docs/content/docs/configuration.mdx similarity index 94% rename from docs/content/docs/options.mdx rename to docs/content/docs/configuration.mdx index 05e1649..b3437ce 100644 --- a/docs/content/docs/options.mdx +++ b/docs/content/docs/configuration.mdx @@ -1,6 +1,6 @@ --- -title: Translation options -description: Control flowx's translation behavior and outputs +title: Configuration +description: Control flowx's translation behavior and outputs. --- import { Callout } from 'fumadocs-ui/components/callout'; @@ -130,6 +130,17 @@ The weighted score is `sum(activity weights) + #datasets + #linked_services + #c - control-flow / parameter-setting (ForEach/If/Switch/SetVariable/AppendVariable/Filter/Wait/Until) = **2** - all other activities (Copy/Web/Lookup/etc.) = **3** (hardest) +## Translation report + +During translation, flowx writes a transient `translation_report.json` under +`/.work/` (default `./flowx_output/.work/`). It lists every activity, its +translation strategy, warnings raised during translation, and the location of any generated +artifacts. Review the translation report for any warnings, unsupported resources, or to-do +items before deploying to your Databricks workspace. + +The `package` phase prunes `.work/` after building the bundle. Pass `--keep-intermediates` to +retain it. + ## Coverage results table & dashboard (Genie Code) When running with workspace auth (Genie Code, or a configured Databricks profile) the `package` diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx index d758899..2572aa5 100644 --- a/docs/content/docs/guide.mdx +++ b/docs/content/docs/guide.mdx @@ -7,14 +7,17 @@ import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; import { Callout } from 'fumadocs-ui/components/callout'; import { Steps, Step } from 'fumadocs-ui/components/steps'; -This guide walks through a complete migration: install the flowx skills in your agentic tool, hand it a directory of Azure Data Factory JSON exports, and end up with a Databricks Asset Bundle you can deploy. +This guide walks through an end-to-end conversion of an Azure Data Factory pipeline to Lakeflow Jobs. You will install the +flowx skills in your agentic tool, hand it a directory of Azure Data Factory JSON exports, and end up with a Databricks +Asset Bundle you can deploy. ## Export pipeline templates as JSON -In the Azure Data Factory portal, open *Manage* → *ARM template* → *Export ARM template*, or use the `Get-AzDataFactoryV2Pipeline` PowerShell cmdlet to dump each pipeline definition as JSON. You'll end up with a directory tree like: +In the Azure Data Factory portal, open *Manage* → *ARM template* → *Export ARM template*. This should create a +directory with the following structure: ```text adf-export/ @@ -23,8 +26,17 @@ adf-export/ ├── linkedService/ # one JSON file per linked service └── trigger/ # one JSON file per trigger (optional) ``` + + + +You can also use the `Get-AzDataFactoryV2Pipeline` PowerShell cmdlet to programmatically dump each pipeline definition as JSON. + + + + +## Upload the exported pipelines -Upload the directory to a Unity Catalog volume (recommended) or a local path the agent can read: +Upload the exported pipelines to a Unity Catalog volume, workspace folder, or local path: ```bash databricks fs cp -r ./adf-export dbfs:/Volumes/main/default/adf_export @@ -35,23 +47,28 @@ databricks fs cp -r ./adf-export dbfs:/Volumes/main/default/adf_export ## Run the end-to-end migration -Open a fresh conversation and prompt your agent with the path to your JSON templates and a target output directory: +Prompt your agent with the path to your JSON templates and a target output directory: -> Use flowx to migrate the ADF pipelines at `/Volumes/main/default/adf_export` into a Databricks Asset Bundle at `./flowx_output/`. +```text +flowx migrate pipelines in `/Volumes/main/default/adf_export`. Save output in `./flowx_output/`. +``` -flowx will use the `migrate` skill to chain 3 other skills. All three phases write into one shared output directory (default `./flowx_output`): +Your agent will use flowx's `migrate` skill to run an end-to-end migration consisting of several phases: -1. `discover` parses every JSON file, builds an inventory, assigns a translation strategy for each resource (deterministic, agentic, or unsupported), and emits a `metadata/profile_report.csv` complexity report (one row per pipeline). +1. `discover` parses JSON files, builds an inventory, assigns a translation strategy for each resource, and creates a complexity report. 2. `convert` converts each activity to an intermediate representation. The agent will ask for confirmation before running any LLM-based translation. -3. `package` creates a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) with job configuration files, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). +3. `package` creates a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) with job configuration, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). + +To run a single step, invoke its skill directly (e.g. `@flowx-discover`) or ask your agent to run a single step. + ## Review the output -flowx writes everything into a single shared output directory (default `./flowx_output`). The generated bundle can be reviewed and modified before deployment. The layout is: +flowx writes all artifacts into a shared output folder (`./flowx_output` by default) with the following structure: ```text flowx_output/ @@ -67,21 +84,16 @@ flowx_output/ └── .work/ # transient intermediates (translation report, IR, gaps.json); pruned by prepare ``` -The bundle itself contains a top-level `databricks.yml` file with deployment targets and other variables, a `resources/` folder with job configuration, -a `src/` folder with code required to run the pipeline, and a `SETUP.md` file describing supporting resources to create. - -During translation, a transient `translation_report.json` is written under `flowx_output/.work/`. It lists every activity, its translation strategy, warnings raised during translation, and the location of any -generated artifacts. Review the translation report for any warnings, unsupported resources, or to-do items before deploying to your Databricks workspace. The `package` phase prunes `.work/` after building the bundle (pass `--keep-intermediates` to retain it). +The bundle contains: +1. `databricks.yml` file with deployment targets, global parameters, and other variables +2. `resources/` folder with job and pipeline configuration +3. `src/` folder with code required to run the pipeline +4. `SETUP.md` file that details any deployment pre-requisites -Connection strings, credentials, and other protected configuration parameters are emitted as `SecretInstruction` setup steps that require [Databricks Secrets](https://docs.databricks.com/aws/en/security/secrets/). -Run the setup scripts and populate secret values before deploying and running pipelines in your Databricks workspace. +Connection strings, credentials, and other protected configuration parameters are emitted as `SecretInstruction` steps that require [Databricks Secrets](https://docs.databricks.com/aws/en/security/secrets/). +Run the setup scripts to add any required secret values before deploying and running pipelines in your workspace. - -When running with workspace auth (e.g. Genie Code), `package` can optionally persist this run's -coverage to a Unity Catalog table — one row per pipeline stamped with a UUID `run_id`, `run_date`, -and `run_by` (`record-results`) — and install a published AI/BI coverage dashboard over that table -(`install-dashboard`). See [Configuration options](/docs/options) for details. @@ -95,9 +107,9 @@ databricks bundle validate databricks bundle deploy --target ``` - -Bundles created by flowx are standard Databricks Asset Bundles. You can target different environments, integrate with CI/CD, -or further customize the YAML before deploying. See the [Databricks Asset Bundles documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/) for more information. + +Bundles created by flowx are standard Declarative Automation Bundles. You can target different environments, integrate with CI/CD, +or further customize the YAML before deploying. See the [Declarative Automation Bundles documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/) for more information. diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index bc36385..ad294e8 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -5,22 +5,18 @@ description: What flowx is and where to start. ## Motivation -Orchestration should be treated as a first class citizen during migrations. Because orchestrators drive the execution of data pipelines, -their configuration can impact data processing results as much as the logic being orchestrated. While significant tooling exists for code -conversion and data reconciliation, migrating from legacy orchestration systems is often manual, time-consuming, and prone to risk. - flowx was created to automate migrations of data pipelines between various orchestrators. It provides a robust, tested set of capabilities to parse existing data pipeline definitions, create migration artifacts, and convert data pipeline definitions to Databricks' [Lakeflow jobs framework](https://docs.databricks.com/aws/en/jobs/). ## How flowx works -flowx is a set of agent skills and deterministic translators. Skills tell agentic tools (e.g. Databricks Genie Code, Claude Code, or any +flowx is a set of agent skills and deterministic translators. Skills tell agent tools (e.g. Databricks Genie Code, Claude Code, or any agent that supports the open [Agent Skills](https://agentskills.io/) format) how to call deterministic translators that parse, translate, and generate Databricks resources. Translation runs in three phases: -1. `discover` parses Azure Resource Manager templates (e.g. for Data Factory pipelines, datasets, linked services, and triggers) into an execution -tree, builds an inventory, and emits a per-pipeline complexity report (`metadata/profile_report.csv`). +1. `discover` parses the input pipeline configuration (e.g. ARM templates for Data Factory pipelines, datasets, linked services, and triggers) +into an execution tree, builds an inventory, and emits a per-pipeline complexity report (`metadata/profile_report.csv`). 2. `convert` processes the inventory and converts each activity into a Databricks-compatible intermediate representation. *Deterministic activities* are translated by Python handlers while *agentic activities* are handed off to an LLM-assisted translator with the right context. *Unsupported activities* are flagged as explicit gaps. @@ -33,5 +29,5 @@ All three phases write into one shared output directory (default `./flowx_output - **[Architecture](/flowx/docs/architecture)** — understand how flowx is deployed and how it translates - **[Installation](/flowx/docs/installation)** — install the flowx plugin in your agentic tool of choice. -- **[Usage Guide](/flowx/docs/guide)** — an end-to-end walkthrough from raw ADF JSON to a deployable bundle. -- **[Options](/flowx/docs/options)** — reference documenting options for customizing output when translating pipelines with flowx. +- **[Usage Guide](/flowx/docs/guide)** — run an end-to-end conversion from raw ADF JSON to a deployable bundle. +- **[Configuration](/flowx/docs/configuration)** — reference documenting options for customizing output when translating pipelines with flowx. diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index cb9f5d9..9b7ec77 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -6,69 +6,75 @@ description: Install flowx in Databricks Genie Code or a local agent harness. import { Callout } from 'fumadocs-ui/components/callout'; import { Steps, Step } from 'fumadocs-ui/components/steps'; -flowx is a set of [agent skills](https://github.com/databricks-solutions/flowx/tree/main/skills) that run from an AI coding assistant. How you install them depends on where your agent runs: +flowx is a set of [agent skills](https://github.com/databricks-solutions/flowx/tree/main/skills) that run from an AI coding assistant. Skills can be run from: -- **Databricks Genie Code** runs the phases as a hosted **MCP server** (a Databricks App). No local Python environment is involved — the app vendors flowx's code and dependencies. -- **A local agent harness (Claude Code, or any Agent Skills tool)** runs the phases from a local **Python virtual environment**, optionally exposing them over a local MCP server too. +- **Databricks Genie Code** using an MCP server hosted in Databricks Apps +- **Local agent harnesses** (e.g. Claude Code) using either a Python virtual environment or a local MCP server -Pick the matching section below and follow it end to end. +The following sections provide instructions for installing flowx using your preferred coding assistant. ## Installing flowx for Databricks Genie Code -In Genie Code the phases run as the single `flowx` tool on a Databricks App you deploy and then register as a custom MCP server. +flowx can be used directly from Databricks Genie Code. To install flowx for Genie Code, you must deploy the flowx MCP server +as a Databricks App in your workspace, then register flowx as a custom MCP server. -### Clone flowx into a shared workspace location +### Clone flowx into a shared workspace folder -Clone the repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos), under **`/Workspace/Shared`** (e.g. `/Workspace/Shared/flowx`). +First clone the repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos). - -The MCP app's service principal cannot read private `/Workspace/Users/` folders by default. Cloning into `/Workspace/Shared` keeps the repo, the deployed app source, and team access all in a location every user and the app's service principal can reach. If your workspace restricts `/Workspace/Shared`, use any other all-users location and pass it to the deployer. + +The flowx MCP app uses a service principal that cannot read private folders in `/Workspace/Users/`. Clone flowx into +`/Workspace/Shared` to allow source code access for all workspace users and service principals. If your workspace restricts +access to `/Workspace/Shared`, use an approved shared folder and configure the deployment notebook to deploy the flowx MCP +app from this shared folder. -### Copy the skills into your skills folder +### Copy the skills into your assistant folder -Genie Code picks up skills from your `.assistant/skills` folder automatically. Copy `skills/` into a user-level folder: +Genie Code references skills in your `.assistant/skills` folder automatically. These can either be installed for a single +user or for all users within a workspace. + + + +To add flowx skills for a single user, copy the skills from the flowx folder to your `.assistant` folder: ```bash -databricks workspace import-dir skills /Users//.assistant/skills +databricks workspace skills /Users//.assistant/skills ``` + -Or make flowx available to all workspace users with a workspace-level folder: + +To add flowx for all workspace users, add skills to a workspace-level folder: ```bash -databricks workspace import-dir skills /Workspace/.assistant/skills +databricks workspace skills /Workspace/.assistant/skills ``` - -Skills fire automatically when their description matches your request. To invoke one explicitly, use the `@` prefix (e.g. `@flowx-migrate translate the ADF pipelines I exported to /Volumes/main/default/adf_export`). See the [Genie Code Skills docs](https://docs.databricks.com/aws/en/genie-code/skills) for more. - - - -### Run the setup skill - -Ask your agent to *"set up the flowx environment"* (or run `@flowx-setup`). On Databricks, setup detects the environment and prepares the MCP path — it does **not** create a virtual environment, because the phases run through the deployed app rather than a local interpreter. + + -### Deploy the MCP server - -Deploy the `mcp-flowx` Databricks App from the flowx checkout. The recommended way runs entirely in the workspace, including on **serverless** compute: +### Deploy the MCP app -Open and run the **`app/deploy_app.py`** notebook. Set its `repo_root` widget to your checkout (e.g. `/Workspace/Shared/flowx`); it uses the Databricks SDK to stage a self-contained source bundle (app entrypoint plus a vendored copy of the flowx package) to `/Workspace/Shared/mcp-flowx` and create/deploy the app. The notebook prints the app URL; the MCP endpoint is **`/mcp`**. +Open and run `app/deploy_app.py`. Set the `repo_root` widget to the location of your flowx folder (e.g. `/Workspace/Shared/flowx`). +flowx uses the Databricks SDK to stage a Databricks Declarative Automation Bundle to `/Workspace/Shared/mcp-flowx` and deploy +the app. The notebook prints the app URL when deployment succeeds. The MCP endpoint is `/mcp`. - -You can instead run `bash app/deploy.sh` from a **workspace web terminal or a local machine**. The `databricks apps deploy` / `databricks sync` commands it uses require a CLI session and are **not** available from serverless notebook Python — which is why `deploy_app.py` (SDK-based) is preferred inside Genie Code. + +You can also run `bash app/deploy.sh` from a shell terminal to deploy the flowx MCP app. This requires a Databricks CLI +session and cannot be run from serverless compute. -### Grant access +### Grant access to the MCP app -- **App access:** grant **Can use** on the `mcp-flowx` app to the users or service principals that will call it (Apps UI → *Permissions*, or `databricks apps set-permissions`). -- **Data access:** grant the app's own service principal access to the catalogs, schemas, and Unity Catalog volumes the migration reads from and writes to, plus any SQL warehouse used by the reporting commands (`flowx(command="record_results")` / `flowx(command="install_dashboard")`). +Grant `CAN USE` access on the `mcp-flowx` app to any users or service principals that will use flowx. Grant the app's service +principal `READ VOLUME` access to any workspace folders or Unity Catalog volumes where your pipeline files are stored. @@ -76,125 +82,100 @@ You can instead run `bash app/deploy.sh` from a **workspace web terminal or a lo MCP servers are available in Genie Code [Agent mode](https://learn.microsoft.com/en-us/azure/databricks/genie-code/use-genie-code#modes): -1. In the Genie Code panel, click **⚙ Settings**. -2. Under **MCP Servers**, click **+ Add Server**. +1. In the Genie Code panel, click **Settings**. +2. Under **MCP Servers**, click **Add Server**. 3. Choose **Custom MCP server** and select the **`mcp-flowx`** Databricks App. 4. Click **Save**. -The single `flowx` tool is available when you use Genie Code in Agent mode. +This installs the `flowx` tool which is called when you run migrations using flowx. -A custom MCP app must be deployed in the **same workspace** and reachable at `https:///mcp`. If Genie Code cannot connect, set the app's `FLOWX_ALLOWED_ORIGINS` environment variable to your workspace URL and redeploy. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp). +A custom MCP app must be deployed in the *same workspace* and reachable at `https:///mcp`. If Genie Code cannot +connect, set the app's `FLOWX_ALLOWED_ORIGINS` environment variable to your workspace URL and redeploy. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp). ### Verify -Open the health endpoint `/` (returns `{"status":"ok"}`), or ask Genie Code *"what flowx MCP tools are available?"*. You should see the single `flowx` tool. You can now run `@flowx-migrate` (or the individual phase skills). +Ask Genie Code *"What flowx MCP tools are available?"*. You should see the single `flowx` tool. You can now run `@flowx-migrate` +and other flowx skills. + + +If Genie Code does not have access to the flowx MCP tool, open the your app's health endpoint at `/`. This should +return `{"status":"ok"}`). If you cannot access the health endpoint, verify the deployment steps and redeploy the app. + -## Installing flowx for a local agent harness (Claude Code) +## Installing flowx for Claude Code -Locally, flowx installs as a Claude Code plugin and runs its phases from a Python virtual environment. +flowx can also be installed with local agent harnesses (e.g. Claude Code) as an agent skills plugin. Skills can from a Python +virtual environment or locally-hosted MCP server. ### Install the plugin -flowx is distributed through its Claude Code marketplace. From a Claude Code session: +flowx is distributed through a Claude Code marketplace. It can be installed as a plugin or directly copied into your skills folder. + + ```bash /plugin marketplace add databricks-solutions/flowx /plugin install flowx@flowx +/reload-plugins ``` + -Then run `/reload-plugins` to activate it. - - + You can also copy the skill folders straight into your local skills directory: -```bash +```shell cp -R skills/{flowx-setup,flowx-discover,flowx-convert,flowx-package,flowx-migrate} ~/.claude/skills/ ``` - + + ### Run the setup skill -Run `/flowx:flowx-setup` (or ask *"set up the flowx environment"*) **once** before any phase. flowx's Python modules depend on third-party packages (`pyyaml`, `databricks-sdk`, `sqlglot`), so setup provisions an isolated virtual environment via `scripts/bootstrap.sh`. It will: +Run `/flowx:flowx-setup` to set up a local Python environment with flowx's required 3rd-party packages. The setup process will: 1. Check that `python3`, `pip`, and the `venv` module are available. 2. Create the virtual environment at `/.venv`. -3. Install `requirements.txt` into it with `pip`. -4. Write the resolved interpreter path to the marker file `/.migration-venv`, which the phase skills read. +3. Install packages listed in `requirements.txt` into the virtual environment with `pip`. +4. Write the interpreter path to a marker file (`/.migration-venv`) used when running the flowx skills. + The environment is created once and reused. No `uv` is required for plugin users. - - -If `python3`, `pip`, or the `venv` module are missing, the script prints a warning and exits **without** creating anything. Install Python, then re-run setup: - -* **macOS:** `brew install python` -* **Debian/Ubuntu:** `sudo apt-get install python3 python3-venv python3-pip` -* **Windows:** [python.org/downloads](https://www.python.org/downloads/) (enable "Add python.exe to PATH") - - - - -### (Optional) Run the phases over a local MCP server - -The phase skills call the venv CLI directly, so this step is optional. To instead drive the phases through MCP tools locally, install the MCP server stack into the venv and register the stdio server with your MCP client: - -```bash -PY="$(cat /.migration-venv)" -"$PY" -m pip install "mcp>=1.12" "uvicorn>=0.30" "starlette>=0.40" -PYTHONPATH="/src" "$PY" -m flowx.mcp -``` - -```json -{ - "mcpServers": { - "flowx": { - "command": "/.venv/bin/python", - "args": ["-m", "flowx.mcp"], - "env": { "PYTHONPATH": "/src" } - } - } -} -``` - - -If you prefer an installed package over `PYTHONPATH`, run `pip install -e ".[mcp]"` from the plugin root; then `python -m flowx.mcp` works without setting `PYTHONPATH`. ### Verify -Open Claude Code and ask *"What flowx skills do you have available?"*. You should see all five skills (`flowx-setup`, `flowx-discover`, `flowx-convert`, `flowx-package`, `flowx-migrate`). Invoke them with `/flowx:flowx-migrate`, `/flowx:flowx-discover`, etc. - -If you hit a `ModuleNotFoundError` while running a phase, the venv is missing or incomplete — re-run `/flowx:flowx-setup`. Every Python command the skills run uses the interpreter recorded in `/.migration-venv`, with `src/` on `PYTHONPATH`: +Open Claude Code and ask *"What flowx skills do you have available?"*. You should see a list of skills (e.g. `flowx-setup`, +`flowx-migrate`). You can now run `/flowx:flowx-migrate`, `/flowx:flowx-discover`, and other flows skills. -```bash -export PYTHONPATH="/src" -PY="$(cat /.migration-venv)" -"$PY" -m flowx.adapter inputs discover -``` + +If calling a skill raises a `ModuleNotFoundError`, the virtual environment is missing or incomplete. Ensure Python is installed +in your environment and that you have access to a Python package registry for installing depenedencies, then re-run `/flowx:flowx-setup`. + -### Other AI tools +### Other Agent tools -Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install the flowx skills: +Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install flowx: -1. Copy each skill folder (`skills/flowx-setup`, `skills/flowx-discover`, `skills/flowx-convert`, `skills/flowx-package`, `skills/flowx-migrate`) into the tool's configured skills directory, so the path contains `SKILL.md` directly. +1. Copy each skill folder (e.g. `skills/flowx-migrate`) into the tool's configured skills folder. Ensure the folder contains `SKILL.md` directly. 2. Restart the tool if it caches skill metadata at startup. -3. Follow the **local agent harness** setup above to provision the Python environment (`scripts/bootstrap.sh`). +3. Run the `flowx-setup` skill to create flowx's Python environment -If your tool expects a single Markdown file instead of a directory tree, concatenate the skills: +If your tool expects a single Markdown file instead of a directory tree, run the following command to concatenate the skills: ```bash cat skills/*/SKILL.md > flowx-skills.md diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index d253728..1e74e97 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -6,7 +6,7 @@ "architecture", "installation", "guide", - "options", + "configuration", "ai-tools-skills" ] } diff --git a/docs/mdx-components.tsx b/docs/mdx-components.tsx new file mode 100644 index 0000000..8ab259e --- /dev/null +++ b/docs/mdx-components.tsx @@ -0,0 +1,24 @@ +import { isValidElement } from 'react'; +import defaultComponents from 'fumadocs-ui/mdx'; +import type { MDXComponents } from 'mdx/types'; +import { CodeBlock, Pre } from 'fumadocs-ui/components/codeblock'; +import * as TabsComponents from 'fumadocs-ui/components/tabs'; + +function getCodeLanguage(children: React.ReactNode): string | undefined { + if (!isValidElement<{ className?: string }>(children)) return undefined; + const match = children.props.className?.match(/language-([a-z0-9]+)/i); + return match?.[1]; +} + +export function getMDXComponents(components?: MDXComponents): MDXComponents { + return { + ...defaultComponents, + ...TabsComponents, + pre: ({ ref: _ref, ...props }) => ( + + 
{props.children}
+ + ), + ...components, + }; +} diff --git a/docs/source.config.ts b/docs/source.config.ts index 0b6ee4d..02d05d6 100644 --- a/docs/source.config.ts +++ b/docs/source.config.ts @@ -1,7 +1,15 @@ import { defineDocs, defineConfig } from 'fumadocs-mdx/config'; +import { rehypeCodeDefaultOptions } from 'fumadocs-core/mdx-plugins'; export const docs = defineDocs({ dir: 'content/docs', }); -export default defineConfig(); +export default defineConfig({ + mdxOptions: { + rehypeCodeOptions: { + ...rehypeCodeDefaultOptions, + addLanguageClass: true, + }, + }, +}); From bb94c2cd53ec0a7651cf701dc85f828bb8b989ee Mon Sep 17 00:00:00 2001 From: Greg Hansen <163584195+ghanse@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:26:10 -0400 Subject: [PATCH 74/77] Fix docs links (#17) ## Changes This PR fixes broken documentation hyperlinks and makes some minor changes to the docs. ### Linked issues N/A ### Tests - [x] manually tested - [ ] added unit tests - [ ] added integration tests --- docs/content/docs/guide.mdx | 4 ++-- docs/content/docs/index.mdx | 10 +++++----- docs/content/docs/installation.mdx | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx index 2572aa5..ef17e20 100644 --- a/docs/content/docs/guide.mdx +++ b/docs/content/docs/guide.mdx @@ -8,8 +8,8 @@ import { Callout } from 'fumadocs-ui/components/callout'; import { Steps, Step } from 'fumadocs-ui/components/steps'; This guide walks through an end-to-end conversion of an Azure Data Factory pipeline to Lakeflow Jobs. You will install the -flowx skills in your agentic tool, hand it a directory of Azure Data Factory JSON exports, and end up with a Databricks -Asset Bundle you can deploy. +flowx skills in your agentic tool, hand it a directory of pipeline templates for conversion, and create a Declarative Automation +Bundle you can use to deploy a Databricks Lakeflow Job. diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index ad294e8..533a109 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -6,7 +6,7 @@ description: What flowx is and where to start. ## Motivation flowx was created to automate migrations of data pipelines between various orchestrators. It provides a robust, tested set of capabilities -to parse existing data pipeline definitions, create migration artifacts, and convert data pipeline definitions to Databricks' [Lakeflow jobs framework](https://docs.databricks.com/aws/en/jobs/). +to parse existing data pipeline definitions, create migration artifacts, and convert data pipeline definitions to [Databricks Lakeflow Jobs](https://docs.databricks.com/aws/en/jobs/). ## How flowx works @@ -27,7 +27,7 @@ All three phases write into one shared output directory (default `./flowx_output ## Next steps -- **[Architecture](/flowx/docs/architecture)** — understand how flowx is deployed and how it translates -- **[Installation](/flowx/docs/installation)** — install the flowx plugin in your agentic tool of choice. -- **[Usage Guide](/flowx/docs/guide)** — run an end-to-end conversion from raw ADF JSON to a deployable bundle. -- **[Configuration](/flowx/docs/configuration)** — reference documenting options for customizing output when translating pipelines with flowx. +- **[Architecture](/docs/architecture)** — understand how flowx is deployed and how it translates +- **[Installation](/docs/installation)** — install the flowx plugin in your agentic tool of choice. +- **[Usage Guide](/docs/guide)** — run an end-to-end conversion from raw ADF JSON to a deployable bundle. +- **[Configuration](/docs/configuration)** — reference documenting options for customizing output when translating pipelines with flowx. diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index 9b7ec77..1d2fb83 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -43,7 +43,7 @@ user or for all users within a workspace. To add flowx skills for a single user, copy the skills from the flowx folder to your `.assistant` folder: ```bash -databricks workspace skills /Users//.assistant/skills +databricks workspace import-dir skills /Users//.assistant/skills ``` @@ -102,7 +102,7 @@ Ask Genie Code *"What flowx MCP tools are available?"*. You should see the singl and other flowx skills. -If Genie Code does not have access to the flowx MCP tool, open the your app's health endpoint at `/`. This should +If Genie Code does not have access to the flowx MCP tool, open your app's health endpoint at `/`. This should return `{"status":"ok"}`). If you cannot access the health endpoint, verify the deployment steps and redeploy the app. @@ -110,8 +110,8 @@ return `{"status":"ok"}`). If you cannot access the health endpoint, verify the ## Installing flowx for Claude Code -flowx can also be installed with local agent harnesses (e.g. Claude Code) as an agent skills plugin. Skills can from a Python -virtual environment or locally-hosted MCP server. +flowx can also be installed with local agent harnesses (e.g. Claude Code) as an agent skills plugin. Skills can run commands +from a Python virtual environment or locally-hosted MCP server. @@ -157,11 +157,11 @@ The environment is created once and reused. No `uv` is required for plugin users ### Verify Open Claude Code and ask *"What flowx skills do you have available?"*. You should see a list of skills (e.g. `flowx-setup`, -`flowx-migrate`). You can now run `/flowx:flowx-migrate`, `/flowx:flowx-discover`, and other flows skills. +`flowx-migrate`). You can now run `/flowx:flowx-migrate`, `/flowx:flowx-discover`, and other flowx skills. If calling a skill raises a `ModuleNotFoundError`, the virtual environment is missing or incomplete. Ensure Python is installed -in your environment and that you have access to a Python package registry for installing depenedencies, then re-run `/flowx:flowx-setup`. +in your environment and that you have access to a Python package registry for installing dependencies, then re-run `/flowx:flowx-setup`. From 1f8edc24b32933abaf7f459299d4327f50a97c46 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Fri, 4 Sep 2026 10:25:57 -0400 Subject: [PATCH 75/77] Consolidate internal reconciliation into public flowx Adopt reconciled behavior from the internal branch: gate the global-parameter-resolution option to ADF, point IR serialization at flowx.ir_serde, and trim CI to the public unit-test suite. The lockfile-normalization step rewrites any index URL to the public package index without naming an internal host. Co-authored-by: Matthew Moorcroft --- .github/workflows/push.yml | 11 +++++------ src/flowx/adapter/session.py | 31 ++++++++++++++++++------------- tests/unit/test_adapter.py | 3 ++- tests/unit/test_translators.py | 6 +++--- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 92abc6a..8c53b68 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -28,11 +28,10 @@ jobs: version: "0.11.2" checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" python-version: "3.12" - - name: Scrub internal proxy URLs from uv.lock - run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock - - run: uv sync --frozen --extra mcp + - name: Normalize uv.lock to the public package index + run: sed -i -E 's#https://[a-zA-Z0-9._-]+/simple#https://pypi.org/simple#g' uv.lock + - run: uv sync --frozen - run: make test - - run: make integration - name: Verify requirements.txt is in sync with the lockfile run: | make requirements @@ -49,8 +48,8 @@ jobs: version: "0.11.2" checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" python-version: "3.12" - - name: Scrub internal proxy URLs from uv.lock - run: sed -i 's|https://pypi-proxy\.dev\.databricks\.com/simple|https://pypi.org/simple|g' uv.lock + - name: Normalize uv.lock to the public package index + run: sed -i -E 's#https://[a-zA-Z0-9._-]+/simple#https://pypi.org/simple#g' uv.lock - run: uv sync --frozen - run: make fmt - name: Check for formatting changes diff --git a/src/flowx/adapter/session.py b/src/flowx/adapter/session.py index 6a51e46..0990e00 100644 --- a/src/flowx/adapter/session.py +++ b/src/flowx/adapter/session.py @@ -298,7 +298,7 @@ def _discover_options(source: str) -> tuple[MigrationInputOption, ...]: def _convert_options(source: str) -> tuple[MigrationInputOption, ...]: """Convert-phase input prompts for *source*.""" spec = _SOURCE_PATH_OPTION[source] - return ( + options = [ MigrationInputOption( option_id=INPUT_INVENTORY_PATH, prompt="Path to the inventory.json from the discover phase?", @@ -313,18 +313,23 @@ def _convert_options(source: str) -> tuple[MigrationInputOption, ...]: required=True, ), _OUTPUT_DIR_OPTION, - MigrationInputOption( - option_id=INPUT_GLOBAL_PARAMETER_RESOLUTION, - prompt="How should factory global parameters be resolved?", - description=( - "Applies to every pipeline. 'literal' bakes each @pipeline().globalParameters.X value in as a " - "literal; 'bundle_variable' emits ${var.X} and declares the global as a DAB bundle variable with " - "the factory value as its default, so it can be changed at deploy time." - ), - default="literal", - required=False, - ), - ) + ] + if source == "adf": + options.append( + MigrationInputOption( + option_id=INPUT_GLOBAL_PARAMETER_RESOLUTION, + prompt="How should factory global parameters be resolved?", + description=( + "Applies to every pipeline. 'literal' bakes each @pipeline().globalParameters.X value in as a " + "literal; 'bundle_variable' emits ${var.X} and declares the global as a DAB bundle variable " + "with the factory value as its default, so it can be changed at deploy time." + ), + default="literal", + required=False, + ) + ) + return tuple(options) + _PACKAGE_OPTIONS: tuple[MigrationInputOption, ...] = ( MigrationInputOption( diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index efd326f..910ec7a 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -535,7 +535,8 @@ def test_convert_session_lists_expected_options(self): from flowx.adapter import MigrationInputSession session = MigrationInputSession(phase="convert", source="adf") - ids = [q.option_id for q in session.pending().options] + options = session.pending().options + ids = [o.option_id for o in options] assert "inventory_path" in ids assert "adf_source_path" in ids assert "global_parameter_resolution" in ids diff --git a/tests/unit/test_translators.py b/tests/unit/test_translators.py index 946264d..aaab5cd 100644 --- a/tests/unit/test_translators.py +++ b/tests/unit/test_translators.py @@ -1715,16 +1715,16 @@ def test_default_policy_is_literal(self): assert notebook_task.libraries == [{"jar": "/Volumes/my.jar"}] def test_bundle_variables_survive_report_round_trip(self): - """bundle_variables serialize via _pipeline_to_dict and reconstruct via pipeline_dict_to_ir.""" + """bundle_variables serialize via ir_serde.pipeline_to_dict and reconstruct via pipeline_dict_to_ir.""" import json from flowx.bundler.dab_writer import pipeline_dict_to_ir - from flowx.translator.engine import _pipeline_to_dict + from flowx.ir_serde import pipeline_to_dict pipeline, definitions = self._pipeline_with_global() report = translate_pipeline(pipeline, definitions, global_parameter_resolution="bundle_variable") # Full JSON round-trip, mirroring how the convert report reaches the package phase. - serialized = json.loads(json.dumps(_pipeline_to_dict(report.pipeline), default=str)) + serialized = json.loads(json.dumps(pipeline_to_dict(report.pipeline), default=str)) reconstructed, _ = pipeline_dict_to_ir(serialized) assert reconstructed.bundle_variables == report.pipeline.bundle_variables assert reconstructed.bundle_variables["libPath"]["default"] == "/Volumes/my.jar" From 95d60f7ffaf4fdfe4328e3d28b443246f9fec401 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Fri, 4 Sep 2026 11:04:21 -0400 Subject: [PATCH 76/77] Format bundler tests --- tests/unit/test_bundler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_bundler.py b/tests/unit/test_bundler.py index 0793af3..5d50144 100644 --- a/tests/unit/test_bundler.py +++ b/tests/unit/test_bundler.py @@ -207,6 +207,7 @@ def test_switch_fanout_emits_no_yaml_anchor(self, tmp_path): if dep.get("task_key") == "switch1_case_a" and dep.get("outcome") == "true": gated_on_true += 1 assert gated_on_true == 2, f"both case roots must gate on the case 'true' outcome, got {gated_on_true}" + def test_databricks_yml_sync_includes_src(self, tmp_path): """databricks.yml forces src/** into the sync set so a gitignored output dir still uploads notebooks.""" wf = _simple_workflow("my_pipeline") From 75d64fce201d01acdbb419f80ad54a7cd892eb0b Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Fri, 4 Sep 2026 11:08:06 -0400 Subject: [PATCH 77/77] Pin MCP version to <2 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b4e9582..f1ebce7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ # Install with `pip install -e .[mcp]` to host the flowx MCP tools # (locally over stdio, or as a streamable-HTTP server / Databricks App). mcp = [ - "mcp>=1.12", + "mcp>=1.12,<2", "uvicorn>=0.30", "starlette>=0.40", ] diff --git a/uv.lock b/uv.lock index 1d45f14..8ae5b65 100644 --- a/uv.lock +++ b/uv.lock @@ -371,7 +371,7 @@ yq = [ [package.metadata] requires-dist = [ { name = "databricks-sdk", specifier = ">=0.40" }, - { name = "mcp", marker = "extra == 'mcp'", specifier = ">=1.12" }, + { name = "mcp", marker = "extra == 'mcp'", specifier = ">=1.12,<2" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "sqlglot", specifier = ">=25.0" }, { name = "starlette", marker = "extra == 'mcp'", specifier = ">=0.40" },